# HG changeset patch # User RhodeCode Admin # Date 2023-01-31 19:31:36 # Node ID d49e3bd84c1f956020962c27fd4d30459c075875 # Parent 72f064effa9ab786c9cc992eef8dce3735f5afc7 requests: cleaned / unified way of handling requests generation from non-web scope. - cli - celery workers diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -381,15 +381,6 @@ def includeme(config, auth_resources=Non config.add_subscriber(write_js_routes_if_enabled, pyramid.events.ApplicationCreated) - # request custom methods - config.add_request_method( - 'rhodecode.lib.partial_renderer.get_partial_renderer', - 'get_partial_renderer') - - config.add_request_method( - 'rhodecode.lib.request_counter.get_request_counter', - 'request_count') - # Set the authorization policy. authz_policy = ACLAuthorizationPolicy() config.set_authorization_policy(authz_policy) diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -35,7 +35,6 @@ from paste.httpexceptions import HTTPUna from paste.httpheaders import WWW_AUTHENTICATE, AUTHORIZATION import rhodecode -from rhodecode.apps._base import TemplateArgs from rhodecode.authentication.base import VCS_TYPE from rhodecode.lib import auth, utils2 from rhodecode.lib import helpers as h @@ -460,12 +459,7 @@ def attach_context_attributes(context, r context.rhodecode_user = request.rpc_user # attach the whole call context to the request - # use set_call_context method if request has it - # sometimes in Celery context requests is "different" - if hasattr(request, 'set_call_context'): - request.set_call_context(context) - else: - request.call_context = context + request.set_call_context(context) def get_auth_user(request): @@ -577,9 +571,9 @@ def add_events_routes(config): pattern='/_hovercard/commit/{repo_name}/{commit_id}') -def bootstrap_config(request): +def bootstrap_config(request, registry_name='RcTestRegistry'): import pyramid.testing - registry = pyramid.testing.Registry('RcTestRegistry') + registry = pyramid.testing.Registry(registry_name) config = pyramid.testing.setUp(registry=registry, request=request) @@ -594,42 +588,24 @@ def bootstrap_config(request): def bootstrap_request(**kwargs): - import pyramid.testing + """ + Returns a thin version of Request Object that is used in non-web context like testing/celery + """ - class TestRequest(pyramid.testing.DummyRequest): + import pyramid.testing + from rhodecode.lib.request import ThinRequest as _ThinRequest + + class ThinRequest(_ThinRequest): application_url = kwargs.pop('application_url', 'http://example.com') host = kwargs.pop('host', 'example.com:80') domain = kwargs.pop('domain', 'example.com') - def translate(self, msg): - return msg - - def plularize(self, singular, plural, n): - return singular - - def get_partial_renderer(self, tmpl_name): - - from rhodecode.lib.partial_renderer import get_partial_renderer - return get_partial_renderer(request=self, tmpl_name=tmpl_name) - - _call_context = TemplateArgs() - _call_context.visual = TemplateArgs() - _call_context.visual.show_sha_length = 12 - _call_context.visual.show_revision_number = True - - @property - def call_context(self): - return self._call_context - - def set_call_context(self, new_context): - self._call_context = new_context - - class TestDummySession(pyramid.testing.DummySession): + class ThinSession(pyramid.testing.DummySession): def save(*arg, **kw): pass - request = TestRequest(**kwargs) - request.session = TestDummySession() + request = ThinRequest(**kwargs) + request.session = ThinSession() return request diff --git a/rhodecode/lib/celerylib/loader.py b/rhodecode/lib/celerylib/loader.py --- a/rhodecode/lib/celerylib/loader.py +++ b/rhodecode/lib/celerylib/loader.py @@ -45,7 +45,7 @@ import rhodecode from rhodecode.lib.auth import AuthUser from rhodecode.lib.celerylib.utils import parse_ini_vars, ping_db from rhodecode.lib.ext_json import json -from rhodecode.lib.pyramid_utils import bootstrap, setup_logging, prepare_request +from rhodecode.lib.pyramid_utils import bootstrap, setup_logging from rhodecode.lib.utils2 import str2bool from rhodecode.model import meta @@ -271,10 +271,20 @@ class RequestContextTask(Task): link=None, link_error=None, shadow=None, **options): """ queue the job to run (we are in web request context here) """ - req = get_current_request() + req = self.app.conf['PYRAMID_REQUEST'] or get_current_request() + log.debug('Running Task with class: %s. Request Class: %s', self.__class__, req.__class__) + proxy_data = getattr(self.request, 'rhodecode_proxy_data', None) + log.debug('celery proxy data:%r', proxy_data) + + user_id = None + ip_addr = None + if proxy_data: + user_id = proxy_data['auth_user']['user_id'] + ip_addr = proxy_data['auth_user']['ip_addr'] + # web case if hasattr(req, 'user'): ip_addr = req.user.ip_addr @@ -285,14 +295,18 @@ class RequestContextTask(Task): ip_addr = req.rpc_user.ip_addr user_id = req.rpc_user.user_id else: - raise Exception( - 'Unable to fetch required data from request: {}. \n' - 'This task is required to be executed from context of ' - 'request in a webapp. Task: {}'.format( - repr(req), - self + if user_id and ip_addr: + log.debug('Using data from celery proxy user') + + else: + raise Exception( + 'Unable to fetch required data from request: {}. \n' + 'This task is required to be executed from context of ' + 'request in a webapp. Task: {}'.format( + repr(req), + self.__class__ + ) ) - ) if req: # we hook into kwargs since it is the only way to pass our data to @@ -311,19 +325,3 @@ class RequestContextTask(Task): return super(RequestContextTask, self).apply_async( args, kwargs, task_id, producer, link, link_error, shadow, **options) - - def __call__(self, *args, **kwargs): - """ rebuild the context and then run task on celery worker """ - - proxy_data = getattr(self.request, 'rhodecode_proxy_data', None) - if not proxy_data: - return super(RequestContextTask, self).__call__(*args, **kwargs) - - log.debug('using celery proxy data to run task: %r', proxy_data) - # re-inject and register threadlocals for proper routing support - request = prepare_request(proxy_data['environ']) - request.user = AuthUser(user_id=proxy_data['auth_user']['user_id'], - ip_addr=proxy_data['auth_user']['ip_addr']) - - return super(RequestContextTask, self).__call__(*args, **kwargs) - diff --git a/rhodecode/lib/pyramid_utils.py b/rhodecode/lib/pyramid_utils.py --- a/rhodecode/lib/pyramid_utils.py +++ b/rhodecode/lib/pyramid_utils.py @@ -37,24 +37,7 @@ def get_app_config(ini_path): return appconfig('config:{}'.format(ini_path), relative_to=os.getcwd()) -class BootstrappedRequest(Request): - """ - Special version of Request Which has some available methods like in pyramid. - Some code (used for template rendering) requires this, and we unsure it's present. - """ - - def translate(self, msg): - return msg - - def plularize(self, singular, plural, n): - return singular - - def get_partial_renderer(self, tmpl_name): - from rhodecode.lib.partial_renderer import get_partial_renderer - return get_partial_renderer(request=self, tmpl_name=tmpl_name) - - -def bootstrap(config_uri, request=None, options=None, env=None): +def bootstrap(config_uri, options=None, env=None): if env: os.environ.update(env) @@ -65,12 +48,7 @@ def bootstrap(config_uri, request=None, except (configparser.NoSectionError, configparser.NoOptionError): pass - request = request or BootstrappedRequest.blank('/', base_url=base_url) + request = Request.blank('/', base_url=base_url) return pyramid_bootstrap(config_uri, request=request, options=options) - -def prepare_request(environ): - request = Request.blank('/', environ=environ) - prepare(request) # set pyramid threadlocal request - return request diff --git a/rhodecode/lib/request.py b/rhodecode/lib/request.py --- a/rhodecode/lib/request.py +++ b/rhodecode/lib/request.py @@ -19,13 +19,24 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ from uuid import uuid4 +import pyramid.testing from pyramid.decorator import reify from pyramid.request import Request as _Request from rhodecode.translation import _ as tsf +from rhodecode.lib.utils2 import StrictAttributeDict + + +class TemplateArgs(StrictAttributeDict): + pass -class Request(_Request): +# Base Class with DummyMethods, testing / CLI scripts +class RequestBase(object): _req_id_bucket = list() + _call_context = TemplateArgs() + _call_context.visual = TemplateArgs() + _call_context.visual.show_sha_length = 12 + _call_context.visual.show_revision_number = True @reify def req_id(self): @@ -38,6 +49,47 @@ class Request(_Request): def req_id_records_init(self): self._req_id_bucket = list() + def translate(self, *args, **kwargs): + raise NotImplementedError() + + def plularize(self, *args, **kwargs): + raise NotImplementedError() + + def get_partial_renderer(self, tmpl_name): + raise NotImplementedError() + + @property + def call_context(self): + return self._call_context + + def set_call_context(self, new_context): + self._call_context = new_context + + +# for thin non-web/cli etc +class ThinRequest(RequestBase, pyramid.testing.DummyRequest): + + def translate(self, msg): + return msg + + def plularize(self, singular, plural, n): + return singular + + def get_partial_renderer(self, tmpl_name): + from rhodecode.lib.partial_renderer import get_partial_renderer + return get_partial_renderer(request=self, tmpl_name=tmpl_name) + + +# for real-web-based +class RealRequest(RequestBase, _Request): + def get_partial_renderer(self, tmpl_name): + from rhodecode.lib.partial_renderer import get_partial_renderer + return get_partial_renderer(request=self, tmpl_name=tmpl_name) + + def request_count(self): + from rhodecode.lib.request_counter import get_request_counter + return get_request_counter() + def plularize(self, *args, **kwargs): return self.localizer.pluralize(*args, **kwargs) @@ -48,3 +100,10 @@ class Request(_Request): return localizer.translate(tsf(*_args, **_kwargs)) return auto_translate(*args, **kwargs) + + +class Request(RealRequest): + """ + This is the main request object used in web-context + """ + pass diff --git a/rhodecode/lib/request_counter.py b/rhodecode/lib/request_counter.py --- a/rhodecode/lib/request_counter.py +++ b/rhodecode/lib/request_counter.py @@ -21,7 +21,7 @@ counter = 0 -def get_request_counter(request): +def get_request_counter(): global counter counter += 1 return counter