requests: cleaned / unified way of handling requests generation from non-web scope.

- cli
- celery workers

rhodecode/config/middleware.py

@@ -381,15 +381,6 b' def includeme(config, auth_resources=Non'
         config.add_subscriber(write_js_routes_if_enabled,
                               pyramid.events.ApplicationCreated)
 
-    # request custom methods
-    config.add_request_method(
-        'rhodecode.lib.partial_renderer.get_partial_renderer',
-        'get_partial_renderer')
-
-    config.add_request_method(
-        'rhodecode.lib.request_counter.get_request_counter',
-        'request_count')
-
     # Set the authorization policy.
     authz_policy = ACLAuthorizationPolicy()
     config.set_authorization_policy(authz_policy)

rhodecode/lib/base.py

@@ -35,7 +35,6 b' from paste.httpexceptions import HTTPUna'
 from paste.httpheaders import WWW_AUTHENTICATE, AUTHORIZATION
 
 import rhodecode
-from rhodecode.apps._base import TemplateArgs
 from rhodecode.authentication.base import VCS_TYPE
 from rhodecode.lib import auth, utils2
 from rhodecode.lib import helpers as h
@@ -460,12 +459,7 b' def attach_context_attributes(context, r'
         context.rhodecode_user = request.rpc_user
 
     # attach the whole call context to the request
-    # use set_call_context method if request has it
-    # sometimes in Celery context requests is "different"
-    if hasattr(request, 'set_call_context'):
-        request.set_call_context(context)
-    else:
-        request.call_context = context
+    request.set_call_context(context)
 
 
 def get_auth_user(request):
@@ -577,9 +571,9 b' def add_events_routes(config):'
                      pattern='/_hovercard/commit/{repo_name}/{commit_id}')
 
 
-def bootstrap_config(request):
+def bootstrap_config(request, registry_name='RcTestRegistry'):
     import pyramid.testing
-    registry = pyramid.testing.Registry('RcTestRegistry')
+    registry = pyramid.testing.Registry(registry_name)
 
     config = pyramid.testing.setUp(registry=registry, request=request)
 
@@ -594,42 +588,24 b' def bootstrap_config(request):'
 
 
 def bootstrap_request(**kwargs):
-    import pyramid.testing
+    """
+    Returns a thin version of Request Object that is used in non-web context like testing/celery
+    """
 
-    class TestRequest(pyramid.testing.DummyRequest):
+    import pyramid.testing
+    from rhodecode.lib.request import ThinRequest as _ThinRequest
+
+    class ThinRequest(_ThinRequest):
         application_url = kwargs.pop('application_url', 'http://example.com')
         host = kwargs.pop('host', 'example.com:80')
         domain = kwargs.pop('domain', 'example.com')
 
-        def translate(self, msg):
-            return msg
-
-        def plularize(self, singular, plural, n):
-            return singular
-
-        def get_partial_renderer(self, tmpl_name):
-
-            from rhodecode.lib.partial_renderer import get_partial_renderer
-            return get_partial_renderer(request=self, tmpl_name=tmpl_name)
-
-        _call_context = TemplateArgs()
-        _call_context.visual = TemplateArgs()
-        _call_context.visual.show_sha_length = 12
-        _call_context.visual.show_revision_number = True
-
-        @property
-        def call_context(self):
-            return self._call_context
-
-        def set_call_context(self, new_context):
-            self._call_context = new_context
-
-    class TestDummySession(pyramid.testing.DummySession):
+    class ThinSession(pyramid.testing.DummySession):
         def save(*arg, **kw):
             pass
 
-    request = TestRequest(**kwargs)
-    request.session = TestDummySession()
+    request = ThinRequest(**kwargs)
+    request.session = ThinSession()
 
     return request
 

rhodecode/lib/celerylib/loader.py

@@ -45,7 +45,7 b' import rhodecode'
 from rhodecode.lib.auth import AuthUser
 from rhodecode.lib.celerylib.utils import parse_ini_vars, ping_db
 from rhodecode.lib.ext_json import json
-from rhodecode.lib.pyramid_utils import bootstrap, setup_logging, prepare_request
+from rhodecode.lib.pyramid_utils import bootstrap, setup_logging
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.model import meta
 
@@ -271,10 +271,20 b' class RequestContextTask(Task):'
                     link=None, link_error=None, shadow=None, **options):
         """ queue the job to run (we are in web request context here) """
 
-        req = get_current_request()
+        req = self.app.conf['PYRAMID_REQUEST'] or get_current_request()
+
         log.debug('Running Task with class: %s. Request Class: %s',
                   self.__class__, req.__class__)
 
+        proxy_data = getattr(self.request, 'rhodecode_proxy_data', None)
+        log.debug('celery proxy data:%r', proxy_data)
+
+        user_id = None
+        ip_addr = None
+        if proxy_data:
+           user_id = proxy_data['auth_user']['user_id']
+           ip_addr = proxy_data['auth_user']['ip_addr']
+
         # web case
         if hasattr(req, 'user'):
             ip_addr = req.user.ip_addr
@@ -285,14 +295,18 b' class RequestContextTask(Task):'
             ip_addr = req.rpc_user.ip_addr
             user_id = req.rpc_user.user_id
         else:
-            raise Exception(
-                'Unable to fetch required data from request: {}. \n'
-                'This task is required to be executed from context of '
-                'request in a webapp. Task: {}'.format(
-                    repr(req),
-                    self
+            if user_id and ip_addr:
+                log.debug('Using data from celery proxy user')
+
+            else:
+                raise Exception(
+                    'Unable to fetch required data from request: {}. \n'
+                    'This task is required to be executed from context of '
+                    'request in a webapp. Task: {}'.format(
+                        repr(req),
+                        self.__class__
+                    )
                 )
-            )
 
         if req:
             # we hook into kwargs since it is the only way to pass our data to
@@ -311,19 +325,3 b' class RequestContextTask(Task):'
 
         return super(RequestContextTask, self).apply_async(
             args, kwargs, task_id, producer, link, link_error, shadow, **options)
-
-    def __call__(self, *args, **kwargs):
-        """ rebuild the context and then run task on celery worker """
-
-        proxy_data = getattr(self.request, 'rhodecode_proxy_data', None)
-        if not proxy_data:
-            return super(RequestContextTask, self).__call__(*args, **kwargs)
-
-        log.debug('using celery proxy data to run task: %r', proxy_data)
-        # re-inject and register threadlocals for proper routing support
-        request = prepare_request(proxy_data['environ'])
-        request.user = AuthUser(user_id=proxy_data['auth_user']['user_id'],
-                                ip_addr=proxy_data['auth_user']['ip_addr'])
-
-        return super(RequestContextTask, self).__call__(*args, **kwargs)
-

rhodecode/lib/pyramid_utils.py

@@ -37,24 +37,7 b' def get_app_config(ini_path):'
     return appconfig('config:{}'.format(ini_path), relative_to=os.getcwd())
 
 
-class BootstrappedRequest(Request):
-    """
-    Special version of Request Which has some available methods like in pyramid.
-    Some code (used for template rendering) requires this, and we unsure it's present.
-    """
-
-    def translate(self, msg):
-        return msg
-
-    def plularize(self, singular, plural, n):
-        return singular
-
-    def get_partial_renderer(self, tmpl_name):
-        from rhodecode.lib.partial_renderer import get_partial_renderer
-        return get_partial_renderer(request=self, tmpl_name=tmpl_name)
-
-
-def bootstrap(config_uri, request=None, options=None, env=None):
+def bootstrap(config_uri, options=None, env=None):
     if env:
         os.environ.update(env)
 
@@ -65,12 +48,7 b' def bootstrap(config_uri, request=None, '
     except (configparser.NoSectionError, configparser.NoOptionError):
         pass
 
-    request = request or BootstrappedRequest.blank('/', base_url=base_url)
+    request = Request.blank('/', base_url=base_url)
 
     return pyramid_bootstrap(config_uri, request=request, options=options)
 
-
-def prepare_request(environ):
-    request = Request.blank('/', environ=environ)
-    prepare(request)  # set pyramid threadlocal request
-    return request

rhodecode/lib/request.py

@@ -19,13 +19,24 b''
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from uuid import uuid4
+import pyramid.testing
 from pyramid.decorator import reify
 from pyramid.request import Request as _Request
 from rhodecode.translation import _ as tsf
+from rhodecode.lib.utils2 import StrictAttributeDict
+
+
+class TemplateArgs(StrictAttributeDict):
+    pass
 
 
-class Request(_Request):
+# Base Class with DummyMethods, testing / CLI scripts
+class RequestBase(object):
     _req_id_bucket = list()
+    _call_context = TemplateArgs()
+    _call_context.visual = TemplateArgs()
+    _call_context.visual.show_sha_length = 12
+    _call_context.visual.show_revision_number = True
 
     @reify
     def req_id(self):
@@ -38,6 +49,47 b' class Request(_Request):'
     def req_id_records_init(self):
         self._req_id_bucket = list()
 
+    def translate(self, *args, **kwargs):
+        raise NotImplementedError()
+
+    def plularize(self, *args, **kwargs):
+        raise NotImplementedError()
+
+    def get_partial_renderer(self, tmpl_name):
+        raise NotImplementedError()
+
+    @property
+    def call_context(self):
+        return self._call_context
+
+    def set_call_context(self, new_context):
+        self._call_context = new_context
+
+
+# for thin non-web/cli etc
+class ThinRequest(RequestBase, pyramid.testing.DummyRequest):
+
+    def translate(self, msg):
+        return msg
+
+    def plularize(self, singular, plural, n):
+        return singular
+
+    def get_partial_renderer(self, tmpl_name):
+        from rhodecode.lib.partial_renderer import get_partial_renderer
+        return get_partial_renderer(request=self, tmpl_name=tmpl_name)
+
+
+# for real-web-based
+class RealRequest(RequestBase, _Request):
+    def get_partial_renderer(self, tmpl_name):
+        from rhodecode.lib.partial_renderer import get_partial_renderer
+        return get_partial_renderer(request=self, tmpl_name=tmpl_name)
+
+    def request_count(self):
+        from rhodecode.lib.request_counter import get_request_counter
+        return get_request_counter()
+
     def plularize(self, *args, **kwargs):
         return self.localizer.pluralize(*args, **kwargs)
 
@@ -48,3 +100,10 b' class Request(_Request):'
             return localizer.translate(tsf(*_args, **_kwargs))
 
         return auto_translate(*args, **kwargs)
+
+
+class Request(RealRequest):
+    """
+    This is the main request object used in web-context
+    """
+    pass

rhodecode/lib/request_counter.py

@@ -21,7 +21,7 b''
 counter = 0
 
 
-def get_request_counter(request):
+def get_request_counter():
     global counter
     counter += 1
     return counter