##// END OF EJS Templates
build(deps): bumped dogpile.cache to 1.3.0
build(deps): bumped dogpile.cache to 1.3.0

File last commit:

r5173:95a4b30f default
r5247:767b4f99 default
Show More
__init__.py
573 lines | 19.2 KiB | text/x-python | PythonLexer
copyrights: updated for 2023
r5088 # Copyright (C) 2011-2023 RhodeCode GmbH
project: added all source files and assets
r1 #
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import itertools
import logging
exc-tracker: store API based exceptions and fix prefixes to no use _
r3335 import sys
api: add get_method API call....
r1417 import fnmatch
project: added all source files and assets
r1
import decorator
api: fixes and changes to always return content type in API...
r5001 import typing
project: added all source files and assets
r1 import venusian
dan
api: make jsonrpc registry ordered so doc generation can be ordered
r617 from collections import OrderedDict
project: added all source files and assets
r1 from pyramid.exceptions import ConfigurationError
from pyramid.renderers import render
from pyramid.response import Response
from pyramid.httpexceptions import HTTPNotFound
gists: use colander schema to validate input data....
r523 from rhodecode.api.exc import (
JSONRPCBaseError, JSONRPCError, JSONRPCForbidden, JSONRPCValidationError)
dan
api: attach the call context variables to request for later usage...
r1794 from rhodecode.apps._base import TemplateArgs
project: added all source files and assets
r1 from rhodecode.lib.auth import AuthUser
dan
api: attach the call context variables to request for later usage...
r1794 from rhodecode.lib.base import get_ip_addr, attach_context_attributes
exc-tracker: store API based exceptions and fix prefixes to no use _
r3335 from rhodecode.lib.exc_tracking import store_exception
api: fixes and changes to always return content type in API...
r5001 from rhodecode.lib import ext_json
project: added all source files and assets
r1 from rhodecode.lib.utils2 import safe_str
from rhodecode.lib.plugins.utils import get_plugin_settings
from rhodecode.model.db import User, UserApiKeys
log = logging.getLogger(__name__)
DEFAULT_RENDERER = 'jsonrpc_renderer'
DEFAULT_URL = '/_admin/apiv2'
api: add get_method API call....
r1417 def find_methods(jsonrpc_methods, pattern):
matches = OrderedDict()
if not isinstance(pattern, (list, tuple)):
pattern = [pattern]
for single_pattern in pattern:
for method_name, method in jsonrpc_methods.items():
if fnmatch.fnmatch(method_name, single_pattern):
matches[method_name] = method
return matches
project: added all source files and assets
r1 class ExtJsonRenderer(object):
"""
api: fixes and changes to always return content type in API...
r5001 Custom renderer that makes use of our ext_json lib
project: added all source files and assets
r1
"""
api: fixes and changes to always return content type in API...
r5001 def __init__(self):
self.serializer = ext_json.formatted_json
project: added all source files and assets
r1
def __call__(self, info):
""" Returns a plain JSON-encoded string with content-type
``application/json``. The content-type may be overridden by
setting ``request.response.content_type``."""
def _render(value, system):
request = system.get('request')
if request is not None:
response = request.response
ct = response.content_type
if ct == response.default_content_type:
response.content_type = 'application/json'
api: fixes and changes to always return content type in API...
r5001 return self.serializer(value)
project: added all source files and assets
r1
return _render
def jsonrpc_response(request, result):
rpc_id = getattr(request, 'rpc_id', None)
ret_value = ''
if rpc_id:
api: fixes and changes to always return content type in API...
r5001 ret_value = {'id': rpc_id, 'result': result, 'error': None}
project: added all source files and assets
r1
# fetch deprecation warnings, and store it inside results
deprecation = getattr(request, 'rpc_deprecation', None)
if deprecation:
ret_value['DEPRECATION_WARNING'] = deprecation
raw_body = render(DEFAULT_RENDERER, ret_value, request=request)
api: fixes and changes to always return content type in API...
r5001 content_type = 'application/json'
content_type_header = 'Content-Type'
headers = {
content_type_header: content_type
}
return Response(
body=raw_body,
content_type=content_type,
headerlist=[(k, v) for k, v in headers.items()]
)
project: added all source files and assets
r1
api: modernize code for python3
r5092 def jsonrpc_error(request, message, retid=None, code: int | None = None, headers: dict | None = None):
project: added all source files and assets
r1 """
Generate a Response object with a JSON-RPC error body
api: fixes and changes to always return content type in API...
r5001 """
headers = headers or {}
content_type = 'application/json'
content_type_header = 'Content-Type'
if content_type_header not in headers:
headers[content_type_header] = content_type
project: added all source files and assets
r1
err_dict = {'id': retid, 'result': None, 'error': message}
api: fixes and changes to always return content type in API...
r5001 raw_body = render(DEFAULT_RENDERER, err_dict, request=request)
exception-tracker: send exc id headers on failed API calls for tracking errors that server generated.
r4112
project: added all source files and assets
r1 return Response(
api: fixes and changes to always return content type in API...
r5001 body=raw_body,
project: added all source files and assets
r1 status=code,
api: fixes and changes to always return content type in API...
r5001 content_type=content_type,
headerlist=[(k, v) for k, v in headers.items()]
project: added all source files and assets
r1 )
def exception_view(exc, request):
rpc_id = getattr(request, 'rpc_id', None)
if isinstance(exc, JSONRPCError):
exceptions: skip extracting exception by deprecated .message attribute
r5104 fault_message = safe_str(exc)
project: added all source files and assets
r1 log.debug('json-rpc error rpc_id:%s "%s"', rpc_id, fault_message)
gists: use colander schema to validate input data....
r523 elif isinstance(exc, JSONRPCValidationError):
colander_exc = exc.colander_exception
code: added more logging, and some notes
r1300 # TODO(marcink): think maybe of nicer way to serialize errors ?
gists: use colander schema to validate input data....
r523 fault_message = colander_exc.asdict()
api: fixed potential crash when returning error response using JSON objects that fail to parse.
r3316 log.debug('json-rpc colander error rpc_id:%s "%s"', rpc_id, fault_message)
project: added all source files and assets
r1 elif isinstance(exc, JSONRPCForbidden):
fault_message = 'Access was denied to this resource.'
log.warning('json-rpc forbidden call rpc_id:%s "%s"', rpc_id, fault_message)
elif isinstance(exc, HTTPNotFound):
method = request.rpc_method
log.debug('json-rpc method `%s` not found in list of '
'api calls: %s, rpc_id:%s',
api: fixes and changes to always return content type in API...
r5001 method, list(request.registry.jsonrpc_methods.keys()), rpc_id)
api: add get_method API call....
r1417
similar = 'none'
try:
api: fixes and changes to always return content type in API...
r5001 similar_paterns = [f'*{x}*' for x in method.split('_')]
api: add get_method API call....
r1417 similar_found = find_methods(
request.registry.jsonrpc_methods, similar_paterns)
similar = ', '.join(similar_found.keys()) or similar
except Exception:
# make the whole above block safe
pass
tests: refactor code to use a single test url generator
r5173 fault_message = f"No such method: {method}. Similar methods: {similar}"
exc-tracker: store API based exceptions and fix prefixes to no use _
r3335 else:
fault_message = 'undefined error'
exc_info = exc.exc_info()
store_exception(id(exc_info), exc_info, prefix='rhodecode-api')
project: added all source files and assets
r1
metrics: expose more metrics via statsd...
r4801 statsd = request.registry.statsd
if statsd:
api: modernize code for python3
r5092 exc_type = f"{exc.__class__.__module__}.{exc.__class__.__name__}"
metrics: expose exc_type in consistent format
r4808 statsd.incr('rhodecode_exception_total',
api: modernize code for python3
r5092 tags=["exc_source:api", f"type:{exc_type}"])
metrics: expose more metrics via statsd...
r4801
project: added all source files and assets
r1 return jsonrpc_error(request, fault_message, rpc_id)
def request_view(request):
"""
Main request handling method. It handles all logic to call a specific
exposed method
"""
core: fixed cython compat inspect that caused some API calls to not work correctly.
r4184 # cython compatible inspect
from rhodecode.config.patches import inspect_getargspec
inspect = inspect_getargspec()
project: added all source files and assets
r1
# check if we can find this session using api_key, get_by_auth_token
# search not expired tokens only
try:
api-events: fix a case events were called from API and we couldn't fetch registered user....
r1431 api_user = User.get_by_auth_token(request.rpc_api_key)
project: added all source files and assets
r1
api-events: fix a case events were called from API and we couldn't fetch registered user....
r1431 if api_user is None:
project: added all source files and assets
r1 return jsonrpc_error(
request, retid=request.rpc_id, message='Invalid API KEY')
api-events: fix a case events were called from API and we couldn't fetch registered user....
r1431 if not api_user.active:
project: added all source files and assets
r1 return jsonrpc_error(
request, retid=request.rpc_id,
message='Request from this user not allowed')
# check if we are allowed to use this IP
auth_u = AuthUser(
api-events: fix a case events were called from API and we couldn't fetch registered user....
r1431 api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr)
project: added all source files and assets
r1 if not auth_u.ip_allowed:
return jsonrpc_error(
request, retid=request.rpc_id,
api: modernize code for python3
r5092 message='Request from IP:{} not allowed'.format(
request.rpc_ip_addr))
project: added all source files and assets
r1 else:
logging: use lazy parameter evaluation in log calls.
r3061 log.info('Access for IP:%s allowed', request.rpc_ip_addr)
project: added all source files and assets
r1
api-events: fix a case events were called from API and we couldn't fetch registered user....
r1431 # register our auth-user
request.rpc_user = auth_u
api: fixes and changes to always return content type in API...
r5001 request.environ['rc_auth_user_id'] = str(auth_u.user_id)
api-events: fix a case events were called from API and we couldn't fetch registered user....
r1431
project: added all source files and assets
r1 # now check if token is valid for API
auth-tokens: updated logic of authentication to a common shared user method.
r1421 auth_token = request.rpc_api_key
token_match = api_user.authenticate_by_token(
auth-tokens: disable authenticating by builtin token.
r1477 auth_token, roles=[UserApiKeys.ROLE_API])
auth-tokens: updated logic of authentication to a common shared user method.
r1421 invalid_token = not token_match
project: added all source files and assets
r1
auth-tokens: updated logic of authentication to a common shared user method.
r1421 log.debug('Checking if API KEY is valid with proper role')
if invalid_token:
project: added all source files and assets
r1 return jsonrpc_error(
request, retid=request.rpc_id,
auth-tokens: updated logic of authentication to a common shared user method.
r1421 message='API KEY invalid or, has bad role for an API call')
project: added all source files and assets
r1
auth-tokens: updated logic of authentication to a common shared user method.
r1421 except Exception:
project: added all source files and assets
r1 log.exception('Error on API AUTH')
return jsonrpc_error(
request, retid=request.rpc_id, message='Invalid API KEY')
method = request.rpc_method
func = request.registry.jsonrpc_methods[method]
# now that we have a method, add request._req_params to
# self.kargs and dispatch control to WGIController
api: fixes and changes to always return content type in API...
r5001
project: added all source files and assets
r1 argspec = inspect.getargspec(func)
arglist = argspec[0]
api: fixes and changes to always return content type in API...
r5001 defs = argspec[3] or []
defaults = [type(a) for a in defs]
default_empty = type(NotImplemented)
project: added all source files and assets
r1
# kw arguments required by this method
python3: fixed various code issues...
r4973 func_kwargs = dict(itertools.zip_longest(
project: added all source files and assets
r1 reversed(arglist), reversed(defaults), fillvalue=default_empty))
# This attribute will need to be first param of a method that uses
# api_key, which is translated to instance of user at that name
user_var = 'apiuser'
request_var = 'request'
for arg in [user_var, request_var]:
if arg not in arglist:
return jsonrpc_error(
request,
retid=request.rpc_id,
message='This method [%s] does not support '
'required parameter `%s`' % (func.__name__, arg))
# get our arglist and check if we provided them as args
for arg, default in func_kwargs.items():
if arg in [user_var, request_var]:
# user_var and request_var are pre-hardcoded parameters and we
# don't need to do any translation
continue
# skip the required param check if it's default value is
# NotImplementedType (default_empty)
if default == default_empty and arg not in request.rpc_params:
return jsonrpc_error(
request,
retid=request.rpc_id,
message=('Missing non optional `%s` arg in JSON DATA' % arg)
)
code: added more logging, and some notes
r1300 # sanitize extra passed arguments
api: fixes and changes to always return content type in API...
r5001 for k in list(request.rpc_params.keys()):
project: added all source files and assets
r1 if k not in func_kwargs:
del request.rpc_params[k]
call_params = request.rpc_params
call_params.update({
'request': request,
'apiuser': auth_u
})
dan
api: attach the call context variables to request for later usage...
r1794
# register some common functions for usage
exception-tracker: send exc id headers on failed API calls for tracking errors that server generated.
r4112 attach_context_attributes(TemplateArgs(), request, request.rpc_user.user_id)
dan
api: attach the call context variables to request for later usage...
r1794
metrics: use prom metrics, and added some additional metrics
r4803 statsd = request.registry.statsd
project: added all source files and assets
r1 try:
ret_value = func(**call_params)
metrics: use prom metrics, and added some additional metrics
r4803 resp = jsonrpc_response(request, ret_value)
if statsd:
statsd.incr('rhodecode_api_call_success_total')
return resp
project: added all source files and assets
r1 except JSONRPCBaseError:
raise
except Exception:
celery: handle pyramid/pylons context better when running async tasks.
r1340 log.exception('Unhandled exception occurred on api call: %s', func)
exc-tracker: store API based exceptions and fix prefixes to no use _
r3335 exc_info = sys.exc_info()
exception-tracker: send exc id headers on failed API calls for tracking errors that server generated.
r4112 exc_id, exc_type_name = store_exception(
id(exc_info), exc_info, prefix='rhodecode-api')
api: fixes and changes to always return content type in API...
r5001 error_headers = {
'RhodeCode-Exception-Id': str(exc_id),
'RhodeCode-Exception-Type': str(exc_type_name)
}
metrics: use prom metrics, and added some additional metrics
r4803 err_resp = jsonrpc_error(
exception-tracker: send exc id headers on failed API calls for tracking errors that server generated.
r4112 request, retid=request.rpc_id, message='Internal server error',
headers=error_headers)
metrics: use prom metrics, and added some additional metrics
r4803 if statsd:
statsd.incr('rhodecode_api_call_fail_total')
return err_resp
project: added all source files and assets
r1
def setup_request(request):
"""
Parse a JSON-RPC request body. It's used inside the predicates method
to validate and bootstrap requests for usage in rpc calls.
We need to raise JSONRPCError here if we want to return some errors back to
user.
"""
code: added more logging, and some notes
r1300
project: added all source files and assets
r1 log.debug('Executing setup request: %r', request)
request.rpc_ip_addr = get_ip_addr(request.environ)
code: added more logging, and some notes
r1300 # TODO(marcink): deprecate GET at some point
project: added all source files and assets
r1 if request.method not in ['POST', 'GET']:
log.debug('unsupported request method "%s"', request.method)
raise JSONRPCError(
'unsupported request method "%s". Please use POST' % request.method)
if 'CONTENT_LENGTH' not in request.environ:
log.debug("No Content-Length")
raise JSONRPCError("Empty body, No Content-Length in request")
else:
length = request.environ['CONTENT_LENGTH']
log.debug('Content-Length: %s', length)
if length == 0:
log.debug("Content-Length is 0")
raise JSONRPCError("Content-Length is 0")
raw_body = request.body
api: don't log full API params as the upload ones can be too much for logging
r3993 log.debug("Loading JSON body now")
project: added all source files and assets
r1 try:
api: fixes and changes to always return content type in API...
r5001 json_body = ext_json.json.loads(raw_body)
project: added all source files and assets
r1 except ValueError as e:
# catch JSON errors Here
modernize: updates for python3
r5095 raise JSONRPCError(f"JSON parse error ERR:{e} RAW:{raw_body!r}")
project: added all source files and assets
r1
request.rpc_id = json_body.get('id')
request.rpc_method = json_body.get('method')
# check required base parameters
try:
api_key = json_body.get('api_key')
if not api_key:
api_key = json_body.get('auth_token')
if not api_key:
raise KeyError('api_key or auth_token')
code: added more logging, and some notes
r1300 # TODO(marcink): support passing in token in request header
project: added all source files and assets
r1 request.rpc_api_key = api_key
request.rpc_id = json_body['id']
request.rpc_method = json_body['method']
request.rpc_params = json_body['args'] \
if isinstance(json_body['args'], dict) else {}
api: don't log full API params as the upload ones can be too much for logging
r3993 log.debug('method: %s, params: %.10240r', request.rpc_method, request.rpc_params)
project: added all source files and assets
r1 except KeyError as e:
api: fixes and changes to always return content type in API...
r5001 raise JSONRPCError(f'Incorrect JSON data. Missing {e}')
project: added all source files and assets
r1
log.debug('setup complete, now handling method:%s rpcid:%s',
request.rpc_method, request.rpc_id, )
class RoutePredicate(object):
def __init__(self, val, config):
self.val = val
def text(self):
api: replaced formatting with fstrings
r5016 return f'jsonrpc route = {self.val}'
project: added all source files and assets
r1
phash = text
def __call__(self, info, request):
if self.val:
# potentially setup and bootstrap our call
setup_request(request)
# Always return True so that even if it isn't a valid RPC it
# will fall through to the underlaying handlers like notfound_view
return True
class NotFoundPredicate(object):
def __init__(self, val, config):
self.val = val
api: add get_method API call....
r1417 self.methods = config.registry.jsonrpc_methods
project: added all source files and assets
r1
def text(self):
api: replaced formatting with fstrings
r5016 return f'jsonrpc method not found = {self.val}'
project: added all source files and assets
r1
phash = text
def __call__(self, info, request):
return hasattr(request, 'rpc_method')
class MethodPredicate(object):
def __init__(self, val, config):
self.method = val
def text(self):
api: replaced formatting with fstrings
r5016 return f'jsonrpc method = {self.method}'
project: added all source files and assets
r1
phash = text
def __call__(self, context, request):
# we need to explicitly return False here, so pyramid doesn't try to
# execute our view directly. We need our main handler to execute things
return getattr(request, 'rpc_method') == self.method
def add_jsonrpc_method(config, view, **kwargs):
# pop the method name
method = kwargs.pop('method', None)
if method is None:
raise ConfigurationError(
exc-tracker: store API based exceptions and fix prefixes to no use _
r3335 'Cannot register a JSON-RPC method without specifying the "method"')
project: added all source files and assets
r1
# we define custom predicate, to enable to detect conflicting methods,
# those predicates are kind of "translation" from the decorator variables
# to internal predicates names
kwargs['jsonrpc_method'] = method
# register our view into global view store for validation
config.registry.jsonrpc_methods[method] = view
# we're using our main request_view handler, here, so each method
# has a unified handler for itself
config.add_view(request_view, route_name='apiv2', **kwargs)
class jsonrpc_method(object):
"""
decorator that works similar to @add_view_config decorator,
but tailored for our JSON RPC
"""
venusian = venusian # for testing injection
def __init__(self, method=None, **kwargs):
self.method = method
self.kwargs = kwargs
def __call__(self, wrapped):
kwargs = self.kwargs.copy()
kwargs['method'] = self.method or wrapped.__name__
depth = kwargs.pop('_depth', 0)
def callback(context, name, ob):
config = context.config.with_package(info.module)
config.add_jsonrpc_method(view=ob, **kwargs)
info = venusian.attach(wrapped, callback, category='pyramid',
depth=depth + 1)
if info.scope == 'class':
# ensure that attr is set if decorating a class method
kwargs.setdefault('attr', wrapped.__name__)
kwargs['_info'] = info.codeinfo # fbo action_method
return wrapped
class jsonrpc_deprecated_method(object):
"""
Marks method as deprecated, adds log.warning, and inject special key to
the request variable to mark method as deprecated.
Also injects special docstring that extract_docs will catch to mark
method as deprecated.
:param use_method: specify which method should be used instead of
the decorated one
Use like::
@jsonrpc_method()
@jsonrpc_deprecated_method(use_method='new_func', deprecated_at_version='3.0.0')
def old_func(request, apiuser, arg1, arg2):
...
"""
def __init__(self, use_method, deprecated_at_version):
self.use_method = use_method
self.deprecated_at_version = deprecated_at_version
self.deprecated_msg = ''
def __call__(self, func):
self.deprecated_msg = 'Please use method `{method}` instead.'.format(
method=self.use_method)
docstring = """\n
.. deprecated:: {version}
{deprecation_message}
{original_docstring}
"""
func.__doc__ = docstring.format(
version=self.deprecated_at_version,
deprecation_message=self.deprecated_msg,
original_docstring=func.__doc__)
return decorator.decorator(self.__wrapper, func)
def __wrapper(self, func, *fargs, **fkwargs):
log.warning('DEPRECATED API CALL on function %s, please '
'use `%s` instead', func, self.use_method)
# alter function docstring to mark as deprecated, this is picked up
# via fabric file that generates API DOC.
result = func(*fargs, **fkwargs)
request = fargs[0]
request.rpc_deprecation = 'DEPRECATED METHOD ' + self.deprecated_msg
return result
application: not use config.scan(), and replace all @add_view decorator into a explicit add_view call for faster app start.
r4610 def add_api_methods(config):
from rhodecode.api.views import (
deprecated_api, gist_api, pull_request_api, repo_api, repo_group_api,
server_api, search_api, testing_api, user_api, user_group_api)
config.scan('rhodecode.api.views')
project: added all source files and assets
r1 def includeme(config):
plugin_module = 'rhodecode.api'
plugin_settings = get_plugin_settings(
plugin_module, config.registry.settings)
if not hasattr(config.registry, 'jsonrpc_methods'):
dan
api: make jsonrpc registry ordered so doc generation can be ordered
r617 config.registry.jsonrpc_methods = OrderedDict()
project: added all source files and assets
r1
# match filter by given method only
code: added more logging, and some notes
r1300 config.add_view_predicate('jsonrpc_method', MethodPredicate)
exc-tracker: store API based exceptions and fix prefixes to no use _
r3335 config.add_view_predicate('jsonrpc_method_not_found', NotFoundPredicate)
project: added all source files and assets
r1
api: fixes and changes to always return content type in API...
r5001 config.add_renderer(DEFAULT_RENDERER, ExtJsonRenderer())
project: added all source files and assets
r1 config.add_directive('add_jsonrpc_method', add_jsonrpc_method)
config.add_route_predicate(
'jsonrpc_call', RoutePredicate)
config.add_route(
'apiv2', plugin_settings.get('url', DEFAULT_URL), jsonrpc_call=True)
# register some exception handling view
config.add_view(exception_view, context=JSONRPCBaseError)
config.add_notfound_view(exception_view, jsonrpc_method_not_found=True)
application: not use config.scan(), and replace all @add_view decorator into a explicit add_view call for faster app start.
r4610
add_api_methods(config)