exc-tracking: fixed API calls with new exceptions store engine

rhodecode/api/views/server_api.py

@@ -1,418 +1,424 b''
 # Copyright (C) 2011-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import itertools
 import base64
 
 from rhodecode.api import (
     jsonrpc_method, JSONRPCError, JSONRPCForbidden, find_methods)
 
 from rhodecode.api.utils import (
     Optional, OAttr, has_superadmin_permission, get_user_or_error)
 from rhodecode.lib.utils import repo2db_mapper
 from rhodecode.lib import system_info
 from rhodecode.lib import user_sessions
 from rhodecode.lib import exc_tracking
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.model.db import UserIpMap
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel
 from rhodecode.apps.file_store import utils
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
     FileOverSizeException
 
 log = logging.getLogger(__name__)
 
 
 @jsonrpc_method()
 def get_server_info(request, apiuser):
     """
     Returns the |RCE| server information.
 
     This includes the running version of |RCE| and all installed
     packages. This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'modules': [<module name>,...]
         'py_version': <python version>,
         'platform': <platform type>,
         'rhodecode_version': <rhodecode version>
       }
       error :  null
     """
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     server_info = ScmModel().get_server_info(request.environ)
     # rhodecode-index requires those
 
     server_info['index_storage'] = server_info['search']['value']['location']
     server_info['storage'] = server_info['storage']['value']['path']
 
     return server_info
 
 
 @jsonrpc_method()
 def get_repo_store(request, apiuser):
     """
     Returns the |RCE| repository storage information.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'modules': [<module name>,...]
         'py_version': <python version>,
         'platform': <platform type>,
         'rhodecode_version': <rhodecode version>
       }
       error :  null
     """
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     path = VcsSettingsModel().get_repos_location()
     return {"path": path}
 
 
 @jsonrpc_method()
 def get_ip(request, apiuser, userid=Optional(OAttr('apiuser'))):
     """
     Displays the IP Address as seen from the |RCE| server.
 
     * This command displays the IP Address, as well as all the defined IP
       addresses for the specified user. If the ``userid`` is not set, the
       data returned is for the user calling the method.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from |authtoken|.
     :type apiuser: AuthUser
     :param userid: Sets the userid for which associated IP Address data
         is returned.
     :type userid: Optional(str or int)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result : {
                      "server_ip_addr": "<ip_from_clien>",
                      "user_ips": [
                                     {
                                        "ip_addr": "<ip_with_mask>",
                                        "ip_range": ["<start_ip>", "<end_ip>"],
                                     },
                                     ...
                                  ]
         }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     userid = Optional.extract(userid, evaluate_locals=locals())
     userid = getattr(userid, 'user_id', userid)
 
     user = get_user_or_error(userid)
     ips = UserIpMap.query().filter(UserIpMap.user == user).all()
     return {
         'server_ip_addr': request.rpc_ip_addr,
         'user_ips': ips
     }
 
 
 @jsonrpc_method()
 def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
     """
     Triggers a rescan of the specified repositories.
 
     * If the ``remove_obsolete`` option is set, it also deletes repositories
       that are found in the database but not on the file system, so called
       "clean zombies".
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param remove_obsolete: Deletes repositories from the database that
         are not found on the filesystem.
     :type remove_obsolete: Optional(``True`` | ``False``)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'added': [<added repository name>,...]
         'removed': [<removed repository name>,...]
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred during rescan repositories action'
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     try:
         rm_obsolete = Optional.extract(remove_obsolete)
         added, removed = repo2db_mapper(ScmModel().repo_scan(),
                                         remove_obsolete=rm_obsolete)
         return {'added': added, 'removed': removed}
     except Exception:
         log.exception('Failed to run repo rescann')
         raise JSONRPCError(
             'Error occurred during rescan repositories action'
         )
 
 
 @jsonrpc_method()
 def cleanup_sessions(request, apiuser, older_then=Optional(60)):
     """
     Triggers a session cleanup action.
 
     If the ``older_then`` option is set, only sessions that hasn't been
     accessed in the given number of days will be removed.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param older_then: Deletes session that hasn't been accessed
         in given number of days.
     :type older_then: Optional(int)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result: {
         "backend": "<type of backend>",
         "sessions_removed": <number_of_removed_sessions>
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred during session cleanup'
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     older_then = safe_int(Optional.extract(older_then)) or 60
     older_than_seconds = 60 * 60 * 24 * older_then
 
     config = system_info.rhodecode_config().get_value()['value']['config']
     session_model = user_sessions.get_session_handler(
         config.get('beaker.session.type', 'memory'))(config)
 
     backend = session_model.SESSION_TYPE
     try:
         cleaned = session_model.clean_sessions(
             older_than_seconds=older_than_seconds)
         return {'sessions_removed': cleaned, 'backend': backend}
     except user_sessions.CleanupCommand as msg:
         return {'cleanup_command': str(msg), 'backend': backend}
     except Exception as e:
         log.exception('Failed session cleanup')
         raise JSONRPCError(
             'Error occurred during session cleanup'
         )
 
 
 @jsonrpc_method()
 def get_method(request, apiuser, pattern=Optional('*')):
     """
     Returns list of all available API methods. By default match pattern
     os "*" but any other pattern can be specified. eg *comment* will return
     all methods with comment inside them. If just single method is matched
     returned data will also include method specification
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param pattern: pattern to match method names against
     :type pattern: Optional("*")
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": [
         "changeset_comment",
         "comment_pull_request",
         "comment_commit"
       ]
       error :  null
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": [
         "comment_commit",
         {
           "apiuser": "<RequiredType>",
           "comment_type": "<Optional:u'note'>",
           "commit_id": "<RequiredType>",
           "message": "<RequiredType>",
           "repoid": "<RequiredType>",
           "request": "<RequiredType>",
           "resolves_comment_id": "<Optional:None>",
           "status": "<Optional:None>",
           "userid": "<Optional:<OptionalAttr:apiuser>>"
         }
       ]
       error :  null
     """
     from rhodecode.config.patches import inspect_getargspec
     inspect = inspect_getargspec()
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     pattern = Optional.extract(pattern)
 
     matches = find_methods(request.registry.jsonrpc_methods, pattern)
 
     args_desc = []
     matches_keys = list(matches.keys())
     if len(matches_keys) == 1:
         func = matches[matches_keys[0]]
 
         argspec = inspect.getargspec(func)
         arglist = argspec[0]
         defaults = list(map(repr, argspec[3] or []))
 
         default_empty = '<RequiredType>'
 
         # kw arguments required by this method
         func_kwargs = dict(itertools.zip_longest(
             reversed(arglist), reversed(defaults), fillvalue=default_empty))
         args_desc.append(func_kwargs)
 
     return matches_keys + args_desc
 
 
 @jsonrpc_method()
 def store_exception(request, apiuser, exc_data_json, prefix=Optional('rhodecode')):
     """
     Stores sent exception inside the built-in exception tracker in |RCE| server.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     :param exc_data_json: JSON data with exception e.g
         {"exc_traceback": "Value `1` is not allowed", "exc_type_name": "ValueError"}
     :type exc_data_json: JSON data
 
     :param prefix: prefix for error type, e.g 'rhodecode', 'vcsserver', 'rhodecode-tools'
     :type prefix: Optional("rhodecode")
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": {
         "exc_id": 139718459226384,
         "exc_url": "http://localhost:8080/_admin/settings/exceptions/139718459226384"
       }
       error :  null
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     prefix = Optional.extract(prefix)
     exc_id = exc_tracking.generate_id()
 
     try:
         exc_data = json.loads(exc_data_json)
     except Exception:
         log.error('Failed to parse JSON: %r', exc_data_json)
         raise JSONRPCError('Failed to parse JSON data from exc_data_json field. '
                            'Please make sure it contains a valid JSON.')
 
     try:
         exc_traceback = exc_data['exc_traceback']
         exc_type_name = exc_data['exc_type_name']
+        exc_value = ''
     except KeyError as err:
-        raise JSONRPCError('Missing exc_traceback, or exc_type_name '
-                           'in exc_data_json field. Missing: {}'.format(err))
+        raise JSONRPCError(
+            f'Missing exc_traceback, or exc_type_name '
+            f'in exc_data_json field. Missing: {err}')
+
+    class ExcType:
+        __name__ = exc_type_name
+
+    exc_info = (ExcType(), exc_value, exc_traceback)
 
     exc_tracking._store_exception(
-        exc_id=exc_id, exc_traceback=exc_traceback,
-        exc_type_name=exc_type_name, prefix=prefix)
+        exc_id=exc_id, exc_info=exc_info, prefix=prefix)
 
     exc_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=exc_id)
     return {'exc_id': exc_id, 'exc_url': exc_url}

rhodecode/lib/exc_tracking.py

@@ -1,315 +1,320 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import io
 import os
 import time
 import sys
 import datetime
 import msgpack
 import logging
 import traceback
 import tempfile
 import glob
 
 log = logging.getLogger(__name__)
 
 # NOTE: Any changes should be synced with exc_tracking at vcsserver.lib.exc_tracking
 global_prefix = "rhodecode"
 exc_store_dir_name = "rc_exception_store_v1"
 
 
 def exc_serialize(exc_id, tb, exc_type, extra_data=None):
     data = {
         "version": "v1",
         "exc_id": exc_id,
         "exc_utc_date": datetime.datetime.utcnow().isoformat(),
         "exc_timestamp": repr(time.time()),
         "exc_message": tb,
         "exc_type": exc_type,
     }
     if extra_data:
         data.update(extra_data)
     return msgpack.packb(data), data
 
 
 def exc_unserialize(tb):
     return msgpack.unpackb(tb)
 
 
 _exc_store = None
 
 
 def maybe_send_exc_email(exc_id, exc_type_name, send_email):
     from pyramid.threadlocal import get_current_request
     import rhodecode as app
 
     request = get_current_request()
 
     if send_email is None:
         # NOTE(marcink): read app config unless we specify explicitly
         send_email = app.CONFIG.get("exception_tracker.send_email", False)
 
     mail_server = app.CONFIG.get("smtp_server") or None
     send_email = send_email and mail_server
     if send_email and request:
         try:
             send_exc_email(request, exc_id, exc_type_name)
         except Exception:
             log.exception("Failed to send exception email")
             exc_info = sys.exc_info()
             store_exception(id(exc_info), exc_info, send_email=False)
 
 
 def send_exc_email(request, exc_id, exc_type_name):
     import rhodecode as app
     from rhodecode.apps._base import TemplateArgs
     from rhodecode.lib.utils2 import aslist
     from rhodecode.lib.celerylib import run_task, tasks
     from rhodecode.lib.base import attach_context_attributes
     from rhodecode.model.notification import EmailNotificationModel
 
     recipients = aslist(app.CONFIG.get("exception_tracker.send_email_recipients", ""))
     log.debug("Sending Email exception to: `%s`", recipients or "all super admins")
 
     # NOTE(marcink): needed for email template rendering
     user_id = None
     if hasattr(request, "user"):
         user_id = request.user.user_id
     attach_context_attributes(TemplateArgs(), request, user_id=user_id, is_api=True)
 
     email_kwargs = {
         "email_prefix": app.CONFIG.get("exception_tracker.email_prefix", "")
         or "[RHODECODE ERROR]",
         "exc_url": request.route_url(
             "admin_settings_exception_tracker_show", exception_id=exc_id
         ),
         "exc_id": exc_id,
         "exc_type_name": exc_type_name,
         "exc_traceback": read_exception(exc_id, prefix=None),
     }
 
     (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
         EmailNotificationModel.TYPE_EMAIL_EXCEPTION, **email_kwargs
     )
 
     run_task(tasks.send_email, recipients, subject, email_body_plaintext, email_body)
 
 
 def get_exc_store():
     """
     Get and create exception store if it's not existing
     """
     global _exc_store
 
     if _exc_store is not None:
         # quick global cache
         return _exc_store
 
     import rhodecode as app
 
     exc_store_dir = (
         app.CONFIG.get("exception_tracker.store_path", "") or tempfile.gettempdir()
     )
     _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
 
     _exc_store_path = os.path.abspath(_exc_store_path)
     if not os.path.isdir(_exc_store_path):
         os.makedirs(_exc_store_path)
         log.debug("Initializing exceptions store at %s", _exc_store_path)
         _exc_store = _exc_store_path
 
     return _exc_store_path
 
 
 def get_detailed_tb(exc_info):
     try:
         from pip._vendor.rich import (
             traceback as rich_tb,
             scope as rich_scope,
             console as rich_console,
         )
     except ImportError:
         try:
             from rich import (
                 traceback as rich_tb,
                 scope as rich_scope,
                 console as rich_console,
             )
         except ImportError:
             return None
 
     console = rich_console.Console(width=160, file=io.StringIO())
 
     exc = rich_tb.Traceback.extract(*exc_info, show_locals=True)
 
     tb_rich = rich_tb.Traceback(
         trace=exc,
         width=160,
         extra_lines=3,
         theme=None,
         word_wrap=False,
         show_locals=False,
         max_frames=100,
     )
 
     # last_stack = exc.stacks[-1]
     # last_frame = last_stack.frames[-1]
     # if last_frame and last_frame.locals:
     #     console.print(
     #         rich_scope.render_scope(
     #             last_frame.locals,
     #             title=f'{last_frame.filename}:{last_frame.lineno}'))
 
     console.print(tb_rich)
     formatted_locals = console.file.getvalue()
 
     return formatted_locals
 
 
 def get_request_metadata(request=None) -> dict:
     request_metadata = {}
     if not request:
         from pyramid.threadlocal import get_current_request
 
         request = get_current_request()
 
     # NOTE(marcink): store request information into exc_data
     if request:
         request_metadata["client_address"] = getattr(request, "client_addr", "")
         request_metadata["user_agent"] = getattr(request, "user_agent", "")
         request_metadata["method"] = getattr(request, "method", "")
         request_metadata["url"] = getattr(request, "url", "")
     return request_metadata
 
 
-def format_exc(exc_info):
+def format_exc(exc_info, use_detailed_tb=True):
     exc_type, exc_value, exc_traceback = exc_info
     tb = "++ TRACEBACK ++\n\n"
+    if isinstance(exc_traceback, str):
+        tb += exc_traceback
+        use_detailed_tb = False
+    else:
     tb += "".join(traceback.format_exception(exc_type, exc_value, exc_traceback, None))
 
+    if use_detailed_tb:
     locals_tb = get_detailed_tb(exc_info)
     if locals_tb:
         tb += f"\n+++ DETAILS +++\n\n{locals_tb}\n" ""
     return tb
 
 
 def _store_exception(exc_id, exc_info, prefix, request_path='', send_email=None):
     """
     Low level function to store exception in the exception tracker
     """
 
     extra_data = {}
     extra_data.update(get_request_metadata())
 
     exc_type, exc_value, exc_traceback = exc_info
     tb = format_exc(exc_info)
 
     exc_type_name = exc_type.__name__
     exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name, extra_data=extra_data)
 
     exc_pref_id = f"{exc_id}_{prefix}_{org_data['exc_timestamp']}"
     exc_store_path = get_exc_store()
     if not os.path.isdir(exc_store_path):
         os.makedirs(exc_store_path)
     stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
     with open(stored_exc_path, "wb") as f:
         f.write(exc_data)
     log.debug("Stored generated exception %s as: %s", exc_id, stored_exc_path)
 
     if request_path:
         log.error(
             'error occurred handling this request.\n'
             'Path: `%s`, %s',
             request_path, tb)
 
     maybe_send_exc_email(exc_id, exc_type_name, send_email)
 
 
 def store_exception(exc_id, exc_info, prefix=global_prefix, request_path='', send_email=None):
     """
     Example usage::
 
         exc_info = sys.exc_info()
         store_exception(id(exc_info), exc_info)
     """
 
     try:
         exc_type = exc_info[0]
         exc_type_name = exc_type.__name__
 
         _store_exception(
             exc_id=exc_id, exc_info=exc_info, prefix=prefix, request_path=request_path,
             send_email=send_email
         )
         return exc_id, exc_type_name
     except Exception:
         log.exception("Failed to store exception `%s` information", exc_id)
         # there's no way this can fail, it will crash server badly if it does.
         pass
 
 
 def _find_exc_file(exc_id, prefix=global_prefix):
     exc_store_path = get_exc_store()
     if prefix:
         exc_id = f"{exc_id}_{prefix}"
     else:
         # search without a prefix
         exc_id = f"{exc_id}"
 
     found_exc_id = None
     matches = glob.glob(os.path.join(exc_store_path, exc_id) + "*")
     if matches:
         found_exc_id = matches[0]
 
     return found_exc_id
 
 
 def _read_exception(exc_id, prefix):
     exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
     if exc_id_file_path:
         with open(exc_id_file_path, "rb") as f:
             return exc_unserialize(f.read())
     else:
         log.debug("Exception File `%s` not found", exc_id_file_path)
     return None
 
 
 def read_exception(exc_id, prefix=global_prefix):
     try:
         return _read_exception(exc_id=exc_id, prefix=prefix)
     except Exception:
         log.exception("Failed to read exception `%s` information", exc_id)
         # there's no way this can fail, it will crash server badly if it does.
     return None
 
 
 def delete_exception(exc_id, prefix=global_prefix):
     try:
         exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
         if exc_id_file_path:
             os.remove(exc_id_file_path)
 
     except Exception:
         log.exception("Failed to remove exception `%s` information", exc_id)
         # there's no way this can fail, it will crash server badly if it does.
         pass
 
 
 def generate_id():
     return id(object())