# HG changeset patch # User RhodeCode Admin # Date 2023-01-21 13:38:37 # Node ID 6f1f534f494c4231e9e6a4d4c58a8eb7ad2a550b # Parent 916d130a1c433a6deb952616fb1769984f102489 statsd/audit-logs: cleanup push/pull user agent code. - add normalization - add user-agent tag into statsd call so we can track protocol/vcs type usage - internal merge also sends in repo type diff --git a/rhodecode/apps/ssh_support/lib/backends/base.py b/rhodecode/apps/ssh_support/lib/backends/base.py --- a/rhodecode/apps/ssh_support/lib/backends/base.py +++ b/rhodecode/apps/ssh_support/lib/backends/base.py @@ -31,6 +31,7 @@ log = logging.getLogger(__name__) class VcsServer(object): + repo_user_agent = None # set in child classes _path = None # set executable path for hg/git/svn binary backend = None # set in child classes tunnel = None # subprocess handling tunnel @@ -106,7 +107,7 @@ class VcsServer(object): 'make_lock': None, 'locked_by': [None, None], 'server_url': None, - 'user_agent': 'ssh-user-agent', + 'user_agent': '{}/ssh-user-agent'.format(self.repo_user_agent), 'hooks': ['push', 'pull'], 'hooks_module': 'rhodecode.lib.hooks_daemon', 'is_shadow_repo': False, diff --git a/rhodecode/apps/ssh_support/lib/backends/git.py b/rhodecode/apps/ssh_support/lib/backends/git.py --- a/rhodecode/apps/ssh_support/lib/backends/git.py +++ b/rhodecode/apps/ssh_support/lib/backends/git.py @@ -59,6 +59,7 @@ class GitTunnelWrapper(object): class GitServer(VcsServer): backend = 'git' + repo_user_agent = 'git' def __init__(self, store, ini_path, repo_name, repo_mode, user, user_permissions, config, env): diff --git a/rhodecode/apps/ssh_support/lib/backends/hg.py b/rhodecode/apps/ssh_support/lib/backends/hg.py --- a/rhodecode/apps/ssh_support/lib/backends/hg.py +++ b/rhodecode/apps/ssh_support/lib/backends/hg.py @@ -97,6 +97,7 @@ class MercurialTunnelWrapper(object): class MercurialServer(VcsServer): backend = 'hg' + repo_user_agent = 'mercurial' cli_flags = ['phases', 'largefiles', 'extensions', 'experimental', 'hooks'] def __init__(self, store, ini_path, repo_name, user, user_permissions, config, env): diff --git a/rhodecode/apps/ssh_support/lib/backends/svn.py b/rhodecode/apps/ssh_support/lib/backends/svn.py --- a/rhodecode/apps/ssh_support/lib/backends/svn.py +++ b/rhodecode/apps/ssh_support/lib/backends/svn.py @@ -222,6 +222,7 @@ class SubversionTunnelWrapper(object): class SubversionServer(VcsServer): backend = 'svn' + repo_user_agent = 'svn' def __init__(self, store, ini_path, repo_name, user, user_permissions, config, env): diff --git a/rhodecode/apps/ssh_support/tests/test_server_git.py b/rhodecode/apps/ssh_support/tests/test_server_git.py --- a/rhodecode/apps/ssh_support/tests/test_server_git.py +++ b/rhodecode/apps/ssh_support/tests/test_server_git.py @@ -148,7 +148,7 @@ class TestGitServer(object): 'hooks_module': 'rhodecode.lib.hooks_daemon', 'check_branch_perms': False, 'detect_force_push': False, - 'user_agent': u'ssh-user-agent', + 'user_agent': u'git/ssh-user-agent', 'SSH': True, 'SSH_PERMISSIONS': 'repository.admin', } diff --git a/rhodecode/lib/hooks_base.py b/rhodecode/lib/hooks_base.py --- a/rhodecode/lib/hooks_base.py +++ b/rhodecode/lib/hooks_base.py @@ -30,7 +30,7 @@ import rhodecode from rhodecode import events from rhodecode.lib import helpers as h from rhodecode.lib import audit_logger -from rhodecode.lib.utils2 import safe_str +from rhodecode.lib.utils2 import safe_str, user_agent_normalizer from rhodecode.lib.exceptions import ( HTTPLockedRC, HTTPBranchProtected, UserCreationError) from rhodecode.model.db import Repository, User @@ -222,8 +222,9 @@ def post_pull(extras): statsd = StatsdClient.statsd if statsd: - statsd.incr('rhodecode_pull_total') - + statsd.incr('rhodecode_pull_total', tags=[ + 'user-agent:{}'.format(user_agent_normalizer(extras.user_agent)), + ]) output = '' # make lock is a tri state False, True, None. We only make lock on True if extras.make_lock is True and not is_shadow_repo(extras): @@ -271,7 +272,9 @@ def post_push(extras): statsd = StatsdClient.statsd if statsd: - statsd.incr('rhodecode_push_total') + statsd.incr('rhodecode_push_total', tags=[ + 'user-agent:{}'.format(user_agent_normalizer(extras.user_agent)), + ]) # Propagate to external components. output = '' diff --git a/rhodecode/lib/utils2.py b/rhodecode/lib/utils2.py --- a/rhodecode/lib/utils2.py +++ b/rhodecode/lib/utils2.py @@ -1146,3 +1146,20 @@ def retry(func=None, exception=Exception return func(*args, **kwargs) return wrapper + + +def user_agent_normalizer(user_agent_raw): + log = logging.getLogger('rhodecode.user_agent_normalizer') + ua = (user_agent_raw or '').strip().lower() + + try: + if 'mercurial/proto-1.0' in ua: + ua = ua.replace('mercurial/proto-1.0', '') + ua = ua.replace('(', '').replace(')', '').strip() + ua = ua.replace('mercurial ', 'mercurial/') + elif ua.startswith('git'): + pass + except Exception: + log.exception('Failed to parse scm user-agent') + + return ua diff --git a/rhodecode/model/pull_request.py b/rhodecode/model/pull_request.py --- a/rhodecode/model/pull_request.py +++ b/rhodecode/model/pull_request.py @@ -939,8 +939,9 @@ class PullRequestModel(BaseModel): return commit_ids def merge_repo(self, pull_request, user, extras): + repo_type = pull_request.source_repo.repo_type log.debug("Merging pull request %s", pull_request.pull_request_id) - extras['user_agent'] = 'internal-merge' + extras['user_agent'] = '{}/internal-merge'.format(repo_type) merge_state = self._merge_pull_request(pull_request, user, extras) if merge_state.executed: log.debug("Merge was successful, updating the pull request comments.")