statsd/audit-logs: cleanup push/pull user agent code.

- add normalization
- add user-agent tag into statsd call so we can track protocol/vcs type usage
- internal merge also sends in repo type

rhodecode/apps/ssh_support/lib/backends/base.py

@@ -31,6 +31,7 b' log = logging.getLogger(__name__)'
 
 
 class VcsServer(object):
+    repo_user_agent = None  # set in child classes
     _path = None  # set executable path for hg/git/svn binary
     backend = None  # set in child classes
     tunnel = None  # subprocess handling tunnel
@@ -106,7 +107,7 b' class VcsServer(object):'
             'make_lock': None,
             'locked_by': [None, None],
             'server_url': None,
-            'user_agent': 'ssh-user-agent',
+            'user_agent': '{}/ssh-user-agent'.format(self.repo_user_agent),
             'hooks': ['push', 'pull'],
             'hooks_module': 'rhodecode.lib.hooks_daemon',
             'is_shadow_repo': False,

rhodecode/apps/ssh_support/lib/backends/git.py

@@ -59,6 +59,7 b' class GitTunnelWrapper(object):'
 
 class GitServer(VcsServer):
     backend = 'git'
+    repo_user_agent = 'git'
 
     def __init__(self, store, ini_path, repo_name, repo_mode,
                  user, user_permissions, config, env):

rhodecode/apps/ssh_support/lib/backends/hg.py

@@ -97,6 +97,7 b' class MercurialTunnelWrapper(object):'
 
 class MercurialServer(VcsServer):
     backend = 'hg'
+    repo_user_agent = 'mercurial'
     cli_flags = ['phases', 'largefiles', 'extensions', 'experimental', 'hooks']
 
     def __init__(self, store, ini_path, repo_name, user, user_permissions, config, env):

rhodecode/apps/ssh_support/lib/backends/svn.py

@@ -222,6 +222,7 b' class SubversionTunnelWrapper(object):'
 
 class SubversionServer(VcsServer):
     backend = 'svn'
+    repo_user_agent = 'svn'
 
     def __init__(self, store, ini_path, repo_name,
                  user, user_permissions, config, env):

rhodecode/apps/ssh_support/tests/test_server_git.py

@@ -148,7 +148,7 b' class TestGitServer(object):'
             'hooks_module': 'rhodecode.lib.hooks_daemon',
             'check_branch_perms': False,
             'detect_force_push': False,
-            'user_agent': u'ssh-user-agent',
+            'user_agent': u'git/ssh-user-agent',
             'SSH': True,
             'SSH_PERMISSIONS': 'repository.admin',
         }

rhodecode/lib/hooks_base.py

@@ -30,7 +30,7 b' import rhodecode'
 from rhodecode import events
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
-from rhodecode.lib.utils2 import safe_str
+from rhodecode.lib.utils2 import safe_str, user_agent_normalizer
 from rhodecode.lib.exceptions import (
     HTTPLockedRC, HTTPBranchProtected, UserCreationError)
 from rhodecode.model.db import Repository, User
@@ -222,8 +222,9 b' def post_pull(extras):'
 
     statsd = StatsdClient.statsd
     if statsd:
-        statsd.incr('rhodecode_pull_total')
-
+        statsd.incr('rhodecode_pull_total', tags=[
+            'user-agent:{}'.format(user_agent_normalizer(extras.user_agent)),
+        ])
     output = ''
     # make lock is a tri state False, True, None. We only make lock on True
     if extras.make_lock is True and not is_shadow_repo(extras):
@@ -271,7 +272,9 b' def post_push(extras):'
 
     statsd = StatsdClient.statsd
     if statsd:
-        statsd.incr('rhodecode_push_total')
+        statsd.incr('rhodecode_push_total', tags=[
+            'user-agent:{}'.format(user_agent_normalizer(extras.user_agent)),
+        ])
 
     # Propagate to external components.
     output = ''

rhodecode/lib/utils2.py

@@ -1146,3 +1146,20 b' def retry(func=None, exception=Exception'
         return func(*args, **kwargs)
 
     return wrapper
+
+
+def user_agent_normalizer(user_agent_raw):
+    log = logging.getLogger('rhodecode.user_agent_normalizer')
+    ua = (user_agent_raw or '').strip().lower()
+
+    try:
+        if 'mercurial/proto-1.0' in ua:
+            ua = ua.replace('mercurial/proto-1.0', '')
+            ua = ua.replace('(', '').replace(')', '').strip()
+            ua = ua.replace('mercurial ', 'mercurial/')
+        elif ua.startswith('git'):
+            pass
+    except Exception:
+        log.exception('Failed to parse scm user-agent')
+
+    return ua

rhodecode/model/pull_request.py

@@ -939,8 +939,9 b' class PullRequestModel(BaseModel):'
         return commit_ids
 
     def merge_repo(self, pull_request, user, extras):
+        repo_type = pull_request.source_repo.repo_type
         log.debug("Merging pull request %s", pull_request.pull_request_id)
-        extras['user_agent'] = 'internal-merge'
+        extras['user_agent'] = '{}/internal-merge'.format(repo_type)
         merge_state = self._merge_pull_request(pull_request, user, extras)
         if merge_state.executed:
             log.debug("Merge was successful, updating the pull request comments.")