system-info: fetch vcs settings from vcsserver. Fixes #4276

- redone the system info page. Errors/Warnings now show next to the
data, and also show up in API call.
- implemented fetching of git/hg/svn version from running vcsserver
- implemented fetching of vcsserver version from itself
- added more warnings/errors detection on RAM/Inodes etc
- few cleanups

rhodecode/lib/system_info.py

@@ -0,0 +1,582 b''
+import os
+import sys
+import time
+import platform
+import pkg_resources
+import logging
+
+
+log = logging.getLogger(__name__)
+
+
+psutil = None
+
+try:
+    # cygwin cannot have yet psutil support.
+    import psutil as psutil
+except ImportError:
+    pass
+
+
+_NA = 'NOT AVAILABLE'
+
+STATE_OK = 'ok'
+STATE_ERR = 'error'
+STATE_WARN = 'warning'
+
+STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK}
+
+
+# HELPERS
+def percentage(part, whole):
+    whole = float(whole)
+    if whole > 0:
+        return 100 * float(part) / whole
+    return 0
+
+
+def get_storage_size(storage_path):
+    sizes = []
+    for file_ in os.listdir(storage_path):
+        storage_file = os.path.join(storage_path, file_)
+        if os.path.isfile(storage_file):
+            try:
+                sizes.append(os.path.getsize(storage_file))
+            except OSError:
+                log.exception('Failed to get size of storage file %s',
+                              storage_file)
+                pass
+
+    return sum(sizes)
+
+
+class SysInfoRes(object):
+    def __init__(self, value, state=STATE_OK_DEFAULT, human_value=None):
+        self.value = value
+        self.state = state
+        self.human_value = human_value or value
+
+    def __json__(self):
+        return {
+            'value': self.value,
+            'state': self.state,
+            'human_value': self.human_value,
+        }
+
+    def __str__(self):
+        return '<SysInfoRes({})>'.format(self.__json__())
+
+
+class SysInfo(object):
+
+    def __init__(self, func_name, **kwargs):
+        self.func_name = func_name
+        self.value = _NA
+        self.state = None
+        self.kwargs = kwargs or {}
+
+    def __call__(self):
+        computed = self.compute(**self.kwargs)
+        if not isinstance(computed, SysInfoRes):
+            raise ValueError(
+                'computed value for {} is not instance of '
+                '{}, got {} instead'.format(
+                    self.func_name, SysInfoRes, type(computed)))
+        return computed.__json__()
+
+    def __str__(self):
+        return '<SysInfo({})>'.format(self.func_name)
+
+    def compute(self, **kwargs):
+        return self.func_name(**kwargs)
+
+
+# SysInfo functions
+def python_info():
+    value = {
+        'version': ' '.join(platform._sys_version()),
+        'executable': sys.executable
+    }
+    return SysInfoRes(value=value)
+
+
+def py_modules():
+    mods = dict([(p.project_name, p.version)
+                 for p in pkg_resources.working_set])
+    value = sorted(mods.items(), key=lambda k: k[0].lower())
+    return SysInfoRes(value=value)
+
+
+def platform_type():
+    from rhodecode.lib.utils import safe_unicode
+    value = safe_unicode(platform.platform())
+    return SysInfoRes(value=value)
+
+
+def uptime():
+    from rhodecode.lib.helpers import age, time_to_datetime
+    from rhodecode.translation import _
+
+    _uptime = _uptime_human = {'boot_time': 0, 'uptime': 0}
+    state = STATE_OK_DEFAULT
+    if not psutil:
+        return SysInfoRes(value=_uptime, state=state)
+
+    boot_time = psutil.boot_time()
+    _uptime['boot_time'] = boot_time
+    _uptime['uptime'] = time.time() - boot_time
+
+    _uptime_human['boot_time'] = time_to_datetime(boot_time)
+    _uptime_human['uptime'] = _('Server started {}').format(
+        age(time_to_datetime(boot_time)))
+
+    return SysInfoRes(value=_uptime, human_value=_uptime_human)
+
+
+def memory():
+    from rhodecode.lib.helpers import format_byte_size_binary
+    _memory = {'available': 0, 'used': 0, 'cached': 0, 'percent': 0,
+               'percent_used': 0, 'free': 0, 'inactive': 0, 'active': 0,
+               'shared': 0, 'total': 0, 'buffers': 0}
+    state = STATE_OK_DEFAULT
+    if not psutil:
+        return SysInfoRes(value=_memory, state=state)
+
+    # memory
+    _memory = dict(psutil.virtual_memory()._asdict())
+    _memory['percent_used'] = psutil._common.usage_percent(
+        (_memory['total'] - _memory['free']), _memory['total'], 1)
+
+    try:
+        human_value = '%s/%s, %s%% used' % (
+            format_byte_size_binary(_memory['used']),
+            format_byte_size_binary(_memory['total']),
+            _memory['percent_used'],)
+    except TypeError:
+        human_value = 'NOT AVAILABLE'
+
+    if state['type'] == STATE_OK and _memory['percent_used'] > 90:
+        msg = 'Critical: your available RAM memory is very low.'
+        state = {'message': msg, 'type': STATE_ERR}
+
+    elif state['type'] == STATE_OK and  _memory['percent_used'] > 70:
+        msg = 'Warning: your available RAM memory is running low.'
+        state = {'message': msg, 'type': STATE_WARN}
+
+    return SysInfoRes(value=_memory, state=state, human_value=human_value)
+
+
+def machine_load():
+    _load = {'1_min': _NA, '5_min': _NA, '15_min': _NA}
+
+    state = STATE_OK_DEFAULT
+    if not psutil:
+        return SysInfoRes(value=_load, state=state)
+
+    # load averages
+    if hasattr(psutil.os, 'getloadavg'):
+        _load = dict(zip(['1_min', '5_min', '15_min'], psutil.os.getloadavg()))
+
+    human_value = '1min: %s, 5min: %s, 15min: %s' % (
+    _load['1_min'], _load['5_min'], _load['15_min'])
+
+    # TODO: warn about too-much load 15 min
+    return SysInfoRes(value=_load, state=state, human_value=human_value)
+
+
+def cpu():
+    state = STATE_OK_DEFAULT
+    cpu_value = 0
+    if not psutil:
+        return SysInfoRes(value=cpu_value, state=state)
+
+    cpu_value = psutil.cpu_percent(0.5)
+    human_value = '{} %'.format(cpu_value)
+    return SysInfoRes(value=cpu_value, state=state, human_value=human_value)
+
+
+def storage():
+    from rhodecode.lib.helpers import format_byte_size_binary
+    from rhodecode.model.settings import VcsSettingsModel
+    path = VcsSettingsModel().get_repos_location()
+
+    # disk storage
+    disk = {'percent': 0, 'used': 0, 'total': 0, 'path': path, 'text': ''}
+    state = STATE_OK_DEFAULT
+    if not psutil:
+        return SysInfoRes(value=disk, state=state)
+
+    try:
+        disk.update(dict(psutil.disk_usage(path)._asdict()))
+    except Exception as e:
+        log.exception('Failed to fetch disk info')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = disk
+    human_value['text'] = "{}/{}, {}% used".format(
+        format_byte_size_binary(disk['used']),
+        format_byte_size_binary(disk['total']),
+        (disk['percent']))
+
+    if state['type'] == STATE_OK and disk['percent'] > 90:
+        msg = 'Critical: your disk space is very low.'
+        state = {'message': msg, 'type': STATE_ERR}
+
+    elif state['type'] == STATE_OK and disk['percent'] > 70:
+        msg = 'Warning: your disk space is running low.'
+        state = {'message': msg, 'type': STATE_WARN}
+
+    return SysInfoRes(value=disk, state=state, human_value=human_value)
+
+
+def storage_inodes():
+    from rhodecode.model.settings import VcsSettingsModel
+    path = VcsSettingsModel().get_repos_location()
+
+    _disk_inodes = dict(percent=0, free=0, used=0, total=0, path=path, text='')
+    state = STATE_OK_DEFAULT
+    if not psutil:
+        return SysInfoRes(value=_disk_inodes, state=state)
+
+    try:
+        i_stat = os.statvfs(path)
+
+        _disk_inodes['used'] = i_stat.f_ffree
+        _disk_inodes['free'] = i_stat.f_favail
+        _disk_inodes['total'] = i_stat.f_files
+        _disk_inodes['percent'] = percentage(
+            _disk_inodes['used'], _disk_inodes['total'])
+    except Exception as e:
+        log.exception('Failed to fetch disk inodes info')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = _disk_inodes
+    human_value['text'] = "{}/{}, {}% used".format(
+        _disk_inodes['used'], _disk_inodes['total'], _disk_inodes['percent'])
+
+    if state['type'] == STATE_OK and _disk_inodes['percent'] > 90:
+        msg = 'Critical: your disk free inodes are very low.'
+        state = {'message': msg, 'type': STATE_ERR}
+
+    elif state['type'] == STATE_OK and _disk_inodes['percent'] > 70:
+        msg = 'Warning: your disk free inodes are running low.'
+        state = {'message': msg, 'type': STATE_WARN}
+
+    return SysInfoRes(value=_disk_inodes, state=state)
+
+
+def storage_archives():
+    import rhodecode
+    from rhodecode.lib.utils import safe_str
+    from rhodecode.lib.helpers import format_byte_size_binary
+
+    msg = 'Enable this by setting ' \
+          'archive_cache_dir=/path/to/cache option in the .ini file'
+    path = safe_str(rhodecode.CONFIG.get('archive_cache_dir', msg))
+
+    disk_archive = dict(percent=0, used=0, total=0, items=0, path=path, text='')
+    state = STATE_OK_DEFAULT
+    try:
+        items_count = 0
+        used = 0
+        for root, dirs, files in os.walk(path):
+            if root == path:
+                items_count = len(dirs)
+
+            for f in files:
+                try:
+                    used += os.path.getsize(os.path.join(root, f))
+                except OSError:
+                    pass
+        disk_archive.update({
+            'percent': 100,
+            'used': used,
+            'total': used,
+            'items': items_count
+        })
+
+    except Exception as e:
+        log.exception('failed to fetch archive cache storage')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = disk_archive
+    human_value['text'] = "{} ({} items)".format(format_byte_size_binary(
+        disk_archive['used']), disk_archive['total'])
+
+    return SysInfoRes(value=disk_archive, state=state, human_value=human_value)
+
+
+def storage_gist():
+    from rhodecode.model.gist import GIST_STORE_LOC
+    from rhodecode.model.settings import VcsSettingsModel
+    from rhodecode.lib.utils import safe_str
+    from rhodecode.lib.helpers import format_byte_size_binary
+    path = safe_str(os.path.join(
+        VcsSettingsModel().get_repos_location(), GIST_STORE_LOC))
+
+    # gist storage
+    _disk_gist = dict(percent=0, used=0, total=0, items=0, path=path, text='')
+    state = STATE_OK_DEFAULT
+
+    try:
+        items_count = 0
+        used = 0
+        for root, dirs, files in os.walk(path):
+            if root == path:
+                items_count = len(dirs)
+
+            for f in files:
+                try:
+                    used += os.path.getsize(os.path.join(root, f))
+                except OSError:
+                    pass
+        _disk_gist.update({
+            'percent': 100,
+            'used': used,
+            'total': used,
+            'items': items_count
+        })
+    except Exception as e:
+        log.exception('failed to fetch gist storage items')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = _disk_gist
+    human_value['text'] = "{} ({} items)".format(format_byte_size_binary(
+        _disk_gist['used']), _disk_gist['items'])
+    return SysInfoRes(value=_disk_gist, state=state, human_value=human_value)
+
+
+def storage_search():
+    import rhodecode
+    path = rhodecode.CONFIG.get('search.location', '')
+
+    # search index storage
+    _disk_index = dict(percent=0, used=0, total=0, path=path, text='')
+    state = STATE_OK_DEFAULT
+    try:
+        search_index_storage_path_exists = os.path.isdir(path)
+        if search_index_storage_path_exists:
+            used = get_storage_size(path)
+            _disk_index.update({
+                'percent': 100,
+                'used': used,
+                'total': used,
+            })
+    except Exception as e:
+        log.exception('failed to fetch search index storage')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = _disk_index
+    human_value['text'] = "{}/{}, {}% used".format(
+        _disk_index['used'], _disk_index['total'], _disk_index['percent'])
+
+    return SysInfoRes(value=_disk_index, state=state, human_value=human_value)
+
+
+def git_info():
+    from rhodecode.lib.vcs.backends import git
+    state = STATE_OK_DEFAULT
+    value = human_value = ''
+    try:
+        value = git.discover_git_version(raise_on_exc=True)
+        human_value = 'version reported from VCSServer: {}'.format(value)
+    except Exception as e:
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    return SysInfoRes(value=value, state=state, human_value=human_value)
+
+
+def hg_info():
+    from rhodecode.lib.vcs.backends import hg
+    state = STATE_OK_DEFAULT
+    value = human_value = ''
+    try:
+        value = hg.discover_hg_version(raise_on_exc=True)
+        human_value = 'version reported from VCSServer: {}'.format(value)
+    except Exception as e:
+        state = {'message': str(e), 'type': STATE_ERR}
+    return SysInfoRes(value=value, state=state, human_value=human_value)
+
+
+def svn_info():
+    from rhodecode.lib.vcs.backends import svn
+    state = STATE_OK_DEFAULT
+    value = human_value = ''
+    try:
+        value = svn.discover_svn_version(raise_on_exc=True)
+        human_value = 'version reported from VCSServer: {}'.format(value)
+    except Exception as e:
+        state = {'message': str(e), 'type': STATE_ERR}
+    return SysInfoRes(value=value, state=state, human_value=human_value)
+
+
+def vcs_backends():
+    import rhodecode
+    value = rhodecode.CONFIG.get('vcs.backends', '').split(',')
+    human_value = 'Enabled backends in order: {}'.format(','.join(value))
+    return SysInfoRes(value=value, human_value=human_value)
+
+
+def vcs_server():
+    import rhodecode
+    from rhodecode.lib.vcs.backends import get_vcsserver_version
+
+    server_url = rhodecode.CONFIG.get('vcs.server')
+    enabled = rhodecode.CONFIG.get('vcs.server.enable')
+    protocol = rhodecode.CONFIG.get('vcs.server.protocol')
+    state = STATE_OK_DEFAULT
+    version = None
+
+    try:
+        version = get_vcsserver_version()
+        connection = 'connected'
+    except Exception as e:
+        connection = 'failed'
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    value = dict(
+        url=server_url,
+        enabled=enabled,
+        protocol=protocol,
+        connection=connection,
+        version=version,
+        text='',
+    )
+
+    human_value = value
+    human_value['text'] = \
+        '{url}@ver:{ver} via {mode} mode, connection:{conn}'.format(
+            url=server_url, ver=version, mode=protocol, conn=connection)
+
+    return SysInfoRes(value='', state=state, human_value=human_value)
+
+
+def rhodecode_app_info():
+    import rhodecode
+    return SysInfoRes(value={'rhodecode_version': rhodecode.__version__})
+
+
+def rhodecode_config():
+    import rhodecode
+    path = rhodecode.CONFIG.get('__file__')
+    rhodecode_ini_safe = rhodecode.CONFIG.copy()
+
+    blacklist = [
+        'rhodecode_license_key',
+        'routes.map',
+        'pylons.h',
+        'pylons.app_globals',
+        'pylons.environ_config',
+        'sqlalchemy.db1.url',
+        'channelstream.secret',
+        'beaker.session.secret',
+        'rhodecode.encrypted_values.secret',
+        'rhodecode_auth_github_consumer_key',
+        'rhodecode_auth_github_consumer_secret',
+        'rhodecode_auth_google_consumer_key',
+        'rhodecode_auth_google_consumer_secret',
+        'rhodecode_auth_bitbucket_consumer_secret',
+        'rhodecode_auth_bitbucket_consumer_key',
+        'rhodecode_auth_twitter_consumer_secret',
+        'rhodecode_auth_twitter_consumer_key',
+
+        'rhodecode_auth_twitter_secret',
+        'rhodecode_auth_github_secret',
+        'rhodecode_auth_google_secret',
+        'rhodecode_auth_bitbucket_secret',
+
+        'appenlight.api_key',
+        ('app_conf', 'sqlalchemy.db1.url')
+    ]
+    for k in blacklist:
+        if isinstance(k, tuple):
+            section, key = k
+            if section in rhodecode_ini_safe:
+                rhodecode_ini_safe[section] = '**OBFUSCATED**'
+        else:
+            rhodecode_ini_safe.pop(k, None)
+
+    # TODO: maybe put some CONFIG checks here ?
+    return SysInfoRes(value={'config': rhodecode_ini_safe, 'path': path})
+
+
+def database_info():
+    import rhodecode
+    from sqlalchemy.engine import url as engine_url
+    from rhodecode.model.meta import Base as sql_base, Session
+    from rhodecode.model.db import DbMigrateVersion
+
+    state = STATE_OK_DEFAULT
+
+    db_migrate = DbMigrateVersion.query().filter(
+        DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
+
+    db_url_obj = engine_url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
+
+    try:
+        engine = sql_base.metadata.bind
+        db_server_info = engine.dialect._get_server_version_info(
+            Session.connection(bind=engine))
+        db_version = '.'.join(map(str, db_server_info))
+    except Exception:
+        log.exception('failed to fetch db version')
+        db_version = 'UNKNOWN'
+
+    db_info = dict(
+        migrate_version=db_migrate.version,
+        type=db_url_obj.get_backend_name(),
+        version=db_version,
+        url=repr(db_url_obj)
+    )
+
+    human_value = db_info
+    human_value['url'] = "{} @ migration version: {}".format(
+        db_info['url'], db_info['migrate_version'])
+    human_value['version'] = "{} {}".format(db_info['type'], db_info['version'])
+    return SysInfoRes(value=db_info, state=state, human_value=human_value)
+
+
+def server_info(environ):
+    import rhodecode
+    from rhodecode.lib.base import get_server_ip_addr, get_server_port
+
+    value = {
+        'server_ip': '%s:%s' % (
+            get_server_ip_addr(environ, log_errors=False),
+            get_server_port(environ)
+        ),
+        'server_id': rhodecode.CONFIG.get('instance_id'),
+    }
+    return SysInfoRes(value=value)
+
+
+def get_system_info(environ):
+    environ = environ or {}
+    return {
+        'rhodecode_app': SysInfo(rhodecode_app_info)(),
+        'rhodecode_config': SysInfo(rhodecode_config)(),
+        'python': SysInfo(python_info)(),
+        'py_modules': SysInfo(py_modules)(),
+
+        'platform': SysInfo(platform_type)(),
+        'server': SysInfo(server_info, environ=environ)(),
+        'database': SysInfo(database_info)(),
+
+        'storage': SysInfo(storage)(),
+        'storage_inodes': SysInfo(storage_inodes)(),
+        'storage_archive': SysInfo(storage_archives)(),
+        'storage_search': SysInfo(storage_search)(),
+        'storage_gist': SysInfo(storage_gist)(),
+
+        'uptime': SysInfo(uptime)(),
+        'load': SysInfo(machine_load)(),
+        'cpu': SysInfo(cpu)(),
+        'memory': SysInfo(memory)(),
+
+        'vcs_backends': SysInfo(vcs_backends)(),
+        'vcs_server': SysInfo(vcs_server)(),
+
+        'git': SysInfo(git_info)(),
+        'hg': SysInfo(hg_info)(),
+        'svn': SysInfo(svn_info)(),
+    }

rhodecode/tests/lib/test_system_info.py

@@ -0,0 +1,15 b''
+import py.test
+
+from rhodecode.lib.system_info import get_system_info
+
+
+def test_system_info(app):
+    info = get_system_info({})
+    assert info['load']['value']['15_min'] != 'NOT AVAILABLE'
+
+
+def test_system_info_without_psutil(monkeypatch, app):
+    import rhodecode.lib.system_info
+    monkeypatch.setattr(rhodecode.lib.system_info, 'psutil', None)
+    info = get_system_info({})
+    assert info['load']['value']['15_min'] == 'NOT AVAILABLE'

rhodecode/api/tests/test_get_server_info.py

@@ -45,9 +45,9 b' class TestGetServerInfo(object):'
         expected['uptime'] = resp['result']['uptime']
         expected['load'] = resp['result']['load']
         expected['cpu'] = resp['result']['cpu']
-        expected['disk'] = resp['result']['disk']
-        expected['disk_inodes'] = resp['result']['disk_inodes']
-        expected['server_ip'] = '127.0.0.1:80'
+        expected['storage'] = resp['result']['storage']
+        expected['storage_inodes'] = resp['result']['storage_inodes']
+        expected['server'] = resp['result']['server']
 
         assert_ok(id_, expected, given=response.body)
 
@@ -60,8 +60,8 b' class TestGetServerInfo(object):'
         expected['uptime'] = resp['result']['uptime']
         expected['load'] = resp['result']['load']
         expected['cpu'] = resp['result']['cpu']
-        expected['disk'] = resp['result']['disk']
-        expected['disk_inodes'] = resp['result']['disk_inodes']
-        expected['server_ip'] = '127.0.0.1:80'
+        expected['storage'] = resp['result']['storage']
+        expected['storage_inodes'] = resp['result']['storage_inodes']
+        expected['server'] = resp['result']['server']
 
         assert_ok(id_, expected, given=response.body)

rhodecode/controllers/admin/settings.py

@@ -548,62 +548,85 b' class SettingsController(BaseController)'
         """GET /admin/settings/system: All items in the collection"""
         # url('admin_settings_system')
         snapshot = str2bool(request.GET.get('snapshot'))
-        c.active = 'system'
+        defaults = self._form_defaults()
 
-        defaults = self._form_defaults()
-        c.rhodecode_ini = rhodecode.CONFIG
+        c.active = 'system'
         c.rhodecode_update_url = defaults.get('rhodecode_update_url')
         server_info = ScmModel().get_server_info(request.environ)
+
         for key, val in server_info.iteritems():
             setattr(c, key, val)
 
-        if c.disk['percent'] > 90:
-            h.flash(h.literal(_(
-                'Critical: your disk space is very low <b>%s%%</b> used' %
-                c.disk['percent'])), 'error')
-        elif c.disk['percent'] > 70:
-            h.flash(h.literal(_(
-                'Warning: your disk space is running low <b>%s%%</b> used' %
-                c.disk['percent'])), 'warning')
+        def val(name, subkey='human_value'):
+            return server_info[name][subkey]
+
+        def state(name):
+            return server_info[name]['state']
+
+        def val2(name):
+            val = server_info[name]['human_value']
+            state = server_info[name]['state']
+            return val, state
 
-        try:
-            c.uptime_age = h._age(
-                h.time_to_datetime(c.boot_time), False, show_suffix=False)
-        except TypeError:
-            c.uptime_age = c.boot_time
+        c.data_items = [
+            # update info
+            (_('Update info'), h.literal(
+                '<span class="link" id="check_for_update" >%s.</span>' % (
+                _('Check for updates')) +
+                '<br/> <span >%s.</span>' % (_('Note: please make sure this server can access `%s` for the update link to work') % c.rhodecode_update_url)
+            ), ''),
+
+            # RhodeCode specific
+            (_('RhodeCode Version'), c.rhodecode_version, ''),
+            (_('RhodeCode Server IP'), val('server')['server_ip'], state('server')),
+            (_('RhodeCode Server ID'), val('server')['server_id'], state('server')),
+            (_('RhodeCode Configuration'), val('rhodecode_config')['path'], state('rhodecode_config')),
+            ('', '', ''),  # spacer
+
+            # Database
+            (_('Database'), val('database')['url'], state('database')),
+            (_('Database version'), val('database')['version'], state('database')),
+            ('', '', ''),  # spacer
 
-        try:
-            c.system_memory = '%s/%s, %s%% (%s%%) used%s' % (
-                h.format_byte_size_binary(c.memory['used']),
-                h.format_byte_size_binary(c.memory['total']),
-                c.memory['percent2'],
-                c.memory['percent'],
-                ' %s' % c.memory['error'] if 'error' in c.memory else '')
-        except TypeError:
-            c.system_memory = 'NOT AVAILABLE'
+            # Platform/Python
+            (_('Platform'), val('platform'), state('platform')),
+            (_('Python version'), val('python')['version'], state('python')),
+            (_('Python path'), val('python')['executable'], state('python')),
+            ('', '', ''),  # spacer
+
+            # Systems stats
+            (_('CPU'), val('cpu'), state('cpu')),
+            (_('Load'), val('load'), state('load')),
+            (_('Memory'), val('memory'), state('memory')),
+            (_('Uptime'), val('uptime')['uptime'], state('uptime')),
+            ('', '', ''),  # spacer
+
+            # Repo storage
+            (_('Storage location'), val('storage')['path'], state('storage')),
+            (_('Storage info'), val('storage')['text'], state('storage')),
+            (_('Storage inodes'), val('storage_inodes')['text'], state('storage_inodes')),
 
-        rhodecode_ini_safe = rhodecode.CONFIG.copy()
-        blacklist = [
-            'rhodecode_license_key',
-            'routes.map',
-            'pylons.h',
-            'pylons.app_globals',
-            'pylons.environ_config',
-            'sqlalchemy.db1.url',
-            ('app_conf', 'sqlalchemy.db1.url')
+            (_('Gist storage location'), val('storage_gist')['path'], state('storage_gist')),
+            (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
+
+            (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
+            (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
+
+            (_('Search storage'), val('storage_search')['path'], state('storage_search')),
+            (_('Search info'), val('storage_search')['text'], state('storage_search')),
+            ('', '', ''),  # spacer
+
+            # VCS specific
+            (_('VCS Backends'), val('vcs_backends'), state('vcs_backends')),
+            (_('VCS Server'), val('vcs_server')['text'], state('vcs_server')),
+            (_('GIT'), val('git'), state('git')),
+            (_('HG'), val('hg'), state('hg')),
+            (_('SVN'), val('svn'), state('svn')),
+
         ]
-        for k in blacklist:
-            if isinstance(k, tuple):
-                section, key = k
-                if section in rhodecode_ini_safe:
-                    rhodecode_ini_safe[section].pop(key, None)
-            else:
-                rhodecode_ini_safe.pop(k, None)
-
-        c.rhodecode_ini_safe = rhodecode_ini_safe
 
         # TODO: marcink, figure out how to allow only selected users to do this
-        c.allowed_to_snapshot = False
+        c.allowed_to_snapshot = c.rhodecode_user.admin
 
         if snapshot:
             if c.allowed_to_snapshot:

rhodecode/lib/vcs/__init__.py

@@ -99,6 +99,7 b' def connect_pyro4(server_and_port):'
         connection.Git = None
         connection.Hg = None
         connection.Svn = None
+        connection.Service = None
 
 
 def connect_http(server_and_port):
@@ -113,6 +114,8 b' def connect_http(server_and_port):'
         server_and_port, '/hg', session_factory)
     connection.Svn = client_http.RepoMaker(
         server_and_port, '/svn', session_factory)
+    connection.Service = client_http.ServiceConnection(
+        server_and_port, '/_service', session_factory)
 
     scm_app.HG_REMOTE_WSGI = client_http.VcsHttpProxy(
         server_and_port, '/proxy/hg')
@@ -124,6 +127,7 b' def connect_http(server_and_port):'
         connection.Git = None
         connection.Hg = None
         connection.Svn = None
+        connection.Service = None
 
 
 def connect_vcs(server_and_port, protocol):

rhodecode/lib/vcs/backends/__init__.py

@@ -76,3 +76,12 b' def get_supported_backends():'
     Returns list of aliases of supported backends.
     """
     return settings.BACKENDS.keys()
+
+
+def get_vcsserver_version():
+    from rhodecode.lib.vcs import connection
+    data = connection.Service.get_vcsserver_service_data()
+    if data and 'version' in data:
+        return data['version']
+
+    return None

rhodecode/lib/vcs/backends/git/__init__.py

@@ -33,7 +33,7 b' from rhodecode.lib.vcs.backends.git.inme'
 log = logging.getLogger(__name__)
 
 
-def discover_git_version():
+def discover_git_version(raise_on_exc=False):
     """
     Returns the string as it was returned by running 'git --version'
 
@@ -44,4 +44,6 b' def discover_git_version():'
         return connection.Git.discover_git_version()
     except Exception:
         log.warning("Failed to discover the Git version", exc_info=True)
+        if raise_on_exc:
+            raise
         return ''

rhodecode/lib/vcs/backends/hg/__init__.py

@@ -21,7 +21,28 b''
 """
 HG module
 """
+import logging
 
+from rhodecode.lib.vcs import connection
 from rhodecode.lib.vcs.backends.hg.commit import MercurialCommit
 from rhodecode.lib.vcs.backends.hg.inmemory import MercurialInMemoryCommit
 from rhodecode.lib.vcs.backends.hg.repository import MercurialRepository
+
+
+log = logging.getLogger(__name__)
+
+
+def discover_hg_version(raise_on_exc=False):
+    """
+    Returns the string as it was returned by running 'git --version'
+
+    It will return an empty string in case the connection is not initialized
+    or no vcsserver is available.
+    """
+    try:
+        return connection.Hg.discover_hg_version()
+    except Exception:
+        log.warning("Failed to discover the HG version", exc_info=True)
+        if raise_on_exc:
+            raise
+        return ''

rhodecode/lib/vcs/backends/svn/__init__.py

@@ -21,6 +21,27 b''
 """
 SVN module
 """
+import logging
 
+from rhodecode.lib.vcs import connection
 from rhodecode.lib.vcs.backends.svn.commit import SubversionCommit
 from rhodecode.lib.vcs.backends.svn.repository import SubversionRepository
+
+
+log = logging.getLogger(__name__)
+
+
+def discover_svn_version(raise_on_exc=False):
+    """
+    Returns the string as it was returned by running 'git --version'
+
+    It will return an empty string in case the connection is not initialized
+    or no vcsserver is available.
+    """
+    try:
+        return connection.Svn.discover_svn_version()
+    except Exception:
+        log.warning("Failed to discover the SVN version", exc_info=True)
+        if raise_on_exc:
+            raise
+        return ''

rhodecode/lib/vcs/connection.py

@@ -30,6 +30,8 b' def _not_initialized(*args, **kwargs):'
         "Make sure `vcs.server` is enabled in your configuration.")
 
 # TODO: figure out a nice default value for these things
+Service = _not_initialized
+
 Git = _not_initialized
 Hg = _not_initialized
 Svn = _not_initialized

rhodecode/model/scm.py

@@ -25,13 +25,11 b' Scm model for RhodeCode'
 import os.path
 import re
 import sys
-import time
 import traceback
 import logging
 import cStringIO
 import pkg_resources
 
-import pylons
 from pylons.i18n.translation import _
 from sqlalchemy import func
 from zope.cachedescriptors.property import Lazy as LazyProperty
@@ -50,12 +48,12 b' from rhodecode.lib.exceptions import Non'
 from rhodecode.lib import hooks_utils, caches
 from rhodecode.lib.utils import (
     get_filesystem_repos, action_logger, make_db_config)
-from rhodecode.lib.utils2 import (
-    safe_str, safe_unicode, get_server_url, md5)
+from rhodecode.lib.utils2 import (safe_str, safe_unicode)
+from rhodecode.lib.system_info import get_system_info
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     Repository, CacheKey, UserFollowing, UserLog, User, RepoGroup,
-    PullRequest, DbMigrateVersion)
+    PullRequest)
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
@@ -882,218 +880,8 b' class ScmModel(BaseModel):'
             self.install_svn_hooks(repo)
 
     def get_server_info(self, environ=None):
-        import platform
-        import rhodecode
-        import pkg_resources
-        from rhodecode.model.meta import Base as sql_base, Session
-        from sqlalchemy.engine import url
-        from rhodecode.lib.base import get_server_ip_addr, get_server_port
-        from rhodecode.lib.vcs.backends.git import discover_git_version
-        from rhodecode.model.gist import GIST_STORE_LOC
-
-        def percentage(part, whole):
-            whole = float(whole)
-            if whole > 0:
-                return 100 * float(part) / whole
-            return 0
-
-        try:
-            # cygwin cannot have yet psutil support.
-            import psutil
-        except ImportError:
-            psutil = None
-
-        environ = environ or {}
-        _NA = 'NOT AVAILABLE'
-        _memory = _NA
-        _uptime = _NA
-        _boot_time = _NA
-        _cpu = _NA
-        _disk = dict(percent=0, used=0, total=0, error='')
-        _disk_inodes = dict(percent=0, free=0, used=0, total=0, error='')
-        _load = {'1_min': _NA, '5_min': _NA, '15_min': _NA}
-
-        model = VcsSettingsModel()
-        storage_path = model.get_repos_location()
-        gist_storage_path = os.path.join(storage_path, GIST_STORE_LOC)
-        archive_storage_path = rhodecode.CONFIG.get('archive_cache_dir', '')
-        search_index_storage_path = rhodecode.CONFIG.get('search.location', '')
-
-        if psutil:
-            # disk storage
-            try:
-                _disk = dict(psutil.disk_usage(storage_path)._asdict())
-            except Exception as e:
-                log.exception('Failed to fetch disk info')
-                _disk = {'percent': 0, 'used': 0, 'total': 0, 'error': str(e)}
-
-            # disk inodes usage
-            try:
-                i_stat = os.statvfs(storage_path)
-
-                _disk_inodes['used'] = i_stat.f_ffree
-                _disk_inodes['free'] = i_stat.f_favail
-                _disk_inodes['total'] = i_stat.f_files
-                _disk_inodes['percent'] = percentage(
-                    _disk_inodes['used'], _disk_inodes['total'])
-            except Exception as e:
-                log.exception('Failed to fetch disk inodes info')
-                _disk_inodes['error'] = str(e)
-
-            # memory
-            _memory = dict(psutil.virtual_memory()._asdict())
-            _memory['percent2'] = psutil._common.usage_percent(
-                (_memory['total'] - _memory['free']),
-                _memory['total'], 1)
-
-            # load averages
-            if hasattr(psutil.os, 'getloadavg'):
-                _load = dict(zip(
-                    ['1_min', '5_min', '15_min'], psutil.os.getloadavg()))
-            _uptime = time.time() - psutil.boot_time()
-            _boot_time = psutil.boot_time()
-            _cpu = psutil.cpu_percent(0.5)
-
-        mods = dict([(p.project_name, p.version)
-                     for p in pkg_resources.working_set])
-
-        def get_storage_size(storage_path):
-            sizes = []
-            for file_ in os.listdir(storage_path):
-                storage_file = os.path.join(storage_path, file_)
-                if os.path.isfile(storage_file):
-                    try:
-                        sizes.append(os.path.getsize(storage_file))
-                    except OSError:
-                        log.exception('Failed to get size of storage file %s',
-                                      storage_file)
-                        pass
-
-            return sum(sizes)
-
-        # archive cache storage
-        _disk_archive = {'percent': 0, 'used': 0, 'total': 0}
-        try:
-            archive_storage_path_exists = os.path.isdir(
-                archive_storage_path)
-            if archive_storage_path and archive_storage_path_exists:
-                used = get_storage_size(archive_storage_path)
-                _disk_archive.update({
-                    'used': used,
-                    'total': used,
-                })
-        except Exception as e:
-            log.exception('failed to fetch archive cache storage')
-            _disk_archive['error'] = str(e)
-
-        # search index storage
-        _disk_index = {'percent': 0, 'used': 0, 'total': 0}
-        try:
-            search_index_storage_path_exists = os.path.isdir(
-                search_index_storage_path)
-            if search_index_storage_path_exists:
-                used = get_storage_size(search_index_storage_path)
-                _disk_index.update({
-                    'percent': 100,
-                    'used': used,
-                    'total': used,
-                })
-        except Exception as e:
-            log.exception('failed to fetch search index storage')
-            _disk_index['error'] = str(e)
-
-        # gist storage
-        _disk_gist = {'percent': 0, 'used': 0, 'total': 0, 'items': 0}
-        try:
-            items_count = 0
-            used = 0
-            for root, dirs, files in os.walk(safe_str(gist_storage_path)):
-                if root == gist_storage_path:
-                    items_count = len(dirs)
-
-                for f in files:
-                    try:
-                        used += os.path.getsize(os.path.join(root, f))
-                    except OSError:
-                        pass
-            _disk_gist.update({
-                'percent': 100,
-                'used': used,
-                'total': used,
-                'items': items_count
-            })
-        except Exception as e:
-            log.exception('failed to fetch gist storage items')
-            _disk_gist['error'] = str(e)
-
-        # GIT info
-        git_ver = discover_git_version()
-
-        # SVN info
-        # TODO: johbo: Add discover_svn_version to replace this code.
-        try:
-            import svn.core
-            svn_ver = svn.core.SVN_VERSION
-        except ImportError:
-            svn_ver = None
-
-        # DB stuff
-        db_info = url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
-        db_type = db_info.__to_string__()
-        try:
-            engine = sql_base.metadata.bind
-            db_server_info = engine.dialect._get_server_version_info(
-                Session.connection(bind=engine))
-            db_version = '%s %s' % (db_info.drivername,
-                                    '.'.join(map(str, db_server_info)))
-        except Exception:
-            log.exception('failed to fetch db version')
-            db_version = '%s %s' % (db_info.drivername, '?')
-
-        db_migrate = DbMigrateVersion.query().filter(
-            DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
-        db_migrate_version = db_migrate.version
-
-        info = {
-            'py_version': ' '.join(platform._sys_version()),
-            'py_path': sys.executable,
-            'py_modules': sorted(mods.items(), key=lambda k: k[0].lower()),
-
-            'platform': safe_unicode(platform.platform()),
-            'storage': storage_path,
-            'archive_storage': archive_storage_path,
-            'index_storage': search_index_storage_path,
-            'gist_storage': gist_storage_path,
-
-
-            'db_type': db_type,
-            'db_version': db_version,
-            'db_migrate_version': db_migrate_version,
-
-            'rhodecode_version': rhodecode.__version__,
-            'rhodecode_config_ini': rhodecode.CONFIG.get('__file__'),
-            'server_ip': '%s:%s' % (
-                get_server_ip_addr(environ, log_errors=False),
-                get_server_port(environ)
-            ),
-            'server_id': rhodecode.CONFIG.get('instance_id'),
-
-            'git_version': safe_unicode(git_ver),
-            'hg_version': mods.get('mercurial'),
-            'svn_version': svn_ver,
-
-            'uptime': _uptime,
-            'boot_time': _boot_time,
-            'load': _load,
-            'cpu': _cpu,
-            'memory': _memory,
-            'disk': _disk,
-            'disk_inodes': _disk_inodes,
-            'disk_archive': _disk_archive,
-            'disk_gist': _disk_gist,
-            'disk_index': _disk_index,
-        }
-        return info
+        server_info = get_system_info(environ)
+        return server_info
 
 
 def _check_rhodecode_hook(hook_path):

rhodecode/templates/admin/settings/settings_system.html

@@ -1,42 +1,3 b''
-<%
- elems = [
-    ## general
-    (_('RhodeCode Enterprise version'), h.literal('%s <div class="link" id="check_for_update" >%s</div>' % (c.rhodecode_version, _('check for updates'))), ''),
-    (_('Upgrade info endpoint'), h.literal('%s <br/><span >%s.</span>' % (c.rhodecode_update_url, _('Note: please make sure this server can access this url'))), ''),
-    (_('Configuration INI file'), c.rhodecode_config_ini, ''),
-    ## systems stats
-    (_('RhodeCode Enterprise Server IP'), c.server_ip, ''),
-    (_('RhodeCode Enterprise Server ID'), c.server_id, ''),
-    (_('Platform'), c.platform, ''),
-    (_('Uptime'), c.uptime_age, ''),
-    (_('Storage location'), c.storage, ''),
-    (_('Storage disk space'), "%s/%s, %s%% used%s" % (h.format_byte_size_binary(c.disk['used']), h.format_byte_size_binary(c.disk['total']),(c.disk['percent']), ' %s' % c.disk['error'] if 'error' in c.disk else ''), ''),
-    (_('Storage file limit (inodes)'), "%s/%s, %.1f%% used%s" % (c.disk_inodes['used'], c.disk_inodes['total'],(c.disk_inodes['percent']), ' %s' % c.disk_inodes['error'] if 'error' in c.disk_inodes else ''), ''),
-
-    (_('Search index storage'), c.index_storage, ''),
-    (_('Search index size'), "%s %s" % (h.format_byte_size_binary(c.disk_index['used']), ' %s' % c.disk_index['error'] if 'error' in c.disk_index else ''), ''),
-
-    (_('Gist storage'), c.gist_storage, ''),
-    (_('Gist storage size'), "%s (%s items)%s" % (h.format_byte_size_binary(c.disk_gist['used']),c.disk_gist['items'], ' %s' % c.disk_gist['error'] if 'error' in c.disk_gist else ''), ''),
-
-    (_('Archive cache'), h.literal('%s <br/><span >%s.</span>' % (c.archive_storage, _('Enable this by setting archive_cache_dir=/path/to/cache option in the .ini file'))), ''),
-    (_('Archive cache size'), "%s%s" % (h.format_byte_size_binary(c.disk_archive['used']), ' %s' % c.disk_archive['error'] if 'error' in c.disk_archive else ''), ''),
-
-    (_('System memory'), c.system_memory, ''),
-    (_('CPU'), '%s %%' %(c.cpu), ''),
-    (_('Load'), '1min: %s, 5min: %s, 15min: %s' %(c.load['1_min'],c.load['5_min'],c.load['15_min']), ''),
-
-    ## rhodecode stuff
-    (_('Python version'), c.py_version, ''),
-    (_('Python path'), c.py_path, ''),
-    (_('GIT version'), c.git_version, ''),
-    (_('HG version'), c.hg_version, ''),
-    (_('SVN version'), c.svn_version, ''),
-    (_('Database'), "%s @ version: %s" % (c.db_type, c.db_migrate_version), ''),
-    (_('Database version'), c.db_version, ''),
-
- ]
-%>
 
 <div id="update_notice" style="display: none; margin: -40px 0px 20px 0px">
     <div>${_('Checking for updates...')}</div>
@@ -47,14 +8,20 b''
     <div class="panel-heading">
         <h3 class="panel-title">${_('System Info')}</h3>
         % if c.allowed_to_snapshot:
-            <a href="${url('admin_settings_system', snapshot=1)}" class="panel-edit">${_('create snapshot')}</a>
+            <a href="${url('admin_settings_system', snapshot=1)}" class="panel-edit">${_('create summary snapshot')}</a>
         % endif
     </div>
     <div class="panel-body">
         <dl class="dl-horizontal settings">
-        %for dt, dd, tt in elems:
-          <dt>${dt}:</dt>
-          <dd title="${tt}">${dd}</dd>
+        % for dt, dd, warn in c.data_items:
+          <dt>${dt}${':' if dt else '---'}</dt>
+          <dd>${dd}${'' if dt else '---'}
+              % if warn and warn['message']:
+                  <div class="alert-${warn['type']}">
+                    <strong>${warn['message']}</strong>
+                  </div>
+              % endif
+          </dd>
         %endfor
         </dl>
     </div>
@@ -71,7 +38,7 b''
               <col class='content'>
           </colgroup>
           <tbody>
-              %for key, value in c.py_modules:
+              % for key, value in c.py_modules['human_value']:
                   <tr>
                       <td>${key}</td>
                       <td>${value}</td>

rhodecode/templates/admin/settings/settings_system_snapshot.html

@@ -1,61 +1,27 b''
-<%
- elems = [
-    ## general
-    (_('RhodeCode Enterprise version'), c.rhodecode_version, ''),
-    (_('Upgrade info endpoint'), c.rhodecode_update_url, ''),
-    (_('Configuration INI file'), c.rhodecode_config_ini, ''),
-    ## systems stats
-    (_('RhodeCode Enterprise Server IP'), c.server_ip, ''),
-    (_('RhodeCode Enterprise Server ID'), c.server_id, ''),
-    (_('Platform'), c.platform, ''),
-    (_('Uptime'), c.uptime_age, ''),
-    (_('Storage location'), c.storage, ''),
-    (_('Storage disk space'), "%s/%s, %s%% used%s" % (h.format_byte_size_binary(c.disk['used']), h.format_byte_size_binary(c.disk['total']),(c.disk['percent']), ' %s' % c.disk['error'] if 'error' in c.disk else ''), ''),
-
-    (_('Search index storage'), c.index_storage, ''),
-    (_('Search index size'), "%s %s" % (h.format_byte_size_binary(c.disk_index['used']), ' %s' % c.disk_index['error'] if 'error' in c.disk_index else ''), ''),
-
-    (_('Gist storage'), c.gist_storage, ''),
-    (_('Gist storage size'), "%s (%s items)%s" % (h.format_byte_size_binary(c.disk_gist['used']),c.disk_gist['items'], ' %s' % c.disk_gist['error'] if 'error' in c.disk_gist else ''), ''),
-
-    (_('Archive cache'), c.archive_storage, ''),
-    (_('Archive cache size'), "%s%s" % (h.format_byte_size_binary(c.disk_archive['used']), ' %s' % c.disk_archive['error'] if 'error' in c.disk_archive else ''), ''),
-
-    (_('System memory'), c.system_memory, ''),
-    (_('CPU'), '%s %%' %(c.cpu), ''),
-    (_('Load'), '1min: %s, 5min: %s, 15min: %s' %(c.load['1_min'],c.load['5_min'],c.load['15_min']), ''),
-
-    ## rhodecode stuff
-    (_('Python version'), c.py_version, ''),
-    (_('Python path'), c.py_path, ''),
-    (_('GIT version'), c.git_version, ''),
-    (_('HG version'), c.hg_version, ''),
-    (_('SVN version'), c.svn_version, ''),
-    (_('Database'), "%s @ version: %s" % (c.db_type, c.db_migrate_version), ''),
-    (_('Database version'), c.db_version, ''),
-
- ]
-%>
 
 <pre>
 SYSTEM INFO
 -----------
 
-% for dt, dd, tt in elems:
-${dt}: ${dd}
+% for dt, dd, warn in c.data_items:
+  ${dt}${':' if dt else '---'}
+  ${dd}
+  % if warn and warn['message']:
+        ALERT_${warn['type'].upper()} ${warn['message']}
+  % endif
 % endfor
 
 PYTHON PACKAGES
 ---------------
 
-% for key, value in c.py_modules:
+% for key, value in c.py_modules['human_value']:
 ${key}: ${value}
 % endfor
 
 SYSTEM SETTINGS
 ---------------
 
-% for key, value in sorted(c.rhodecode_ini_safe.items()):
+% for key, value in sorted(c.rhodecode_config['human_value'].items()):
   % if isinstance(value, dict):
 
     % for key2, value2 in value.items():