system-info: fetch vcs settings from vcsserver. Fixes #4276

- redone the system info page. Errors/Warnings now show next to the
data, and also show up in API call.
- implemented fetching of git/hg/svn version from running vcsserver
- implemented fetching of vcsserver version from itself
- added more warnings/errors detection on RAM/Inodes etc
- few cleanups

rhodecode/lib/system_info.py

@@ -0,0 +1,582 b''
+import os
+import sys
+import time
+import platform
+import pkg_resources
+import logging
+
+
+log = logging.getLogger(__name__)
+
+
+psutil = None
+
+try:
+    # cygwin cannot have yet psutil support.
+    import psutil as psutil
+except ImportError:
+    pass
+
+
+_NA = 'NOT AVAILABLE'
+
+STATE_OK = 'ok'
+STATE_ERR = 'error'
+STATE_WARN = 'warning'
+
+STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK}
+
+
+# HELPERS
+def percentage(part, whole):
+    whole = float(whole)
+    if whole > 0:
+        return 100 * float(part) / whole
+    return 0
+
+
+def get_storage_size(storage_path):
+    sizes = []
+    for file_ in os.listdir(storage_path):
+        storage_file = os.path.join(storage_path, file_)
+        if os.path.isfile(storage_file):
+            try:
+                sizes.append(os.path.getsize(storage_file))
+            except OSError:
+                log.exception('Failed to get size of storage file %s',
+                              storage_file)
+                pass
+
+    return sum(sizes)
+
+
+class SysInfoRes(object):
+    def __init__(self, value, state=STATE_OK_DEFAULT, human_value=None):
+        self.value = value
+        self.state = state
+        self.human_value = human_value or value
+
+    def __json__(self):
+        return {
+            'value': self.value,
+            'state': self.state,
+            'human_value': self.human_value,
+        }
+
+    def __str__(self):
+        return '<SysInfoRes({})>'.format(self.__json__())
+
+
+class SysInfo(object):
+
+    def __init__(self, func_name, **kwargs):
+        self.func_name = func_name
+        self.value = _NA
+        self.state = None
+        self.kwargs = kwargs or {}
+
+    def __call__(self):
+        computed = self.compute(**self.kwargs)
+        if not isinstance(computed, SysInfoRes):
+            raise ValueError(
+                'computed value for {} is not instance of '
+                '{}, got {} instead'.format(
+                    self.func_name, SysInfoRes, type(computed)))
+        return computed.__json__()
+
+    def __str__(self):
+        return '<SysInfo({})>'.format(self.func_name)
+
+    def compute(self, **kwargs):
+        return self.func_name(**kwargs)
+
+
+# SysInfo functions
+def python_info():
+    value = {
+        'version': ' '.join(platform._sys_version()),
+        'executable': sys.executable
+    }
+    return SysInfoRes(value=value)
+
+
+def py_modules():
+    mods = dict([(p.project_name, p.version)
+                 for p in pkg_resources.working_set])
+    value = sorted(mods.items(), key=lambda k: k[0].lower())
+    return SysInfoRes(value=value)
+
+
+def platform_type():
+    from rhodecode.lib.utils import safe_unicode
+    value = safe_unicode(platform.platform())
+    return SysInfoRes(value=value)
+
+
+def uptime():
+    from rhodecode.lib.helpers import age, time_to_datetime
+    from rhodecode.translation import _
+
+    _uptime = _uptime_human = {'boot_time': 0, 'uptime': 0}
+    state = STATE_OK_DEFAULT
+    if not psutil:
+        return SysInfoRes(value=_uptime, state=state)
+
+    boot_time = psutil.boot_time()
+    _uptime['boot_time'] = boot_time
+    _uptime['uptime'] = time.time() - boot_time
+
+    _uptime_human['boot_time'] = time_to_datetime(boot_time)
+    _uptime_human['uptime'] = _('Server started {}').format(
+        age(time_to_datetime(boot_time)))
+
+    return SysInfoRes(value=_uptime, human_value=_uptime_human)
+
+
+def memory():
+    from rhodecode.lib.helpers import format_byte_size_binary
+    _memory = {'available': 0, 'used': 0, 'cached': 0, 'percent': 0,
+               'percent_used': 0, 'free': 0, 'inactive': 0, 'active': 0,
+               'shared': 0, 'total': 0, 'buffers': 0}
+    state = STATE_OK_DEFAULT
+    if not psutil:
+        return SysInfoRes(value=_memory, state=state)
+
+    # memory
+    _memory = dict(psutil.virtual_memory()._asdict())
+    _memory['percent_used'] = psutil._common.usage_percent(
+        (_memory['total'] - _memory['free']), _memory['total'], 1)
+
+    try:
+        human_value = '%s/%s, %s%% used' % (
+            format_byte_size_binary(_memory['used']),
+            format_byte_size_binary(_memory['total']),
+            _memory['percent_used'],)
+    except TypeError:
+        human_value = 'NOT AVAILABLE'
+
+    if state['type'] == STATE_OK and _memory['percent_used'] > 90:
+        msg = 'Critical: your available RAM memory is very low.'
+        state = {'message': msg, 'type': STATE_ERR}
+
+    elif state['type'] == STATE_OK and  _memory['percent_used'] > 70:
+        msg = 'Warning: your available RAM memory is running low.'
+        state = {'message': msg, 'type': STATE_WARN}
+
+    return SysInfoRes(value=_memory, state=state, human_value=human_value)
+
+
+def machine_load():
+    _load = {'1_min': _NA, '5_min': _NA, '15_min': _NA}
+
+    state = STATE_OK_DEFAULT
+    if not psutil:
+        return SysInfoRes(value=_load, state=state)
+
+    # load averages
+    if hasattr(psutil.os, 'getloadavg'):
+        _load = dict(zip(['1_min', '5_min', '15_min'], psutil.os.getloadavg()))
+
+    human_value = '1min: %s, 5min: %s, 15min: %s' % (
+    _load['1_min'], _load['5_min'], _load['15_min'])
+
+    # TODO: warn about too-much load 15 min
+    return SysInfoRes(value=_load, state=state, human_value=human_value)
+
+
+def cpu():
+    state = STATE_OK_DEFAULT
+    cpu_value = 0
+    if not psutil:
+        return SysInfoRes(value=cpu_value, state=state)
+
+    cpu_value = psutil.cpu_percent(0.5)
+    human_value = '{} %'.format(cpu_value)
+    return SysInfoRes(value=cpu_value, state=state, human_value=human_value)
+
+
+def storage():
+    from rhodecode.lib.helpers import format_byte_size_binary
+    from rhodecode.model.settings import VcsSettingsModel
+    path = VcsSettingsModel().get_repos_location()
+
+    # disk storage
+    disk = {'percent': 0, 'used': 0, 'total': 0, 'path': path, 'text': ''}
+    state = STATE_OK_DEFAULT
+    if not psutil:
+        return SysInfoRes(value=disk, state=state)
+
+    try:
+        disk.update(dict(psutil.disk_usage(path)._asdict()))
+    except Exception as e:
+        log.exception('Failed to fetch disk info')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = disk
+    human_value['text'] = "{}/{}, {}% used".format(
+        format_byte_size_binary(disk['used']),
+        format_byte_size_binary(disk['total']),
+        (disk['percent']))
+
+    if state['type'] == STATE_OK and disk['percent'] > 90:
+        msg = 'Critical: your disk space is very low.'
+        state = {'message': msg, 'type': STATE_ERR}
+
+    elif state['type'] == STATE_OK and disk['percent'] > 70:
+        msg = 'Warning: your disk space is running low.'
+        state = {'message': msg, 'type': STATE_WARN}
+
+    return SysInfoRes(value=disk, state=state, human_value=human_value)
+
+
+def storage_inodes():
+    from rhodecode.model.settings import VcsSettingsModel
+    path = VcsSettingsModel().get_repos_location()
+
+    _disk_inodes = dict(percent=0, free=0, used=0, total=0, path=path, text='')
+    state = STATE_OK_DEFAULT
+    if not psutil:
+        return SysInfoRes(value=_disk_inodes, state=state)
+
+    try:
+        i_stat = os.statvfs(path)
+
+        _disk_inodes['used'] = i_stat.f_ffree
+        _disk_inodes['free'] = i_stat.f_favail
+        _disk_inodes['total'] = i_stat.f_files
+        _disk_inodes['percent'] = percentage(
+            _disk_inodes['used'], _disk_inodes['total'])
+    except Exception as e:
+        log.exception('Failed to fetch disk inodes info')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = _disk_inodes
+    human_value['text'] = "{}/{}, {}% used".format(
+        _disk_inodes['used'], _disk_inodes['total'], _disk_inodes['percent'])
+
+    if state['type'] == STATE_OK and _disk_inodes['percent'] > 90:
+        msg = 'Critical: your disk free inodes are very low.'
+        state = {'message': msg, 'type': STATE_ERR}
+
+    elif state['type'] == STATE_OK and _disk_inodes['percent'] > 70:
+        msg = 'Warning: your disk free inodes are running low.'
+        state = {'message': msg, 'type': STATE_WARN}
+
+    return SysInfoRes(value=_disk_inodes, state=state)
+
+
+def storage_archives():
+    import rhodecode
+    from rhodecode.lib.utils import safe_str
+    from rhodecode.lib.helpers import format_byte_size_binary
+
+    msg = 'Enable this by setting ' \
+          'archive_cache_dir=/path/to/cache option in the .ini file'
+    path = safe_str(rhodecode.CONFIG.get('archive_cache_dir', msg))
+
+    disk_archive = dict(percent=0, used=0, total=0, items=0, path=path, text='')
+    state = STATE_OK_DEFAULT
+    try:
+        items_count = 0
+        used = 0
+        for root, dirs, files in os.walk(path):
+            if root == path:
+                items_count = len(dirs)
+
+            for f in files:
+                try:
+                    used += os.path.getsize(os.path.join(root, f))
+                except OSError:
+                    pass
+        disk_archive.update({
+            'percent': 100,
+            'used': used,
+            'total': used,
+            'items': items_count
+        })
+
+    except Exception as e:
+        log.exception('failed to fetch archive cache storage')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = disk_archive
+    human_value['text'] = "{} ({} items)".format(format_byte_size_binary(
+        disk_archive['used']), disk_archive['total'])
+
+    return SysInfoRes(value=disk_archive, state=state, human_value=human_value)
+
+
+def storage_gist():
+    from rhodecode.model.gist import GIST_STORE_LOC
+    from rhodecode.model.settings import VcsSettingsModel
+    from rhodecode.lib.utils import safe_str
+    from rhodecode.lib.helpers import format_byte_size_binary
+    path = safe_str(os.path.join(
+        VcsSettingsModel().get_repos_location(), GIST_STORE_LOC))
+
+    # gist storage
+    _disk_gist = dict(percent=0, used=0, total=0, items=0, path=path, text='')
+    state = STATE_OK_DEFAULT
+
+    try:
+        items_count = 0
+        used = 0
+        for root, dirs, files in os.walk(path):
+            if root == path:
+                items_count = len(dirs)
+
+            for f in files:
+                try:
+                    used += os.path.getsize(os.path.join(root, f))
+                except OSError:
+                    pass
+        _disk_gist.update({
+            'percent': 100,
+            'used': used,
+            'total': used,
+            'items': items_count
+        })
+    except Exception as e:
+        log.exception('failed to fetch gist storage items')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = _disk_gist
+    human_value['text'] = "{} ({} items)".format(format_byte_size_binary(
+        _disk_gist['used']), _disk_gist['items'])
+    return SysInfoRes(value=_disk_gist, state=state, human_value=human_value)
+
+
+def storage_search():
+    import rhodecode
+    path = rhodecode.CONFIG.get('search.location', '')
+
+    # search index storage
+    _disk_index = dict(percent=0, used=0, total=0, path=path, text='')
+    state = STATE_OK_DEFAULT
+    try:
+        search_index_storage_path_exists = os.path.isdir(path)
+        if search_index_storage_path_exists:
+            used = get_storage_size(path)
+            _disk_index.update({
+                'percent': 100,
+                'used': used,
+                'total': used,
+            })
+    except Exception as e:
+        log.exception('failed to fetch search index storage')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = _disk_index
+    human_value['text'] = "{}/{}, {}% used".format(
+        _disk_index['used'], _disk_index['total'], _disk_index['percent'])
+
+    return SysInfoRes(value=_disk_index, state=state, human_value=human_value)
+
+
+def git_info():
+    from rhodecode.lib.vcs.backends import git
+    state = STATE_OK_DEFAULT
+    value = human_value = ''
+    try:
+        value = git.discover_git_version(raise_on_exc=True)
+        human_value = 'version reported from VCSServer: {}'.format(value)
+    except Exception as e:
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    return SysInfoRes(value=value, state=state, human_value=human_value)
+
+
+def hg_info():
+    from rhodecode.lib.vcs.backends import hg
+    state = STATE_OK_DEFAULT
+    value = human_value = ''
+    try:
+        value = hg.discover_hg_version(raise_on_exc=True)
+        human_value = 'version reported from VCSServer: {}'.format(value)
+    except Exception as e:
+        state = {'message': str(e), 'type': STATE_ERR}
+    return SysInfoRes(value=value, state=state, human_value=human_value)
+
+
+def svn_info():
+    from rhodecode.lib.vcs.backends import svn
+    state = STATE_OK_DEFAULT
+    value = human_value = ''
+    try:
+        value = svn.discover_svn_version(raise_on_exc=True)
+        human_value = 'version reported from VCSServer: {}'.format(value)
+    except Exception as e:
+        state = {'message': str(e), 'type': STATE_ERR}
+    return SysInfoRes(value=value, state=state, human_value=human_value)
+
+
+def vcs_backends():
+    import rhodecode
+    value = rhodecode.CONFIG.get('vcs.backends', '').split(',')
+    human_value = 'Enabled backends in order: {}'.format(','.join(value))
+    return SysInfoRes(value=value, human_value=human_value)
+
+
+def vcs_server():
+    import rhodecode
+    from rhodecode.lib.vcs.backends import get_vcsserver_version
+
+    server_url = rhodecode.CONFIG.get('vcs.server')
+    enabled = rhodecode.CONFIG.get('vcs.server.enable')
+    protocol = rhodecode.CONFIG.get('vcs.server.protocol')
+    state = STATE_OK_DEFAULT
+    version = None
+
+    try:
+        version = get_vcsserver_version()
+        connection = 'connected'
+    except Exception as e:
+        connection = 'failed'
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    value = dict(
+        url=server_url,
+        enabled=enabled,
+        protocol=protocol,
+        connection=connection,
+        version=version,
+        text='',
+    )
+
+    human_value = value
+    human_value['text'] = \
+        '{url}@ver:{ver} via {mode} mode, connection:{conn}'.format(
+            url=server_url, ver=version, mode=protocol, conn=connection)
+
+    return SysInfoRes(value='', state=state, human_value=human_value)
+
+
+def rhodecode_app_info():
+    import rhodecode
+    return SysInfoRes(value={'rhodecode_version': rhodecode.__version__})
+
+
+def rhodecode_config():
+    import rhodecode
+    path = rhodecode.CONFIG.get('__file__')
+    rhodecode_ini_safe = rhodecode.CONFIG.copy()
+
+    blacklist = [
+        'rhodecode_license_key',
+        'routes.map',
+        'pylons.h',
+        'pylons.app_globals',
+        'pylons.environ_config',
+        'sqlalchemy.db1.url',
+        'channelstream.secret',
+        'beaker.session.secret',
+        'rhodecode.encrypted_values.secret',
+        'rhodecode_auth_github_consumer_key',
+        'rhodecode_auth_github_consumer_secret',
+        'rhodecode_auth_google_consumer_key',
+        'rhodecode_auth_google_consumer_secret',
+        'rhodecode_auth_bitbucket_consumer_secret',
+        'rhodecode_auth_bitbucket_consumer_key',
+        'rhodecode_auth_twitter_consumer_secret',
+        'rhodecode_auth_twitter_consumer_key',
+
+        'rhodecode_auth_twitter_secret',
+        'rhodecode_auth_github_secret',
+        'rhodecode_auth_google_secret',
+        'rhodecode_auth_bitbucket_secret',
+
+        'appenlight.api_key',
+        ('app_conf', 'sqlalchemy.db1.url')
+    ]
+    for k in blacklist:
+        if isinstance(k, tuple):
+            section, key = k
+            if section in rhodecode_ini_safe:
+                rhodecode_ini_safe[section] = '**OBFUSCATED**'
+        else:
+            rhodecode_ini_safe.pop(k, None)
+
+    # TODO: maybe put some CONFIG checks here ?
+    return SysInfoRes(value={'config': rhodecode_ini_safe, 'path': path})
+
+
+def database_info():
+    import rhodecode
+    from sqlalchemy.engine import url as engine_url
+    from rhodecode.model.meta import Base as sql_base, Session
+    from rhodecode.model.db import DbMigrateVersion
+
+    state = STATE_OK_DEFAULT
+
+    db_migrate = DbMigrateVersion.query().filter(
+        DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
+
+    db_url_obj = engine_url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
+
+    try:
+        engine = sql_base.metadata.bind
+        db_server_info = engine.dialect._get_server_version_info(
+            Session.connection(bind=engine))
+        db_version = '.'.join(map(str, db_server_info))
+    except Exception:
+        log.exception('failed to fetch db version')
+        db_version = 'UNKNOWN'
+
+    db_info = dict(
+        migrate_version=db_migrate.version,
+        type=db_url_obj.get_backend_name(),
+        version=db_version,
+        url=repr(db_url_obj)
+    )
+
+    human_value = db_info
+    human_value['url'] = "{} @ migration version: {}".format(
+        db_info['url'], db_info['migrate_version'])
+    human_value['version'] = "{} {}".format(db_info['type'], db_info['version'])
+    return SysInfoRes(value=db_info, state=state, human_value=human_value)
+
+
+def server_info(environ):
+    import rhodecode
+    from rhodecode.lib.base import get_server_ip_addr, get_server_port
+
+    value = {
+        'server_ip': '%s:%s' % (
+            get_server_ip_addr(environ, log_errors=False),
+            get_server_port(environ)
+        ),
+        'server_id': rhodecode.CONFIG.get('instance_id'),
+    }
+    return SysInfoRes(value=value)
+
+
+def get_system_info(environ):
+    environ = environ or {}
+    return {
+        'rhodecode_app': SysInfo(rhodecode_app_info)(),
+        'rhodecode_config': SysInfo(rhodecode_config)(),
+        'python': SysInfo(python_info)(),
+        'py_modules': SysInfo(py_modules)(),
+
+        'platform': SysInfo(platform_type)(),
+        'server': SysInfo(server_info, environ=environ)(),
+        'database': SysInfo(database_info)(),
+
+        'storage': SysInfo(storage)(),
+        'storage_inodes': SysInfo(storage_inodes)(),
+        'storage_archive': SysInfo(storage_archives)(),
+        'storage_search': SysInfo(storage_search)(),
+        'storage_gist': SysInfo(storage_gist)(),
+
+        'uptime': SysInfo(uptime)(),
+        'load': SysInfo(machine_load)(),
+        'cpu': SysInfo(cpu)(),
+        'memory': SysInfo(memory)(),
+
+        'vcs_backends': SysInfo(vcs_backends)(),
+        'vcs_server': SysInfo(vcs_server)(),
+
+        'git': SysInfo(git_info)(),
+        'hg': SysInfo(hg_info)(),
+        'svn': SysInfo(svn_info)(),
+    }

rhodecode/tests/lib/test_system_info.py

@@ -0,0 +1,15 b''
+import py.test
+
+from rhodecode.lib.system_info import get_system_info
+
+
+def test_system_info(app):
+    info = get_system_info({})
+    assert info['load']['value']['15_min'] != 'NOT AVAILABLE'
+
+
+def test_system_info_without_psutil(monkeypatch, app):
+    import rhodecode.lib.system_info
+    monkeypatch.setattr(rhodecode.lib.system_info, 'psutil', None)
+    info = get_system_info({})
+    assert info['load']['value']['15_min'] == 'NOT AVAILABLE'

rhodecode/api/tests/test_get_server_info.py

@@ -1,67 +1,67 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import pytest
 
 from rhodecode.model.scm import ScmModel
 from rhodecode.api.tests.utils import build_data, api_call, assert_ok
 
 
 @pytest.fixture
 def http_host_stub():
     """
     To ensure that we can get an IP address, this test shall run with a
     hostname set to "localhost".
     """
     return 'localhost:80'
 
 
 @pytest.mark.usefixtures("testuser_api", "app")
 class TestGetServerInfo(object):
     def test_api_get_server_info(self):
         id_, params = build_data(self.apikey, 'get_server_info')
         response = api_call(self.app, params)
         resp = response.json
         expected = ScmModel().get_server_info()
         expected['memory'] = resp['result']['memory']
         expected['uptime'] = resp['result']['uptime']
         expected['load'] = resp['result']['load']
         expected['cpu'] = resp['result']['cpu']
-        expected['disk'] = resp['result']['disk']
-        expected['disk_inodes'] = resp['result']['disk_inodes']
-        expected['server_ip'] = '127.0.0.1:80'
+        expected['storage'] = resp['result']['storage']
+        expected['storage_inodes'] = resp['result']['storage_inodes']
+        expected['server'] = resp['result']['server']
 
         assert_ok(id_, expected, given=response.body)
 
     def test_api_get_server_info_ip(self):
         id_, params = build_data(self.apikey, 'get_server_info')
         response = api_call(self.app, params)
         resp = response.json
         expected = ScmModel().get_server_info({'SERVER_NAME': 'unknown'})
         expected['memory'] = resp['result']['memory']
         expected['uptime'] = resp['result']['uptime']
         expected['load'] = resp['result']['load']
         expected['cpu'] = resp['result']['cpu']
-        expected['disk'] = resp['result']['disk']
-        expected['disk_inodes'] = resp['result']['disk_inodes']
-        expected['server_ip'] = '127.0.0.1:80'
+        expected['storage'] = resp['result']['storage']
+        expected['storage_inodes'] = resp['result']['storage_inodes']
+        expected['server'] = resp['result']['server']
 
         assert_ok(id_, expected, given=response.body)

rhodecode/controllers/admin/settings.py

@@ -1,814 +1,837 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 settings controller for rhodecode admin
 """
 
 import collections
 import logging
 import urllib2
 
 import datetime
 import formencode
 from formencode import htmlfill
 import packaging.version
 from pylons import request, tmpl_context as c, url, config
 from pylons.controllers.util import redirect
 from pylons.i18n.translation import _, lazy_ugettext
 from pyramid.threadlocal import get_current_registry
 from webob.exc import HTTPBadRequest
 
 import rhodecode
 from rhodecode.admin.navigation import navigation_list
 from rhodecode.lib import auth
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
 from rhodecode.lib.base import BaseController, render
 from rhodecode.lib.celerylib import tasks, run_task
 from rhodecode.lib.utils import repo2db_mapper
 from rhodecode.lib.utils2 import (
     str2bool, safe_unicode, AttributeDict, safe_int)
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils import jsonify
 
 from rhodecode.model.db import RhodeCodeUi, Repository
 from rhodecode.model.forms import ApplicationSettingsForm, \
     ApplicationUiSettingsForm, ApplicationVisualisationForm, \
     LabsSettingsForm, IssueTrackerPatternsForm
 from rhodecode.model.repo_group import RepoGroupModel
 
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.notification import EmailNotificationModel
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import (
     IssueTrackerSettingsModel, VcsSettingsModel, SettingNotFound,
     SettingsModel)
 
 from rhodecode.model.supervisor import SupervisorModel, SUPERVISOR_MASTER
 from rhodecode.svn_support.config_keys import generate_config
 
 
 log = logging.getLogger(__name__)
 
 
 class SettingsController(BaseController):
     """REST Controller styled on the Atom Publishing Protocol"""
     # To properly map this controller, ensure your config/routing.py
     # file has a resource setup:
     #     map.resource('setting', 'settings', controller='admin/settings',
     #         path_prefix='/admin', name_prefix='admin_')
 
     @LoginRequired()
     def __before__(self):
         super(SettingsController, self).__before__()
         c.labs_active = str2bool(
             rhodecode.CONFIG.get('labs_settings_active', 'true'))
         c.navlist = navigation_list(request)
 
     def _get_hg_ui_settings(self):
         ret = RhodeCodeUi.query().all()
 
         if not ret:
             raise Exception('Could not get application ui settings !')
         settings = {}
         for each in ret:
             k = each.ui_key
             v = each.ui_value
             if k == '/':
                 k = 'root_path'
 
             if k in ['push_ssl', 'publish']:
                 v = str2bool(v)
 
             if k.find('.') != -1:
                 k = k.replace('.', '_')
 
             if each.ui_section in ['hooks', 'extensions']:
                 v = each.ui_active
 
             settings[each.ui_section + '_' + k] = v
         return settings
 
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
     @jsonify
     def delete_svn_pattern(self):
         if not request.is_xhr:
             raise HTTPBadRequest()
 
         delete_pattern_id = request.POST.get('delete_svn_pattern')
         model = VcsSettingsModel()
         try:
             model.delete_global_svn_pattern(delete_pattern_id)
         except SettingNotFound:
             raise HTTPBadRequest()
 
         Session().commit()
         return True
 
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
     def settings_vcs_update(self):
         """POST /admin/settings: All items in the collection"""
         # url('admin_settings_vcs')
         c.active = 'vcs'
 
         model = VcsSettingsModel()
         c.svn_branch_patterns = model.get_global_svn_branch_patterns()
         c.svn_tag_patterns = model.get_global_svn_tag_patterns()
 
         # TODO: Replace with request.registry after migrating to pyramid.
         pyramid_settings = get_current_registry().settings
         c.svn_proxy_generate_config = pyramid_settings[generate_config]
 
         application_form = ApplicationUiSettingsForm()()
 
         try:
             form_result = application_form.to_python(dict(request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             return htmlfill.render(
                 render('admin/settings/settings.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
 
         try:
             if c.visual.allow_repo_location_change:
                 model.update_global_path_setting(
                     form_result['paths_root_path'])
 
             model.update_global_ssl_setting(form_result['web_push_ssl'])
             model.update_global_hook_settings(form_result)
 
             model.create_or_update_global_svn_settings(form_result)
             model.create_or_update_global_hg_settings(form_result)
             model.create_or_update_global_pr_settings(form_result)
         except Exception:
             log.exception("Exception while updating settings")
             h.flash(_('Error occurred during updating '
                       'application settings'), category='error')
         else:
             Session().commit()
             h.flash(_('Updated VCS settings'), category='success')
             return redirect(url('admin_settings_vcs'))
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_vcs(self):
         """GET /admin/settings: All items in the collection"""
         # url('admin_settings_vcs')
         c.active = 'vcs'
         model = VcsSettingsModel()
         c.svn_branch_patterns = model.get_global_svn_branch_patterns()
         c.svn_tag_patterns = model.get_global_svn_tag_patterns()
 
         # TODO: Replace with request.registry after migrating to pyramid.
         pyramid_settings = get_current_registry().settings
         c.svn_proxy_generate_config = pyramid_settings[generate_config]
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
     def settings_mapping_update(self):
         """POST /admin/settings/mapping: All items in the collection"""
         # url('admin_settings_mapping')
         c.active = 'mapping'
         rm_obsolete = request.POST.get('destroy', False)
         invalidate_cache = request.POST.get('invalidate', False)
         log.debug(
             'rescanning repo location with destroy obsolete=%s', rm_obsolete)
 
         if invalidate_cache:
             log.debug('invalidating all repositories cache')
             for repo in Repository.get_all():
                 ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
 
         filesystem_repos = ScmModel().repo_scan()
         added, removed = repo2db_mapper(filesystem_repos, rm_obsolete)
         _repr = lambda l: ', '.join(map(safe_unicode, l)) or '-'
         h.flash(_('Repositories successfully '
                   'rescanned added: %s ; removed: %s') %
                 (_repr(added), _repr(removed)),
                 category='success')
         return redirect(url('admin_settings_mapping'))
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_mapping(self):
         """GET /admin/settings/mapping: All items in the collection"""
         # url('admin_settings_mapping')
         c.active = 'mapping'
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
     def settings_global_update(self):
         """POST /admin/settings/global: All items in the collection"""
         # url('admin_settings_global')
         c.active = 'global'
         c.personal_repo_group_default_pattern = RepoGroupModel()\
             .get_personal_group_name_pattern()
         application_form = ApplicationSettingsForm()()
         try:
             form_result = application_form.to_python(dict(request.POST))
         except formencode.Invalid as errors:
             return htmlfill.render(
                 render('admin/settings/settings.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False)
 
         try:
             settings = [
                 ('title', 'rhodecode_title', 'unicode'),
                 ('realm', 'rhodecode_realm', 'unicode'),
                 ('pre_code', 'rhodecode_pre_code', 'unicode'),
                 ('post_code', 'rhodecode_post_code', 'unicode'),
                 ('captcha_public_key', 'rhodecode_captcha_public_key', 'unicode'),
                 ('captcha_private_key', 'rhodecode_captcha_private_key', 'unicode'),
                 ('create_personal_repo_group', 'rhodecode_create_personal_repo_group', 'bool'),
                 ('personal_repo_group_pattern', 'rhodecode_personal_repo_group_pattern', 'unicode'),
             ]
             for setting, form_key, type_ in settings:
                 sett = SettingsModel().create_or_update_setting(
                     setting, form_result[form_key], type_)
                 Session().add(sett)
 
             Session().commit()
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated application settings'), category='success')
         except Exception:
             log.exception("Exception while updating application settings")
             h.flash(
                 _('Error occurred during updating application settings'),
                 category='error')
 
         return redirect(url('admin_settings_global'))
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_global(self):
         """GET /admin/settings/global: All items in the collection"""
         # url('admin_settings_global')
         c.active = 'global'
         c.personal_repo_group_default_pattern = RepoGroupModel()\
             .get_personal_group_name_pattern()
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
     def settings_visual_update(self):
         """POST /admin/settings/visual: All items in the collection"""
         # url('admin_settings_visual')
         c.active = 'visual'
         application_form = ApplicationVisualisationForm()()
         try:
             form_result = application_form.to_python(dict(request.POST))
         except formencode.Invalid as errors:
             return htmlfill.render(
                 render('admin/settings/settings.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
 
         try:
             settings = [
                 ('show_public_icon', 'rhodecode_show_public_icon', 'bool'),
                 ('show_private_icon', 'rhodecode_show_private_icon', 'bool'),
                 ('stylify_metatags', 'rhodecode_stylify_metatags', 'bool'),
                 ('repository_fields', 'rhodecode_repository_fields', 'bool'),
                 ('dashboard_items', 'rhodecode_dashboard_items', 'int'),
                 ('admin_grid_items', 'rhodecode_admin_grid_items', 'int'),
                 ('show_version', 'rhodecode_show_version', 'bool'),
                 ('use_gravatar', 'rhodecode_use_gravatar', 'bool'),
                 ('markup_renderer', 'rhodecode_markup_renderer', 'unicode'),
                 ('gravatar_url', 'rhodecode_gravatar_url', 'unicode'),
                 ('clone_uri_tmpl', 'rhodecode_clone_uri_tmpl', 'unicode'),
                 ('support_url', 'rhodecode_support_url', 'unicode'),
                 ('show_revision_number', 'rhodecode_show_revision_number', 'bool'),
                 ('show_sha_length', 'rhodecode_show_sha_length', 'int'),
             ]
             for setting, form_key, type_ in settings:
                 sett = SettingsModel().create_or_update_setting(
                     setting, form_result[form_key], type_)
                 Session().add(sett)
 
             Session().commit()
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated visualisation settings'), category='success')
         except Exception:
             log.exception("Exception updating visualization settings")
             h.flash(_('Error occurred during updating '
                       'visualisation settings'),
                     category='error')
 
         return redirect(url('admin_settings_visual'))
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_visual(self):
         """GET /admin/settings/visual: All items in the collection"""
         # url('admin_settings_visual')
         c.active = 'visual'
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
     def settings_issuetracker_test(self):
         if request.is_xhr:
             return h.urlify_commit_message(
                 request.POST.get('test_text', ''),
                 'repo_group/test_repo1')
         else:
             raise HTTPBadRequest()
 
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
     def settings_issuetracker_delete(self):
         uid = request.POST.get('uid')
         IssueTrackerSettingsModel().delete_entries(uid)
         h.flash(_('Removed issue tracker entry'), category='success')
         return redirect(url('admin_settings_issuetracker'))
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_issuetracker(self):
         """GET /admin/settings/issue-tracker: All items in the collection"""
         # url('admin_settings_issuetracker')
         c.active = 'issuetracker'
         defaults = SettingsModel().get_all_settings()
 
         entry_key = 'rhodecode_issuetracker_pat_'
 
         c.issuetracker_entries = {}
         for k, v in defaults.items():
             if k.startswith(entry_key):
                 uid = k[len(entry_key):]
                 c.issuetracker_entries[uid] = None
 
         for uid in c.issuetracker_entries:
             c.issuetracker_entries[uid] = AttributeDict({
                 'pat': defaults.get('rhodecode_issuetracker_pat_' + uid),
                 'url': defaults.get('rhodecode_issuetracker_url_' + uid),
                 'pref': defaults.get('rhodecode_issuetracker_pref_' + uid),
                 'desc': defaults.get('rhodecode_issuetracker_desc_' + uid),
             })
 
         return render('admin/settings/settings.html')
 
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
     def settings_issuetracker_save(self):
         settings_model = IssueTrackerSettingsModel()
 
         form = IssueTrackerPatternsForm()().to_python(request.POST)
         if form:
             for uid in form.get('delete_patterns', []):
                 settings_model.delete_entries(uid)
 
             for pattern in form.get('patterns', []):
                 for setting, value, type_ in pattern:
                     sett = settings_model.create_or_update_setting(
                         setting, value, type_)
                     Session().add(sett)
 
                 Session().commit()
 
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated issue tracker entries'), category='success')
         return redirect(url('admin_settings_issuetracker'))
 
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
     def settings_email_update(self):
         """POST /admin/settings/email: All items in the collection"""
         # url('admin_settings_email')
         c.active = 'email'
 
         test_email = request.POST.get('test_email')
 
         if not test_email:
             h.flash(_('Please enter email address'), category='error')
             return redirect(url('admin_settings_email'))
 
         email_kwargs = {
             'date': datetime.datetime.now(),
             'user': c.rhodecode_user,
             'rhodecode_version': c.rhodecode_version
         }
 
         (subject, headers, email_body,
          email_body_plaintext) = EmailNotificationModel().render_email(
             EmailNotificationModel.TYPE_EMAIL_TEST, **email_kwargs)
 
         recipients = [test_email] if test_email else None
 
         run_task(tasks.send_email, recipients, subject,
                  email_body_plaintext, email_body)
 
         h.flash(_('Send email task created'), category='success')
         return redirect(url('admin_settings_email'))
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_email(self):
         """GET /admin/settings/email: All items in the collection"""
         # url('admin_settings_email')
         c.active = 'email'
         c.rhodecode_ini = rhodecode.CONFIG
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
     def settings_hooks_update(self):
         """POST or DELETE /admin/settings/hooks: All items in the collection"""
         # url('admin_settings_hooks')
         c.active = 'hooks'
         if c.visual.allow_custom_hooks_settings:
             ui_key = request.POST.get('new_hook_ui_key')
             ui_value = request.POST.get('new_hook_ui_value')
 
             hook_id = request.POST.get('hook_id')
             new_hook = False
 
             model = SettingsModel()
             try:
                 if ui_value and ui_key:
                     model.create_or_update_hook(ui_key, ui_value)
                     h.flash(_('Added new hook'), category='success')
                     new_hook = True
                 elif hook_id:
                     RhodeCodeUi.delete(hook_id)
                     Session().commit()
 
                 # check for edits
                 update = False
                 _d = request.POST.dict_of_lists()
                 for k, v in zip(_d.get('hook_ui_key', []),
                                 _d.get('hook_ui_value_new', [])):
                     model.create_or_update_hook(k, v)
                     update = True
 
                 if update and not new_hook:
                     h.flash(_('Updated hooks'), category='success')
                 Session().commit()
             except Exception:
                 log.exception("Exception during hook creation")
                 h.flash(_('Error occurred during hook creation'),
                         category='error')
 
         return redirect(url('admin_settings_hooks'))
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_hooks(self):
         """GET /admin/settings/hooks: All items in the collection"""
         # url('admin_settings_hooks')
         c.active = 'hooks'
 
         model = SettingsModel()
         c.hooks = model.get_builtin_hooks()
         c.custom_hooks = model.get_custom_hooks()
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False)
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_search(self):
         """GET /admin/settings/search: All items in the collection"""
         # url('admin_settings_search')
         c.active = 'search'
 
         from rhodecode.lib.index import searcher_from_config
         searcher = searcher_from_config(config)
         c.statistics = searcher.statistics()
 
         return render('admin/settings/settings.html')
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_system(self):
         """GET /admin/settings/system: All items in the collection"""
         # url('admin_settings_system')
         snapshot = str2bool(request.GET.get('snapshot'))
-        c.active = 'system'
+        defaults = self._form_defaults()
 
-        defaults = self._form_defaults()
-        c.rhodecode_ini = rhodecode.CONFIG
+        c.active = 'system'
         c.rhodecode_update_url = defaults.get('rhodecode_update_url')
         server_info = ScmModel().get_server_info(request.environ)
+
         for key, val in server_info.iteritems():
             setattr(c, key, val)
 
-        if c.disk['percent'] > 90:
-            h.flash(h.literal(_(
-                'Critical: your disk space is very low <b>%s%%</b> used' %
-                c.disk['percent'])), 'error')
-        elif c.disk['percent'] > 70:
-            h.flash(h.literal(_(
-                'Warning: your disk space is running low <b>%s%%</b> used' %
-                c.disk['percent'])), 'warning')
+        def val(name, subkey='human_value'):
+            return server_info[name][subkey]
+
+        def state(name):
+            return server_info[name]['state']
+
+        def val2(name):
+            val = server_info[name]['human_value']
+            state = server_info[name]['state']
+            return val, state
 
-        try:
-            c.uptime_age = h._age(
-                h.time_to_datetime(c.boot_time), False, show_suffix=False)
-        except TypeError:
-            c.uptime_age = c.boot_time
+        c.data_items = [
+            # update info
+            (_('Update info'), h.literal(
+                '<span class="link" id="check_for_update" >%s.</span>' % (
+                _('Check for updates')) +
+                '<br/> <span >%s.</span>' % (_('Note: please make sure this server can access `%s` for the update link to work') % c.rhodecode_update_url)
+            ), ''),
+
+            # RhodeCode specific
+            (_('RhodeCode Version'), c.rhodecode_version, ''),
+            (_('RhodeCode Server IP'), val('server')['server_ip'], state('server')),
+            (_('RhodeCode Server ID'), val('server')['server_id'], state('server')),
+            (_('RhodeCode Configuration'), val('rhodecode_config')['path'], state('rhodecode_config')),
+            ('', '', ''),  # spacer
+
+            # Database
+            (_('Database'), val('database')['url'], state('database')),
+            (_('Database version'), val('database')['version'], state('database')),
+            ('', '', ''),  # spacer
 
-        try:
-            c.system_memory = '%s/%s, %s%% (%s%%) used%s' % (
-                h.format_byte_size_binary(c.memory['used']),
-                h.format_byte_size_binary(c.memory['total']),
-                c.memory['percent2'],
-                c.memory['percent'],
-                ' %s' % c.memory['error'] if 'error' in c.memory else '')
-        except TypeError:
-            c.system_memory = 'NOT AVAILABLE'
+            # Platform/Python
+            (_('Platform'), val('platform'), state('platform')),
+            (_('Python version'), val('python')['version'], state('python')),
+            (_('Python path'), val('python')['executable'], state('python')),
+            ('', '', ''),  # spacer
+
+            # Systems stats
+            (_('CPU'), val('cpu'), state('cpu')),
+            (_('Load'), val('load'), state('load')),
+            (_('Memory'), val('memory'), state('memory')),
+            (_('Uptime'), val('uptime')['uptime'], state('uptime')),
+            ('', '', ''),  # spacer
+
+            # Repo storage
+            (_('Storage location'), val('storage')['path'], state('storage')),
+            (_('Storage info'), val('storage')['text'], state('storage')),
+            (_('Storage inodes'), val('storage_inodes')['text'], state('storage_inodes')),
 
-        rhodecode_ini_safe = rhodecode.CONFIG.copy()
-        blacklist = [
-            'rhodecode_license_key',
-            'routes.map',
-            'pylons.h',
-            'pylons.app_globals',
-            'pylons.environ_config',
-            'sqlalchemy.db1.url',
-            ('app_conf', 'sqlalchemy.db1.url')
+            (_('Gist storage location'), val('storage_gist')['path'], state('storage_gist')),
+            (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
+
+            (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
+            (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
+
+            (_('Search storage'), val('storage_search')['path'], state('storage_search')),
+            (_('Search info'), val('storage_search')['text'], state('storage_search')),
+            ('', '', ''),  # spacer
+
+            # VCS specific
+            (_('VCS Backends'), val('vcs_backends'), state('vcs_backends')),
+            (_('VCS Server'), val('vcs_server')['text'], state('vcs_server')),
+            (_('GIT'), val('git'), state('git')),
+            (_('HG'), val('hg'), state('hg')),
+            (_('SVN'), val('svn'), state('svn')),
+
         ]
-        for k in blacklist:
-            if isinstance(k, tuple):
-                section, key = k
-                if section in rhodecode_ini_safe:
-                    rhodecode_ini_safe[section].pop(key, None)
-            else:
-                rhodecode_ini_safe.pop(k, None)
-
-        c.rhodecode_ini_safe = rhodecode_ini_safe
 
         # TODO: marcink, figure out how to allow only selected users to do this
-        c.allowed_to_snapshot = False
+        c.allowed_to_snapshot = c.rhodecode_user.admin
 
         if snapshot:
             if c.allowed_to_snapshot:
                 return render('admin/settings/settings_system_snapshot.html')
             else:
                 h.flash('You are not allowed to do this', category='warning')
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False)
 
     @staticmethod
     def get_update_data(update_url):
         """Return the JSON update data."""
         ver = rhodecode.__version__
         log.debug('Checking for upgrade on `%s` server', update_url)
         opener = urllib2.build_opener()
         opener.addheaders = [('User-agent', 'RhodeCode-SCM/%s' % ver)]
         response = opener.open(update_url)
         response_data = response.read()
         data = json.loads(response_data)
 
         return data
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_system_update(self):
         """GET /admin/settings/system/updates: All items in the collection"""
         # url('admin_settings_system_update')
         defaults = self._form_defaults()
         update_url = defaults.get('rhodecode_update_url', '')
 
         _err = lambda s: '<div style="color:#ff8888; padding:4px 0px">%s</div>' % (s)
         try:
             data = self.get_update_data(update_url)
         except urllib2.URLError as e:
             log.exception("Exception contacting upgrade server")
             return _err('Failed to contact upgrade server: %r' % e)
         except ValueError as e:
             log.exception("Bad data sent from update server")
             return _err('Bad data sent from update server')
 
         latest = data['versions'][0]
 
         c.update_url = update_url
         c.latest_data = latest
         c.latest_ver = latest['version']
         c.cur_ver = rhodecode.__version__
         c.should_upgrade = False
 
         if (packaging.version.Version(c.latest_ver) >
                 packaging.version.Version(c.cur_ver)):
             c.should_upgrade = True
         c.important_notices = latest['general']
 
         return render('admin/settings/settings_system_update.html')
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_supervisor(self):
         c.rhodecode_ini = rhodecode.CONFIG
         c.active = 'supervisor'
 
         c.supervisor_procs = OrderedDict([
             (SUPERVISOR_MASTER, {}),
         ])
 
         c.log_size = 10240
         supervisor = SupervisorModel()
 
         _connection = supervisor.get_connection(
             c.rhodecode_ini.get('supervisor.uri'))
         c.connection_error = None
         try:
             _connection.supervisor.getAllProcessInfo()
         except Exception as e:
             c.connection_error = str(e)
             log.exception("Exception reading supervisor data")
             return render('admin/settings/settings.html')
 
         groupid = c.rhodecode_ini.get('supervisor.group_id')
 
         # feed our group processes to the main
         for proc in supervisor.get_group_processes(_connection, groupid):
             c.supervisor_procs[proc['name']] = {}
 
         for k in c.supervisor_procs.keys():
             try:
                 # master process info
                 if k == SUPERVISOR_MASTER:
                     _data = supervisor.get_master_state(_connection)
                     _data['name'] = 'supervisor master'
                     _data['description'] = 'pid %s, id: %s, ver: %s' % (
                         _data['pid'], _data['id'], _data['ver'])
                     c.supervisor_procs[k] = _data
                 else:
                     procid = groupid + ":" + k
                     c.supervisor_procs[k] = supervisor.get_process_info(_connection, procid)
             except Exception as e:
                 log.exception("Exception reading supervisor data")
                 c.supervisor_procs[k] = {'_rhodecode_error': str(e)}
 
         return render('admin/settings/settings.html')
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_supervisor_log(self, procid):
         import rhodecode
         c.rhodecode_ini = rhodecode.CONFIG
         c.active = 'supervisor_tail'
 
         supervisor = SupervisorModel()
         _connection = supervisor.get_connection(c.rhodecode_ini.get('supervisor.uri'))
         groupid = c.rhodecode_ini.get('supervisor.group_id')
         procid = groupid + ":" + procid if procid != SUPERVISOR_MASTER else procid
 
         c.log_size = 10240
         offset = abs(safe_int(request.GET.get('offset', c.log_size))) * -1
         c.log = supervisor.read_process_log(_connection, procid, offset, 0)
 
         return render('admin/settings/settings.html')
 
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
     def settings_labs_update(self):
         """POST /admin/settings/labs: All items in the collection"""
         # url('admin_settings/labs', method={'POST'})
         c.active = 'labs'
 
         application_form = LabsSettingsForm()()
         try:
             form_result = application_form.to_python(dict(request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _('Some form inputs contain invalid data.'),
                 category='error')
             return htmlfill.render(
                 render('admin/settings/settings.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding='UTF-8',
                 force_defaults=False
             )
 
         try:
             session = Session()
             for setting in _LAB_SETTINGS:
                 setting_name = setting.key[len('rhodecode_'):]
                 sett = SettingsModel().create_or_update_setting(
                     setting_name, form_result[setting.key], setting.type)
                 session.add(sett)
 
         except Exception:
             log.exception('Exception while updating lab settings')
             h.flash(_('Error occurred during updating labs settings'),
                     category='error')
         else:
             Session().commit()
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated Labs settings'), category='success')
             return redirect(url('admin_settings_labs'))
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=self._form_defaults(),
             encoding='UTF-8',
             force_defaults=False)
 
     @HasPermissionAllDecorator('hg.admin')
     def settings_labs(self):
         """GET /admin/settings/labs: All items in the collection"""
         # url('admin_settings_labs')
         if not c.labs_active:
             redirect(url('admin_settings'))
 
         c.active = 'labs'
         c.lab_settings = _LAB_SETTINGS
 
         return htmlfill.render(
             render('admin/settings/settings.html'),
             defaults=self._form_defaults(),
             encoding='UTF-8',
             force_defaults=False)
 
     def _form_defaults(self):
         defaults = SettingsModel().get_all_settings()
         defaults.update(self._get_hg_ui_settings())
         defaults.update({
             'new_svn_branch': '',
             'new_svn_tag': '',
         })
         return defaults
 
 
 # :param key: name of the setting including the 'rhodecode_' prefix
 # :param type: the RhodeCodeSetting type to use.
 # :param group: the i18ned group in which we should dispaly this setting
 # :param label: the i18ned label we should display for this setting
 # :param help: the i18ned help we should dispaly for this setting
 LabSetting = collections.namedtuple(
     'LabSetting', ('key', 'type', 'group', 'label', 'help'))
 
 
 # This list has to be kept in sync with the form
 # rhodecode.model.forms.LabsSettingsForm.
 _LAB_SETTINGS = [
 
 ]

rhodecode/lib/vcs/__init__.py

@@ -1,300 +1,304 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Various version Control System version lib (vcs) management abstraction layer
 for Python. Build with server client architecture.
 """
 
 
 VERSION = (0, 5, 0, 'dev')
 
 __version__ = '.'.join((str(each) for each in VERSION[:4]))
 
 __all__ = [
     'get_version', 'get_vcs_instance', 'get_backend',
     'VCSError', 'RepositoryError', 'CommitError'
     ]
 
 import atexit
 import logging
 import subprocess32
 import time
 import urlparse
 from cStringIO import StringIO
 
 import Pyro4
 from Pyro4.errors import CommunicationError
 
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.backends import get_vcs_instance, get_backend
 from rhodecode.lib.vcs.exceptions import (
     VCSError, RepositoryError, CommitError, VCSCommunicationError)
 
 log = logging.getLogger(__name__)
 
 # The pycurl library directly accesses C API functions and is not patched by
 # gevent. This will potentially lead to deadlocks due to incompatibility to
 # gevent. Therefore we check if gevent is active and import a gevent compatible
 # wrapper in that case.
 try:
     from gevent import monkey
     if monkey.is_module_patched('__builtin__'):
         import geventcurl as pycurl
         log.debug('Using gevent comapatible pycurl: %s', pycurl)
     else:
         import pycurl
 except ImportError:
     import pycurl
 
 
 def get_version():
     """
     Returns shorter version (digit parts only) as string.
     """
     return '.'.join((str(each) for each in VERSION[:3]))
 
 
 def connect_pyro4(server_and_port):
     from rhodecode.lib.vcs import connection, client
     from rhodecode.lib.middleware.utils import scm_app
 
     git_remote = client.RequestScopeProxyFactory(
         settings.pyro_remote(settings.PYRO_GIT, server_and_port))
     hg_remote = client.RequestScopeProxyFactory(
         settings.pyro_remote(settings.PYRO_HG, server_and_port))
     svn_remote = client.RequestScopeProxyFactory(
         settings.pyro_remote(settings.PYRO_SVN, server_and_port))
 
     connection.Git = client.RepoMaker(proxy_factory=git_remote)
     connection.Hg = client.RepoMaker(proxy_factory=hg_remote)
     connection.Svn = client.RepoMaker(proxy_factory=svn_remote)
 
     scm_app.GIT_REMOTE_WSGI = Pyro4.Proxy(
         settings.pyro_remote(
             settings.PYRO_GIT_REMOTE_WSGI, server_and_port))
     scm_app.HG_REMOTE_WSGI = Pyro4.Proxy(
         settings.pyro_remote(
             settings.PYRO_HG_REMOTE_WSGI, server_and_port))
 
     @atexit.register
     def free_connection_resources():
         connection.Git = None
         connection.Hg = None
         connection.Svn = None
+        connection.Service = None
 
 
 def connect_http(server_and_port):
     from rhodecode.lib.vcs import connection, client_http
     from rhodecode.lib.middleware.utils import scm_app
 
     session_factory = client_http.ThreadlocalSessionFactory()
 
     connection.Git = client_http.RepoMaker(
         server_and_port, '/git', session_factory)
     connection.Hg = client_http.RepoMaker(
         server_and_port, '/hg', session_factory)
     connection.Svn = client_http.RepoMaker(
         server_and_port, '/svn', session_factory)
+    connection.Service = client_http.ServiceConnection(
+        server_and_port, '/_service', session_factory)
 
     scm_app.HG_REMOTE_WSGI = client_http.VcsHttpProxy(
         server_and_port, '/proxy/hg')
     scm_app.GIT_REMOTE_WSGI = client_http.VcsHttpProxy(
         server_and_port, '/proxy/git')
 
     @atexit.register
     def free_connection_resources():
         connection.Git = None
         connection.Hg = None
         connection.Svn = None
+        connection.Service = None
 
 
 def connect_vcs(server_and_port, protocol):
     """
     Initializes the connection to the vcs server.
 
     :param server_and_port: str, e.g. "localhost:9900"
     :param protocol: str, "pyro4" or "http"
     """
     if protocol == 'pyro4':
         connect_pyro4(server_and_port)
     elif protocol == 'http':
         connect_http(server_and_port)
     else:
         raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
 
 
 # TODO: johbo: This function should be moved into our test suite, there is
 # no reason to support starting the vcsserver in Enterprise itself.
 def start_vcs_server(server_and_port, protocol, log_level=None):
     """
     Starts the vcs server in a subprocess.
     """
     log.info('Starting VCSServer as a sub process with %s protocol', protocol)
     if protocol == 'http':
         return _start_http_vcs_server(server_and_port, log_level)
     elif protocol == 'pyro4':
         return _start_pyro4_vcs_server(server_and_port, log_level)
     else:
         raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
 
 
 def _start_pyro4_vcs_server(server_and_port, log_level=None):
     _try_to_shutdown_running_server(server_and_port, protocol='pyro4')
     host, port = server_and_port.rsplit(":", 1)
     host = host.strip('[]')
     args = [
         'vcsserver', '--port', port, '--host', host, '--locale', 'en_US.UTF-8',
         '--threadpool', '32']
     if log_level:
         args += ['--log-level', log_level]
     proc = subprocess32.Popen(args)
 
     def cleanup_server_process():
         proc.kill()
     atexit.register(cleanup_server_process)
 
     server = create_vcsserver_proxy(server_and_port, protocol='pyro4')
     _wait_until_vcs_server_is_reachable(server)
 
 
 def _start_http_vcs_server(server_and_port, log_level=None):
     # TODO: mikhail: shutdown if an http server already runs
 
     host, port = server_and_port.rsplit(":", 1)
     args = [
         'pserve', 'rhodecode/tests/vcsserver_http.ini',
         'http_port=%s' % (port, ), 'http_host=%s' % (host, )]
     proc = subprocess32.Popen(args)
 
     def cleanup_server_process():
         proc.kill()
     atexit.register(cleanup_server_process)
 
     server = create_vcsserver_proxy(server_and_port, protocol='http')
     _wait_until_vcs_server_is_reachable(server)
 
 
 def _wait_until_vcs_server_is_reachable(server, timeout=40):
     begin = time.time()
     while (time.time() - begin) < timeout:
         try:
             server.ping()
             return
         except (VCSCommunicationError, CommunicationError, pycurl.error):
             log.debug('VCSServer not started yet, retry to connect.')
         time.sleep(0.5)
     raise Exception(
         'Starting the VCSServer failed or took more than {} '
         'seconds.'.format(timeout))
 
 
 def _try_to_shutdown_running_server(server_and_port, protocol):
     server = create_vcsserver_proxy(server_and_port, protocol)
     try:
         server.shutdown()
     except (CommunicationError, pycurl.error):
         return
 
     # TODO: Not sure why this is important, but without it the following start
     # of the server fails.
     server = create_vcsserver_proxy(server_and_port, protocol)
     server.ping()
 
 
 def create_vcsserver_proxy(server_and_port, protocol):
     if protocol == 'pyro4':
         return _create_vcsserver_proxy_pyro4(server_and_port)
     elif protocol == 'http':
         return _create_vcsserver_proxy_http(server_and_port)
     else:
         raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
 
 
 def _create_vcsserver_proxy_pyro4(server_and_port):
     server = Pyro4.Proxy(
         settings.pyro_remote(settings.PYRO_VCSSERVER, server_and_port))
     return server
 
 
 def _create_vcsserver_proxy_http(server_and_port):
     from rhodecode.lib.vcs import client_http
 
     session = _create_http_rpc_session()
     url = urlparse.urljoin('http://%s' % server_and_port, '/server')
     return client_http.RemoteObject(url, session)
 
 
 class CurlSession(object):
     """
     Modeled so that it provides a subset of the requests interface.
 
     This has been created so that it does only provide a minimal API for our
     needs. The parts which it provides are based on the API of the library
     `requests` which allows us to easily benchmark against it.
 
     Please have a look at the class :class:`requests.Session` when you extend
     it.
     """
 
     def __init__(self):
         curl = pycurl.Curl()
         # TODO: johbo: I did test with 7.19 of libcurl. This version has
         # trouble with 100 - continue being set in the expect header. This
         # can lead to massive performance drops, switching it off here.
         curl.setopt(curl.HTTPHEADER, ["Expect:"])
         curl.setopt(curl.TCP_NODELAY, True)
         curl.setopt(curl.PROTOCOLS, curl.PROTO_HTTP)
         self._curl = curl
 
     def post(self, url, data, allow_redirects=False):
         response_buffer = StringIO()
 
         curl = self._curl
         curl.setopt(curl.URL, url)
         curl.setopt(curl.POST, True)
         curl.setopt(curl.POSTFIELDS, data)
         curl.setopt(curl.FOLLOWLOCATION, allow_redirects)
         curl.setopt(curl.WRITEDATA, response_buffer)
         curl.perform()
 
         return CurlResponse(response_buffer)
 
 
 class CurlResponse(object):
     """
     The response of a request, modeled after the requests API.
 
     This class provides a subset of the response interface known from the
     library `requests`. It is intentionally kept similar, so that we can use
     `requests` as a drop in replacement for benchmarking purposes.
     """
 
     def __init__(self, response_buffer):
         self._response_buffer = response_buffer
 
     @property
     def content(self):
         return self._response_buffer.getvalue()
 
 
 def _create_http_rpc_session():
     session = CurlSession()
     return session

rhodecode/lib/vcs/backends/__init__.py

@@ -1,78 +1,87 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 VCS Backends module
 """
 
 import logging
 
 from pprint import pformat
 
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.utils.helpers import get_scm
 from rhodecode.lib.vcs.utils.imports import import_class
 
 
 log = logging.getLogger(__name__)
 
 
 def get_vcs_instance(repo_path, *args, **kwargs):
     """
     Given a path to a repository an instance of the corresponding vcs backend
     repository class is created and returned. If no repository can be found
     for the path it returns None. Arguments and keyword arguments are passed
     to the vcs backend repository class.
     """
     try:
         vcs_alias = get_scm(repo_path)[0]
         log.debug(
             'Creating instance of %s repository from %s', vcs_alias, repo_path)
         backend = get_backend(vcs_alias)
     except VCSError:
         log.exception(
             'Perhaps this repository is in db and not in '
             'filesystem run rescan repositories with '
             '"destroy old data" option from admin panel')
         return None
 
     return backend(repo_path=repo_path, *args, **kwargs)
 
 
 def get_backend(alias):
     """
     Returns ``Repository`` class identified by the given alias or raises
     VCSError if alias is not recognized or backend class cannot be imported.
     """
     if alias not in settings.BACKENDS:
         raise VCSError(
             "Given alias '%s' is not recognized! Allowed aliases:\n%s" %
             (alias, pformat(settings.BACKENDS.keys())))
     backend_path = settings.BACKENDS[alias]
     klass = import_class(backend_path)
     return klass
 
 
 def get_supported_backends():
     """
     Returns list of aliases of supported backends.
     """
     return settings.BACKENDS.keys()
+
+
+def get_vcsserver_version():
+    from rhodecode.lib.vcs import connection
+    data = connection.Service.get_vcsserver_service_data()
+    if data and 'version' in data:
+        return data['version']
+
+    return None

rhodecode/lib/vcs/backends/git/__init__.py

@@ -1,47 +1,49 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 GIT module
 """
 
 import logging
 
 from rhodecode.lib.vcs import connection
 from rhodecode.lib.vcs.backends.git.repository import GitRepository
 from rhodecode.lib.vcs.backends.git.commit import GitCommit
 from rhodecode.lib.vcs.backends.git.inmemory import GitInMemoryCommit
 
 
 log = logging.getLogger(__name__)
 
 
-def discover_git_version():
+def discover_git_version(raise_on_exc=False):
     """
     Returns the string as it was returned by running 'git --version'
 
     It will return an empty string in case the connection is not initialized
     or no vcsserver is available.
     """
     try:
         return connection.Git.discover_git_version()
     except Exception:
         log.warning("Failed to discover the Git version", exc_info=True)
+        if raise_on_exc:
+            raise
         return ''

rhodecode/lib/vcs/backends/hg/__init__.py

@@ -1,27 +1,48 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 HG module
 """
+import logging
 
+from rhodecode.lib.vcs import connection
 from rhodecode.lib.vcs.backends.hg.commit import MercurialCommit
 from rhodecode.lib.vcs.backends.hg.inmemory import MercurialInMemoryCommit
 from rhodecode.lib.vcs.backends.hg.repository import MercurialRepository
+
+
+log = logging.getLogger(__name__)
+
+
+def discover_hg_version(raise_on_exc=False):
+    """
+    Returns the string as it was returned by running 'git --version'
+
+    It will return an empty string in case the connection is not initialized
+    or no vcsserver is available.
+    """
+    try:
+        return connection.Hg.discover_hg_version()
+    except Exception:
+        log.warning("Failed to discover the HG version", exc_info=True)
+        if raise_on_exc:
+            raise
+        return ''

rhodecode/lib/vcs/backends/svn/__init__.py

@@ -1,26 +1,47 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SVN module
 """
+import logging
 
+from rhodecode.lib.vcs import connection
 from rhodecode.lib.vcs.backends.svn.commit import SubversionCommit
 from rhodecode.lib.vcs.backends.svn.repository import SubversionRepository
+
+
+log = logging.getLogger(__name__)
+
+
+def discover_svn_version(raise_on_exc=False):
+    """
+    Returns the string as it was returned by running 'git --version'
+
+    It will return an empty string in case the connection is not initialized
+    or no vcsserver is available.
+    """
+    try:
+        return connection.Svn.discover_svn_version()
+    except Exception:
+        log.warning("Failed to discover the SVN version", exc_info=True)
+        if raise_on_exc:
+            raise
+        return ''

rhodecode/lib/vcs/connection.py

@@ -1,35 +1,37 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Holds connection for remote server.
 """
 
 
 def _not_initialized(*args, **kwargs):
     """Placeholder for objects which have to be initialized first."""
     raise Exception(
         "rhodecode.lib.vcs is not yet initialized. "
         "Make sure `vcs.server` is enabled in your configuration.")
 
 # TODO: figure out a nice default value for these things
+Service = _not_initialized
+
 Git = _not_initialized
 Hg = _not_initialized
 Svn = _not_initialized

rhodecode/model/scm.py

@@ -1,1123 +1,911 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Scm model for RhodeCode
 """
 
 import os.path
 import re
 import sys
-import time
 import traceback
 import logging
 import cStringIO
 import pkg_resources
 
-import pylons
 from pylons.i18n.translation import _
 from sqlalchemy import func
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
 from rhodecode.lib.vcs import get_backend
 from rhodecode.lib.vcs.exceptions import RepositoryError, NodeNotChangedError
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib import helpers as h
 
 from rhodecode.lib.auth import (
     HasRepoPermissionAny, HasRepoGroupPermissionAny,
     HasUserGroupPermissionAny)
 from rhodecode.lib.exceptions import NonRelativePathError, IMCCommitError
 from rhodecode.lib import hooks_utils, caches
 from rhodecode.lib.utils import (
     get_filesystem_repos, action_logger, make_db_config)
-from rhodecode.lib.utils2 import (
-    safe_str, safe_unicode, get_server_url, md5)
+from rhodecode.lib.utils2 import (safe_str, safe_unicode)
+from rhodecode.lib.system_info import get_system_info
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     Repository, CacheKey, UserFollowing, UserLog, User, RepoGroup,
-    PullRequest, DbMigrateVersion)
+    PullRequest)
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 class UserTemp(object):
     def __init__(self, user_id):
         self.user_id = user_id
 
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
 
 
 class RepoTemp(object):
     def __init__(self, repo_id):
         self.repo_id = repo_id
 
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
 
 
 class SimpleCachedRepoList(object):
     """
     Lighter version of of iteration of repos without the scm initialisation,
     and with cache usage
     """
     def __init__(self, db_repo_list, repos_path, order_by=None, perm_set=None):
         self.db_repo_list = db_repo_list
         self.repos_path = repos_path
         self.order_by = order_by
         self.reversed = (order_by or '').startswith('-')
         if not perm_set:
             perm_set = ['repository.read', 'repository.write',
                         'repository.admin']
         self.perm_set = perm_set
 
     def __len__(self):
         return len(self.db_repo_list)
 
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
 
     def __iter__(self):
         for dbr in self.db_repo_list:
             # check permission at this level
             has_perm = HasRepoPermissionAny(*self.perm_set)(
                 dbr.repo_name, 'SimpleCachedRepoList check')
             if not has_perm:
                 continue
 
             tmp_d = {
                 'name': dbr.repo_name,
                 'dbrepo': dbr.get_dict(),
                 'dbrepo_fork': dbr.fork.get_dict() if dbr.fork else {}
             }
             yield tmp_d
 
 
 class _PermCheckIterator(object):
 
     def __init__(
             self, obj_list, obj_attr, perm_set, perm_checker,
             extra_kwargs=None):
         """
         Creates iterator from given list of objects, additionally
         checking permission for them from perm_set var
 
         :param obj_list: list of db objects
         :param obj_attr: attribute of object to pass into perm_checker
         :param perm_set: list of permissions to check
         :param perm_checker: callable to check permissions against
         """
         self.obj_list = obj_list
         self.obj_attr = obj_attr
         self.perm_set = perm_set
         self.perm_checker = perm_checker
         self.extra_kwargs = extra_kwargs or {}
 
     def __len__(self):
         return len(self.obj_list)
 
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
 
     def __iter__(self):
         checker = self.perm_checker(*self.perm_set)
         for db_obj in self.obj_list:
             # check permission at this level
             name = getattr(db_obj, self.obj_attr, None)
             if not checker(name, self.__class__.__name__, **self.extra_kwargs):
                 continue
 
             yield db_obj
 
 
 class RepoList(_PermCheckIterator):
 
     def __init__(self, db_repo_list, perm_set=None, extra_kwargs=None):
         if not perm_set:
             perm_set = [
                 'repository.read', 'repository.write', 'repository.admin']
 
         super(RepoList, self).__init__(
             obj_list=db_repo_list,
             obj_attr='repo_name', perm_set=perm_set,
             perm_checker=HasRepoPermissionAny,
             extra_kwargs=extra_kwargs)
 
 
 class RepoGroupList(_PermCheckIterator):
 
     def __init__(self, db_repo_group_list, perm_set=None, extra_kwargs=None):
         if not perm_set:
             perm_set = ['group.read', 'group.write', 'group.admin']
 
         super(RepoGroupList, self).__init__(
             obj_list=db_repo_group_list,
             obj_attr='group_name', perm_set=perm_set,
             perm_checker=HasRepoGroupPermissionAny,
             extra_kwargs=extra_kwargs)
 
 
 class UserGroupList(_PermCheckIterator):
 
     def __init__(self, db_user_group_list, perm_set=None, extra_kwargs=None):
         if not perm_set:
             perm_set = ['usergroup.read', 'usergroup.write', 'usergroup.admin']
 
         super(UserGroupList, self).__init__(
             obj_list=db_user_group_list,
             obj_attr='users_group_name', perm_set=perm_set,
             perm_checker=HasUserGroupPermissionAny,
             extra_kwargs=extra_kwargs)
 
 
 class ScmModel(BaseModel):
     """
     Generic Scm Model
     """
 
     @LazyProperty
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
 
         settings_model = VcsSettingsModel(sa=self.sa)
         return settings_model.get_repos_location()
 
     def repo_scan(self, repos_path=None):
         """
         Listing of repositories in given path. This path should not be a
         repository itself. Return a dictionary of repository objects
 
         :param repos_path: path to directory containing repositories
         """
 
         if repos_path is None:
             repos_path = self.repos_path
 
         log.info('scanning for repositories in %s', repos_path)
 
         config = make_db_config()
         config.set('extensions', 'largefiles', '')
         repos = {}
 
         for name, path in get_filesystem_repos(repos_path, recursive=True):
             # name need to be decomposed and put back together using the /
             # since this is internal storage separator for rhodecode
             name = Repository.normalize_repo_name(name)
 
             try:
                 if name in repos:
                     raise RepositoryError('Duplicate repository name %s '
                                           'found in %s' % (name, path))
                 elif path[0] in rhodecode.BACKENDS:
                     klass = get_backend(path[0])
                     repos[name] = klass(path[1], config=config)
             except OSError:
                 continue
         log.debug('found %s paths with repositories', len(repos))
         return repos
 
     def get_repos(self, all_repos=None, sort_key=None):
         """
         Get all repositories from db and for each repo create it's
         backend instance and fill that backed with information from database
 
         :param all_repos: list of repository names as strings
             give specific repositories list, good for filtering
 
         :param sort_key: initial sorting of repositories
         """
         if all_repos is None:
             all_repos = self.sa.query(Repository)\
                 .filter(Repository.group_id == None)\
                 .order_by(func.lower(Repository.repo_name)).all()
         repo_iter = SimpleCachedRepoList(
             all_repos, repos_path=self.repos_path, order_by=sort_key)
         return repo_iter
 
     def get_repo_groups(self, all_groups=None):
         if all_groups is None:
             all_groups = RepoGroup.query()\
                 .filter(RepoGroup.group_parent_id == None).all()
         return [x for x in RepoGroupList(all_groups)]
 
     def mark_for_invalidation(self, repo_name, delete=False):
         """
         Mark caches of this repo invalid in the database. `delete` flag
         removes the cache entries
 
         :param repo_name: the repo_name for which caches should be marked
             invalid, or deleted
         :param delete: delete the entry keys instead of setting bool
             flag on them
         """
         CacheKey.set_invalidate(repo_name, delete=delete)
         repo = Repository.get_by_repo_name(repo_name)
 
         if repo:
             config = repo._config
             config.set('extensions', 'largefiles', '')
             repo.update_commit_cache(config=config, cs_cache=None)
             caches.clear_repo_caches(repo_name)
 
     def toggle_following_repo(self, follow_repo_id, user_id):
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repo_id == follow_repo_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 self.sa.delete(f)
                 action_logger(UserTemp(user_id),
                               'stopped_following_repo',
                               RepoTemp(follow_repo_id))
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_repo_id = follow_repo_id
             self.sa.add(f)
 
             action_logger(UserTemp(user_id),
                           'started_following_repo',
                           RepoTemp(follow_repo_id))
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def toggle_following_user(self, follow_user_id, user_id):
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user_id == follow_user_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 self.sa.delete(f)
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_user_id = follow_user_id
             self.sa.add(f)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def is_following_repo(self, repo_name, user_id, cache=False):
         r = self.sa.query(Repository)\
             .filter(Repository.repo_name == repo_name).scalar()
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repository == r)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def is_following_user(self, username, user_id, cache=False):
         u = User.get_by_username(username)
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user == u)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def get_followers(self, repo):
         repo = self._get_repo(repo)
 
         return self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repository == repo).count()
 
     def get_forks(self, repo):
         repo = self._get_repo(repo)
         return self.sa.query(Repository)\
             .filter(Repository.fork == repo).count()
 
     def get_pull_requests(self, repo):
         repo = self._get_repo(repo)
         return self.sa.query(PullRequest)\
             .filter(PullRequest.target_repo == repo)\
             .filter(PullRequest.status != PullRequest.STATUS_CLOSED).count()
 
     def mark_as_fork(self, repo, fork, user):
         repo = self._get_repo(repo)
         fork = self._get_repo(fork)
         if fork and repo.repo_id == fork.repo_id:
             raise Exception("Cannot set repository as fork of itself")
 
         if fork and repo.repo_type != fork.repo_type:
             raise RepositoryError(
                 "Cannot set repository as fork of repository with other type")
 
         repo.fork = fork
         self.sa.add(repo)
         return repo
 
     def pull_changes(self, repo, username):
         dbrepo = self._get_repo(repo)
         clone_uri = dbrepo.clone_uri
         if not clone_uri:
             raise Exception("This repository doesn't have a clone uri")
 
         repo = dbrepo.scm_instance(cache=False)
         # TODO: marcink fix this an re-enable since we need common logic
         # for hg/git remove hooks so we don't trigger them on fetching
         # commits from remote
         repo.config.clear_section('hooks')
 
         repo_name = dbrepo.repo_name
         try:
             # TODO: we need to make sure those operations call proper hooks !
             repo.pull(clone_uri)
 
             self.mark_for_invalidation(repo_name)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def commit_change(self, repo, repo_name, commit, user, author, message,
                       content, f_path):
         """
         Commits changes
 
         :param repo: SCM instance
 
         """
         user = self._get_user(user)
 
         # decoding here will force that we have proper encoded values
         # in any other case this will throw exceptions and deny commit
         content = safe_str(content)
         path = safe_str(f_path)
         # message and author needs to be unicode
         # proper backend should then translate that into required type
         message = safe_unicode(message)
         author = safe_unicode(author)
         imc = repo.in_memory_commit
         imc.change(FileNode(path, content, mode=commit.get_file_mode(f_path)))
         try:
             # TODO: handle pre-push action !
             tip = imc.commit(
                 message=message, author=author, parents=[commit],
                 branch=commit.branch)
         except Exception as e:
             log.error(traceback.format_exc())
             raise IMCCommitError(str(e))
         finally:
             # always clear caches, if commit fails we want fresh object also
             self.mark_for_invalidation(repo_name)
 
         # We trigger the post-push action
         hooks_utils.trigger_post_push_hook(
             username=user.username, action='push_local', repo_name=repo_name,
             repo_alias=repo.alias, commit_ids=[tip.raw_id])
         return tip
 
     def _sanitize_path(self, f_path):
         if f_path.startswith('/') or f_path.startswith('./') or '../' in f_path:
             raise NonRelativePathError('%s is not an relative path' % f_path)
         if f_path:
             f_path = os.path.normpath(f_path)
         return f_path
 
     def get_dirnode_metadata(self, commit, dir_node):
         if not dir_node.is_dir():
             return []
 
         data = []
         for node in dir_node:
             if not node.is_file():
                 # we skip file-nodes
                 continue
 
             last_commit = node.last_commit
             last_commit_date = last_commit.date
             data.append({
                 'name': node.name,
                 'size': h.format_byte_size_binary(node.size),
                 'modified_at': h.format_date(last_commit_date),
                 'modified_ts': last_commit_date.isoformat(),
                 'revision': last_commit.revision,
                 'short_id': last_commit.short_id,
                 'message': h.escape(last_commit.message),
                 'author': h.escape(last_commit.author),
                 'user_profile': h.gravatar_with_user(last_commit.author),
             })
 
         return data
 
     def get_nodes(self, repo_name, commit_id, root_path='/', flat=True,
                   extended_info=False, content=False, max_file_bytes=None):
         """
         recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
 
         :param repo_name: name of repository
         :param commit_id: commit id for which to list nodes
         :param root_path: root path to list
         :param flat: return as a list, if False returns a dict with description
         :param max_file_bytes: will not return file contents over this limit
 
         """
         _files = list()
         _dirs = list()
         try:
             _repo = self._get_repo(repo_name)
             commit = _repo.scm_instance().get_commit(commit_id=commit_id)
             root_path = root_path.lstrip('/')
             for __, dirs, files in commit.walk(root_path):
                 for f in files:
                     _content = None
                     _data = f.unicode_path
                     over_size_limit = (max_file_bytes is not None
                                        and f.size > max_file_bytes)
 
                     if not flat:
                         _data = {
                             "name": f.unicode_path,
                             "type": "file",
                             }
                         if extended_info:
                             _data.update({
                                 "md5": f.md5,
                                 "binary": f.is_binary,
                                 "size": f.size,
                                 "extension": f.extension,
                                 "mimetype": f.mimetype,
                                 "lines": f.lines()[0]
                             })
 
                         if content:
                             full_content = None
                             if not f.is_binary and not over_size_limit:
                                 full_content = safe_str(f.content)
 
                             _data.update({
                                 "content": full_content,
                             })
                     _files.append(_data)
                 for d in dirs:
                     _data = d.unicode_path
                     if not flat:
                         _data = {
                             "name": d.unicode_path,
                             "type": "dir",
                             }
                     if extended_info:
                         _data.update({
                             "md5": None,
                             "binary": None,
                             "size": None,
                             "extension": None,
                         })
                     if content:
                         _data.update({
                             "content": None
                         })
                     _dirs.append(_data)
         except RepositoryError:
             log.debug("Exception in get_nodes", exc_info=True)
             raise
 
         return _dirs, _files
 
     def create_nodes(self, user, repo, message, nodes, parent_commit=None,
                      author=None, trigger_push_hook=True):
         """
         Commits given multiple nodes into repo
 
         :param user: RhodeCode User object or user_id, the commiter
         :param repo: RhodeCode Repository object
         :param message: commit message
         :param nodes: mapping {filename:{'content':content},...}
         :param parent_commit: parent commit, can be empty than it's
            initial commit
         :param author: author of commit, cna be different that commiter
            only for git
         :param trigger_push_hook: trigger push hooks
 
         :returns: new commited commit
         """
 
         user = self._get_user(user)
         scm_instance = repo.scm_instance(cache=False)
 
         processed_nodes = []
         for f_path in nodes:
             f_path = self._sanitize_path(f_path)
             content = nodes[f_path]['content']
             f_path = safe_str(f_path)
             # decoding here will force that we have proper encoded values
             # in any other case this will throw exceptions and deny commit
             if isinstance(content, (basestring,)):
                 content = safe_str(content)
             elif isinstance(content, (file, cStringIO.OutputType,)):
                 content = content.read()
             else:
                 raise Exception('Content is of unrecognized type %s' % (
                     type(content)
                 ))
             processed_nodes.append((f_path, content))
 
         message = safe_unicode(message)
         commiter = user.full_contact
         author = safe_unicode(author) if author else commiter
 
         imc = scm_instance.in_memory_commit
 
         if not parent_commit:
             parent_commit = EmptyCommit(alias=scm_instance.alias)
 
         if isinstance(parent_commit, EmptyCommit):
             # EmptyCommit means we we're editing empty repository
             parents = None
         else:
             parents = [parent_commit]
         # add multiple nodes
         for path, content in processed_nodes:
             imc.add(FileNode(path, content=content))
         # TODO: handle pre push scenario
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_commit.branch)
 
         self.mark_for_invalidation(repo.repo_name)
         if trigger_push_hook:
             hooks_utils.trigger_post_push_hook(
                 username=user.username, action='push_local',
                 repo_name=repo.repo_name, repo_alias=scm_instance.alias,
                 commit_ids=[tip.raw_id])
         return tip
 
     def update_nodes(self, user, repo, message, nodes, parent_commit=None,
                      author=None, trigger_push_hook=True):
         user = self._get_user(user)
         scm_instance = repo.scm_instance(cache=False)
 
         message = safe_unicode(message)
         commiter = user.full_contact
         author = safe_unicode(author) if author else commiter
 
         imc = scm_instance.in_memory_commit
 
         if not parent_commit:
             parent_commit = EmptyCommit(alias=scm_instance.alias)
 
         if isinstance(parent_commit, EmptyCommit):
             # EmptyCommit means we we're editing empty repository
             parents = None
         else:
             parents = [parent_commit]
 
         # add multiple nodes
         for _filename, data in nodes.items():
             # new filename, can be renamed from the old one, also sanitaze
             # the path for any hack around relative paths like ../../ etc.
             filename = self._sanitize_path(data['filename'])
             old_filename = self._sanitize_path(_filename)
             content = data['content']
 
             filenode = FileNode(old_filename, content=content)
             op = data['op']
             if op == 'add':
                 imc.add(filenode)
             elif op == 'del':
                 imc.remove(filenode)
             elif op == 'mod':
                 if filename != old_filename:
                     # TODO: handle renames more efficient, needs vcs lib
                     # changes
                     imc.remove(filenode)
                     imc.add(FileNode(filename, content=content))
                 else:
                     imc.change(filenode)
 
         try:
             # TODO: handle pre push scenario
             # commit changes
             tip = imc.commit(message=message,
                              author=author,
                              parents=parents,
                              branch=parent_commit.branch)
         except NodeNotChangedError:
             raise
         except Exception as e:
             log.exception("Unexpected exception during call to imc.commit")
             raise IMCCommitError(str(e))
         finally:
             # always clear caches, if commit fails we want fresh object also
             self.mark_for_invalidation(repo.repo_name)
 
         if trigger_push_hook:
             hooks_utils.trigger_post_push_hook(
                 username=user.username, action='push_local',
                 repo_name=repo.repo_name, repo_alias=scm_instance.alias,
                 commit_ids=[tip.raw_id])
 
     def delete_nodes(self, user, repo, message, nodes, parent_commit=None,
                      author=None, trigger_push_hook=True):
         """
         Deletes given multiple nodes into `repo`
 
         :param user: RhodeCode User object or user_id, the committer
         :param repo: RhodeCode Repository object
         :param message: commit message
         :param nodes: mapping {filename:{'content':content},...}
         :param parent_commit: parent commit, can be empty than it's initial
            commit
         :param author: author of commit, cna be different that commiter only
            for git
         :param trigger_push_hook: trigger push hooks
 
         :returns: new commit after deletion
         """
 
         user = self._get_user(user)
         scm_instance = repo.scm_instance(cache=False)
 
         processed_nodes = []
         for f_path in nodes:
             f_path = self._sanitize_path(f_path)
             # content can be empty but for compatabilty it allows same dicts
             # structure as add_nodes
             content = nodes[f_path].get('content')
             processed_nodes.append((f_path, content))
 
         message = safe_unicode(message)
         commiter = user.full_contact
         author = safe_unicode(author) if author else commiter
 
         imc = scm_instance.in_memory_commit
 
         if not parent_commit:
             parent_commit = EmptyCommit(alias=scm_instance.alias)
 
         if isinstance(parent_commit, EmptyCommit):
             # EmptyCommit means we we're editing empty repository
             parents = None
         else:
             parents = [parent_commit]
         # add multiple nodes
         for path, content in processed_nodes:
             imc.remove(FileNode(path, content=content))
 
         # TODO: handle pre push scenario
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_commit.branch)
 
         self.mark_for_invalidation(repo.repo_name)
         if trigger_push_hook:
             hooks_utils.trigger_post_push_hook(
                 username=user.username, action='push_local',
                 repo_name=repo.repo_name, repo_alias=scm_instance.alias,
                 commit_ids=[tip.raw_id])
         return tip
 
     def strip(self, repo, commit_id, branch):
         scm_instance = repo.scm_instance(cache=False)
         scm_instance.config.clear_section('hooks')
         scm_instance.strip(commit_id, branch)
         self.mark_for_invalidation(repo.repo_name)
 
     def get_unread_journal(self):
         return self.sa.query(UserLog).count()
 
     def get_repo_landing_revs(self, repo=None):
         """
         Generates select option with tags branches and bookmarks (for hg only)
         grouped by type
 
         :param repo:
         """
 
         hist_l = []
         choices = []
         repo = self._get_repo(repo)
         hist_l.append(['rev:tip', _('latest tip')])
         choices.append('rev:tip')
         if not repo:
             return choices, hist_l
 
         repo = repo.scm_instance()
 
         branches_group = (
             [(u'branch:%s' % safe_unicode(b), safe_unicode(b))
                 for b in repo.branches],
             _("Branches"))
         hist_l.append(branches_group)
         choices.extend([x[0] for x in branches_group[0]])
 
         if repo.alias == 'hg':
             bookmarks_group = (
                 [(u'book:%s' % safe_unicode(b), safe_unicode(b))
                     for b in repo.bookmarks],
                 _("Bookmarks"))
             hist_l.append(bookmarks_group)
             choices.extend([x[0] for x in bookmarks_group[0]])
 
         tags_group = (
             [(u'tag:%s' % safe_unicode(t), safe_unicode(t))
                 for t in repo.tags],
             _("Tags"))
         hist_l.append(tags_group)
         choices.extend([x[0] for x in tags_group[0]])
 
         return choices, hist_l
 
     def install_git_hook(self, repo, force_create=False):
         """
         Creates a rhodecode hook inside a git repository
 
         :param repo: Instance of VCS repo
         :param force_create: Create even if same name hook exists
         """
 
         loc = os.path.join(repo.path, 'hooks')
         if not repo.bare:
             loc = os.path.join(repo.path, '.git', 'hooks')
         if not os.path.isdir(loc):
             os.makedirs(loc, mode=0777)
 
         tmpl_post = pkg_resources.resource_string(
             'rhodecode', '/'.join(
                 ('config', 'hook_templates', 'git_post_receive.py.tmpl')))
         tmpl_pre = pkg_resources.resource_string(
             'rhodecode', '/'.join(
                 ('config', 'hook_templates', 'git_pre_receive.py.tmpl')))
 
         for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:
             _hook_file = os.path.join(loc, '%s-receive' % h_type)
             log.debug('Installing git hook in repo %s', repo)
             _rhodecode_hook = _check_rhodecode_hook(_hook_file)
 
             if _rhodecode_hook or force_create:
                 log.debug('writing %s hook file !', h_type)
                 try:
                     with open(_hook_file, 'wb') as f:
                         tmpl = tmpl.replace('_TMPL_', rhodecode.__version__)
                         tmpl = tmpl.replace('_ENV_', sys.executable)
                         f.write(tmpl)
                     os.chmod(_hook_file, 0755)
                 except IOError:
                     log.exception('error writing hook file %s', _hook_file)
             else:
                 log.debug('skipping writing hook file')
 
     def install_svn_hooks(self, repo, force_create=False):
         """
         Creates rhodecode hooks inside a svn repository
 
         :param repo: Instance of VCS repo
         :param force_create: Create even if same name hook exists
         """
         hooks_path = os.path.join(repo.path, 'hooks')
         if not os.path.isdir(hooks_path):
             os.makedirs(hooks_path)
         post_commit_tmpl = pkg_resources.resource_string(
             'rhodecode', '/'.join(
                 ('config', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
         pre_commit_template = pkg_resources.resource_string(
             'rhodecode', '/'.join(
                 ('config', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
         templates = {
             'post-commit': post_commit_tmpl,
             'pre-commit': pre_commit_template
         }
         for filename in templates:
             _hook_file = os.path.join(hooks_path, filename)
             _rhodecode_hook = _check_rhodecode_hook(_hook_file)
             if _rhodecode_hook or force_create:
                 log.debug('writing %s hook file !', filename)
                 template = templates[filename]
                 try:
                     with open(_hook_file, 'wb') as f:
                         template = template.replace(
                             '_TMPL_', rhodecode.__version__)
                         template = template.replace('_ENV_', sys.executable)
                         f.write(template)
                     os.chmod(_hook_file, 0755)
                 except IOError:
                     log.exception('error writing hook file %s', filename)
             else:
                 log.debug('skipping writing hook file')
 
     def install_hooks(self, repo, repo_type):
         if repo_type == 'git':
             self.install_git_hook(repo)
         elif repo_type == 'svn':
             self.install_svn_hooks(repo)
 
     def get_server_info(self, environ=None):
-        import platform
-        import rhodecode
-        import pkg_resources
-        from rhodecode.model.meta import Base as sql_base, Session
-        from sqlalchemy.engine import url
-        from rhodecode.lib.base import get_server_ip_addr, get_server_port
-        from rhodecode.lib.vcs.backends.git import discover_git_version
-        from rhodecode.model.gist import GIST_STORE_LOC
-
-        def percentage(part, whole):
-            whole = float(whole)
-            if whole > 0:
-                return 100 * float(part) / whole
-            return 0
-
-        try:
-            # cygwin cannot have yet psutil support.
-            import psutil
-        except ImportError:
-            psutil = None
-
-        environ = environ or {}
-        _NA = 'NOT AVAILABLE'
-        _memory = _NA
-        _uptime = _NA
-        _boot_time = _NA
-        _cpu = _NA
-        _disk = dict(percent=0, used=0, total=0, error='')
-        _disk_inodes = dict(percent=0, free=0, used=0, total=0, error='')
-        _load = {'1_min': _NA, '5_min': _NA, '15_min': _NA}
-
-        model = VcsSettingsModel()
-        storage_path = model.get_repos_location()
-        gist_storage_path = os.path.join(storage_path, GIST_STORE_LOC)
-        archive_storage_path = rhodecode.CONFIG.get('archive_cache_dir', '')
-        search_index_storage_path = rhodecode.CONFIG.get('search.location', '')
-
-        if psutil:
-            # disk storage
-            try:
-                _disk = dict(psutil.disk_usage(storage_path)._asdict())
-            except Exception as e:
-                log.exception('Failed to fetch disk info')
-                _disk = {'percent': 0, 'used': 0, 'total': 0, 'error': str(e)}
-
-            # disk inodes usage
-            try:
-                i_stat = os.statvfs(storage_path)
-
-                _disk_inodes['used'] = i_stat.f_ffree
-                _disk_inodes['free'] = i_stat.f_favail
-                _disk_inodes['total'] = i_stat.f_files
-                _disk_inodes['percent'] = percentage(
-                    _disk_inodes['used'], _disk_inodes['total'])
-            except Exception as e:
-                log.exception('Failed to fetch disk inodes info')
-                _disk_inodes['error'] = str(e)
-
-            # memory
-            _memory = dict(psutil.virtual_memory()._asdict())
-            _memory['percent2'] = psutil._common.usage_percent(
-                (_memory['total'] - _memory['free']),
-                _memory['total'], 1)
-
-            # load averages
-            if hasattr(psutil.os, 'getloadavg'):
-                _load = dict(zip(
-                    ['1_min', '5_min', '15_min'], psutil.os.getloadavg()))
-            _uptime = time.time() - psutil.boot_time()
-            _boot_time = psutil.boot_time()
-            _cpu = psutil.cpu_percent(0.5)
-
-        mods = dict([(p.project_name, p.version)
-                     for p in pkg_resources.working_set])
-
-        def get_storage_size(storage_path):
-            sizes = []
-            for file_ in os.listdir(storage_path):
-                storage_file = os.path.join(storage_path, file_)
-                if os.path.isfile(storage_file):
-                    try:
-                        sizes.append(os.path.getsize(storage_file))
-                    except OSError:
-                        log.exception('Failed to get size of storage file %s',
-                                      storage_file)
-                        pass
-
-            return sum(sizes)
-
-        # archive cache storage
-        _disk_archive = {'percent': 0, 'used': 0, 'total': 0}
-        try:
-            archive_storage_path_exists = os.path.isdir(
-                archive_storage_path)
-            if archive_storage_path and archive_storage_path_exists:
-                used = get_storage_size(archive_storage_path)
-                _disk_archive.update({
-                    'used': used,
-                    'total': used,
-                })
-        except Exception as e:
-            log.exception('failed to fetch archive cache storage')
-            _disk_archive['error'] = str(e)
-
-        # search index storage
-        _disk_index = {'percent': 0, 'used': 0, 'total': 0}
-        try:
-            search_index_storage_path_exists = os.path.isdir(
-                search_index_storage_path)
-            if search_index_storage_path_exists:
-                used = get_storage_size(search_index_storage_path)
-                _disk_index.update({
-                    'percent': 100,
-                    'used': used,
-                    'total': used,
-                })
-        except Exception as e:
-            log.exception('failed to fetch search index storage')
-            _disk_index['error'] = str(e)
-
-        # gist storage
-        _disk_gist = {'percent': 0, 'used': 0, 'total': 0, 'items': 0}
-        try:
-            items_count = 0
-            used = 0
-            for root, dirs, files in os.walk(safe_str(gist_storage_path)):
-                if root == gist_storage_path:
-                    items_count = len(dirs)
-
-                for f in files:
-                    try:
-                        used += os.path.getsize(os.path.join(root, f))
-                    except OSError:
-                        pass
-            _disk_gist.update({
-                'percent': 100,
-                'used': used,
-                'total': used,
-                'items': items_count
-            })
-        except Exception as e:
-            log.exception('failed to fetch gist storage items')
-            _disk_gist['error'] = str(e)
-
-        # GIT info
-        git_ver = discover_git_version()
-
-        # SVN info
-        # TODO: johbo: Add discover_svn_version to replace this code.
-        try:
-            import svn.core
-            svn_ver = svn.core.SVN_VERSION
-        except ImportError:
-            svn_ver = None
-
-        # DB stuff
-        db_info = url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
-        db_type = db_info.__to_string__()
-        try:
-            engine = sql_base.metadata.bind
-            db_server_info = engine.dialect._get_server_version_info(
-                Session.connection(bind=engine))
-            db_version = '%s %s' % (db_info.drivername,
-                                    '.'.join(map(str, db_server_info)))
-        except Exception:
-            log.exception('failed to fetch db version')
-            db_version = '%s %s' % (db_info.drivername, '?')
-
-        db_migrate = DbMigrateVersion.query().filter(
-            DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
-        db_migrate_version = db_migrate.version
-
-        info = {
-            'py_version': ' '.join(platform._sys_version()),
-            'py_path': sys.executable,
-            'py_modules': sorted(mods.items(), key=lambda k: k[0].lower()),
-
-            'platform': safe_unicode(platform.platform()),
-            'storage': storage_path,
-            'archive_storage': archive_storage_path,
-            'index_storage': search_index_storage_path,
-            'gist_storage': gist_storage_path,
-
-
-            'db_type': db_type,
-            'db_version': db_version,
-            'db_migrate_version': db_migrate_version,
-
-            'rhodecode_version': rhodecode.__version__,
-            'rhodecode_config_ini': rhodecode.CONFIG.get('__file__'),
-            'server_ip': '%s:%s' % (
-                get_server_ip_addr(environ, log_errors=False),
-                get_server_port(environ)
-            ),
-            'server_id': rhodecode.CONFIG.get('instance_id'),
-
-            'git_version': safe_unicode(git_ver),
-            'hg_version': mods.get('mercurial'),
-            'svn_version': svn_ver,
-
-            'uptime': _uptime,
-            'boot_time': _boot_time,
-            'load': _load,
-            'cpu': _cpu,
-            'memory': _memory,
-            'disk': _disk,
-            'disk_inodes': _disk_inodes,
-            'disk_archive': _disk_archive,
-            'disk_gist': _disk_gist,
-            'disk_index': _disk_index,
-        }
-        return info
+        server_info = get_system_info(environ)
+        return server_info
 
 
 def _check_rhodecode_hook(hook_path):
     """
     Check if the hook was created by RhodeCode
     """
     if not os.path.exists(hook_path):
         return True
 
     log.debug('hook exists, checking if it is from rhodecode')
     hook_content = _read_hook(hook_path)
     matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content)
     if matches:
         try:
             version = matches.groups()[0]
             log.debug('got %s, it is rhodecode', version)
             return True
         except Exception:
             log.exception("Exception while reading the hook version.")
 
     return False
 
 
 def _read_hook(hook_path):
     with open(hook_path, 'rb') as f:
         content = f.read()
     return content

rhodecode/templates/admin/settings/settings_system.html

@@ -1,90 +1,57 b''
-<%
- elems = [
-    ## general
-    (_('RhodeCode Enterprise version'), h.literal('%s <div class="link" id="check_for_update" >%s</div>' % (c.rhodecode_version, _('check for updates'))), ''),
-    (_('Upgrade info endpoint'), h.literal('%s <br/><span >%s.</span>' % (c.rhodecode_update_url, _('Note: please make sure this server can access this url'))), ''),
-    (_('Configuration INI file'), c.rhodecode_config_ini, ''),
-    ## systems stats
-    (_('RhodeCode Enterprise Server IP'), c.server_ip, ''),
-    (_('RhodeCode Enterprise Server ID'), c.server_id, ''),
-    (_('Platform'), c.platform, ''),
-    (_('Uptime'), c.uptime_age, ''),
-    (_('Storage location'), c.storage, ''),
-    (_('Storage disk space'), "%s/%s, %s%% used%s" % (h.format_byte_size_binary(c.disk['used']), h.format_byte_size_binary(c.disk['total']),(c.disk['percent']), ' %s' % c.disk['error'] if 'error' in c.disk else ''), ''),
-    (_('Storage file limit (inodes)'), "%s/%s, %.1f%% used%s" % (c.disk_inodes['used'], c.disk_inodes['total'],(c.disk_inodes['percent']), ' %s' % c.disk_inodes['error'] if 'error' in c.disk_inodes else ''), ''),
-
-    (_('Search index storage'), c.index_storage, ''),
-    (_('Search index size'), "%s %s" % (h.format_byte_size_binary(c.disk_index['used']), ' %s' % c.disk_index['error'] if 'error' in c.disk_index else ''), ''),
-
-    (_('Gist storage'), c.gist_storage, ''),
-    (_('Gist storage size'), "%s (%s items)%s" % (h.format_byte_size_binary(c.disk_gist['used']),c.disk_gist['items'], ' %s' % c.disk_gist['error'] if 'error' in c.disk_gist else ''), ''),
-
-    (_('Archive cache'), h.literal('%s <br/><span >%s.</span>' % (c.archive_storage, _('Enable this by setting archive_cache_dir=/path/to/cache option in the .ini file'))), ''),
-    (_('Archive cache size'), "%s%s" % (h.format_byte_size_binary(c.disk_archive['used']), ' %s' % c.disk_archive['error'] if 'error' in c.disk_archive else ''), ''),
-
-    (_('System memory'), c.system_memory, ''),
-    (_('CPU'), '%s %%' %(c.cpu), ''),
-    (_('Load'), '1min: %s, 5min: %s, 15min: %s' %(c.load['1_min'],c.load['5_min'],c.load['15_min']), ''),
-
-    ## rhodecode stuff
-    (_('Python version'), c.py_version, ''),
-    (_('Python path'), c.py_path, ''),
-    (_('GIT version'), c.git_version, ''),
-    (_('HG version'), c.hg_version, ''),
-    (_('SVN version'), c.svn_version, ''),
-    (_('Database'), "%s @ version: %s" % (c.db_type, c.db_migrate_version), ''),
-    (_('Database version'), c.db_version, ''),
-
- ]
-%>
 
 <div id="update_notice" style="display: none; margin: -40px 0px 20px 0px">
     <div>${_('Checking for updates...')}</div>
 </div>
 
 
 <div class="panel panel-default">
     <div class="panel-heading">
         <h3 class="panel-title">${_('System Info')}</h3>
         % if c.allowed_to_snapshot:
-            <a href="${url('admin_settings_system', snapshot=1)}" class="panel-edit">${_('create snapshot')}</a>
+            <a href="${url('admin_settings_system', snapshot=1)}" class="panel-edit">${_('create summary snapshot')}</a>
         % endif
     </div>
     <div class="panel-body">
         <dl class="dl-horizontal settings">
-        %for dt, dd, tt in elems:
-          <dt>${dt}:</dt>
-          <dd title="${tt}">${dd}</dd>
-        %endfor
+        % for dt, dd, warn in c.data_items:
+          <dt>${dt}${':' if dt else '---'}</dt>
+          <dd>${dd}${'' if dt else '---'}
+              % if warn and warn['message']:
+                  <div class="alert-${warn['type']}">
+                    <strong>${warn['message']}</strong>
+                  </div>
+              % endif
+          </dd>
+        % endfor
         </dl>
     </div>
 </div>
 
 <div class="panel panel-default">
     <div class="panel-heading">
         <h3 class="panel-title">${_('Python Packages')}</h3>
     </div>
     <div class="panel-body">
         <table class="table">
           <colgroup>
               <col class='label'>
               <col class='content'>
           </colgroup>
           <tbody>
-              %for key, value in c.py_modules:
+              % for key, value in c.py_modules['human_value']:
                   <tr>
                       <td>${key}</td>
                       <td>${value}</td>
                   </tr>
-              %endfor
+              % endfor
           </tbody>
         </table>
     </div>
 </div>
 
 <script>
     $('#check_for_update').click(function(e){
         $('#update_notice').show();
         $('#update_notice').load("${h.url('admin_settings_system_update',version=c.rhodecode_version, platform=c.platform)}");
     })
 </script>

rhodecode/templates/admin/settings/settings_system_snapshot.html

@@ -1,75 +1,41 b''
-<%
- elems = [
-    ## general
-    (_('RhodeCode Enterprise version'), c.rhodecode_version, ''),
-    (_('Upgrade info endpoint'), c.rhodecode_update_url, ''),
-    (_('Configuration INI file'), c.rhodecode_config_ini, ''),
-    ## systems stats
-    (_('RhodeCode Enterprise Server IP'), c.server_ip, ''),
-    (_('RhodeCode Enterprise Server ID'), c.server_id, ''),
-    (_('Platform'), c.platform, ''),
-    (_('Uptime'), c.uptime_age, ''),
-    (_('Storage location'), c.storage, ''),
-    (_('Storage disk space'), "%s/%s, %s%% used%s" % (h.format_byte_size_binary(c.disk['used']), h.format_byte_size_binary(c.disk['total']),(c.disk['percent']), ' %s' % c.disk['error'] if 'error' in c.disk else ''), ''),
-
-    (_('Search index storage'), c.index_storage, ''),
-    (_('Search index size'), "%s %s" % (h.format_byte_size_binary(c.disk_index['used']), ' %s' % c.disk_index['error'] if 'error' in c.disk_index else ''), ''),
-
-    (_('Gist storage'), c.gist_storage, ''),
-    (_('Gist storage size'), "%s (%s items)%s" % (h.format_byte_size_binary(c.disk_gist['used']),c.disk_gist['items'], ' %s' % c.disk_gist['error'] if 'error' in c.disk_gist else ''), ''),
-
-    (_('Archive cache'), c.archive_storage, ''),
-    (_('Archive cache size'), "%s%s" % (h.format_byte_size_binary(c.disk_archive['used']), ' %s' % c.disk_archive['error'] if 'error' in c.disk_archive else ''), ''),
-
-    (_('System memory'), c.system_memory, ''),
-    (_('CPU'), '%s %%' %(c.cpu), ''),
-    (_('Load'), '1min: %s, 5min: %s, 15min: %s' %(c.load['1_min'],c.load['5_min'],c.load['15_min']), ''),
-
-    ## rhodecode stuff
-    (_('Python version'), c.py_version, ''),
-    (_('Python path'), c.py_path, ''),
-    (_('GIT version'), c.git_version, ''),
-    (_('HG version'), c.hg_version, ''),
-    (_('SVN version'), c.svn_version, ''),
-    (_('Database'), "%s @ version: %s" % (c.db_type, c.db_migrate_version), ''),
-    (_('Database version'), c.db_version, ''),
-
- ]
-%>
 
 <pre>
 SYSTEM INFO
 -----------
 
-% for dt, dd, tt in elems:
-${dt}: ${dd}
+% for dt, dd, warn in c.data_items:
+  ${dt}${':' if dt else '---'}
+  ${dd}
+  % if warn and warn['message']:
+        ALERT_${warn['type'].upper()} ${warn['message']}
+  % endif
 % endfor
 
 PYTHON PACKAGES
 ---------------
 
-% for key, value in c.py_modules:
+% for key, value in c.py_modules['human_value']:
 ${key}: ${value}
 % endfor
 
 SYSTEM SETTINGS
 ---------------
 
-% for key, value in sorted(c.rhodecode_ini_safe.items()):
+% for key, value in sorted(c.rhodecode_config['human_value'].items()):
   % if isinstance(value, dict):
 
     % for key2, value2 in value.items():
 [${key}]${key2}: ${value2}
     % endfor
 
   % else:
 ${key}: ${value}
   % endif
 % endfor
 
 </pre>