# HG changeset patch # User Marcin Kuzminski # Date 2016-11-14 20:14:29 # Node ID 5dd5a06369e48bc632d6ab1217c3d9da8555ccb6 # Parent 17efd44c89dc2ed9f598ea5699b8a5ca493e6e51 system-info: fetch vcs settings from vcsserver. Fixes #4276 - redone the system info page. Errors/Warnings now show next to the data, and also show up in API call. - implemented fetching of git/hg/svn version from running vcsserver - implemented fetching of vcsserver version from itself - added more warnings/errors detection on RAM/Inodes etc - few cleanups diff --git a/rhodecode/api/tests/test_get_server_info.py b/rhodecode/api/tests/test_get_server_info.py --- a/rhodecode/api/tests/test_get_server_info.py +++ b/rhodecode/api/tests/test_get_server_info.py @@ -45,9 +45,9 @@ class TestGetServerInfo(object): expected['uptime'] = resp['result']['uptime'] expected['load'] = resp['result']['load'] expected['cpu'] = resp['result']['cpu'] - expected['disk'] = resp['result']['disk'] - expected['disk_inodes'] = resp['result']['disk_inodes'] - expected['server_ip'] = '127.0.0.1:80' + expected['storage'] = resp['result']['storage'] + expected['storage_inodes'] = resp['result']['storage_inodes'] + expected['server'] = resp['result']['server'] assert_ok(id_, expected, given=response.body) @@ -60,8 +60,8 @@ class TestGetServerInfo(object): expected['uptime'] = resp['result']['uptime'] expected['load'] = resp['result']['load'] expected['cpu'] = resp['result']['cpu'] - expected['disk'] = resp['result']['disk'] - expected['disk_inodes'] = resp['result']['disk_inodes'] - expected['server_ip'] = '127.0.0.1:80' + expected['storage'] = resp['result']['storage'] + expected['storage_inodes'] = resp['result']['storage_inodes'] + expected['server'] = resp['result']['server'] assert_ok(id_, expected, given=response.body) diff --git a/rhodecode/controllers/admin/settings.py b/rhodecode/controllers/admin/settings.py --- a/rhodecode/controllers/admin/settings.py +++ b/rhodecode/controllers/admin/settings.py @@ -548,62 +548,85 @@ class SettingsController(BaseController) """GET /admin/settings/system: All items in the collection""" # url('admin_settings_system') snapshot = str2bool(request.GET.get('snapshot')) - c.active = 'system' + defaults = self._form_defaults() - defaults = self._form_defaults() - c.rhodecode_ini = rhodecode.CONFIG + c.active = 'system' c.rhodecode_update_url = defaults.get('rhodecode_update_url') server_info = ScmModel().get_server_info(request.environ) + for key, val in server_info.iteritems(): setattr(c, key, val) - if c.disk['percent'] > 90: - h.flash(h.literal(_( - 'Critical: your disk space is very low %s%% used' % - c.disk['percent'])), 'error') - elif c.disk['percent'] > 70: - h.flash(h.literal(_( - 'Warning: your disk space is running low %s%% used' % - c.disk['percent'])), 'warning') + def val(name, subkey='human_value'): + return server_info[name][subkey] + + def state(name): + return server_info[name]['state'] + + def val2(name): + val = server_info[name]['human_value'] + state = server_info[name]['state'] + return val, state - try: - c.uptime_age = h._age( - h.time_to_datetime(c.boot_time), False, show_suffix=False) - except TypeError: - c.uptime_age = c.boot_time + c.data_items = [ + # update info + (_('Update info'), h.literal( + '%s.' % ( + _('Check for updates')) + + '
%s.' % (_('Note: please make sure this server can access `%s` for the update link to work') % c.rhodecode_update_url) + ), ''), + + # RhodeCode specific + (_('RhodeCode Version'), c.rhodecode_version, ''), + (_('RhodeCode Server IP'), val('server')['server_ip'], state('server')), + (_('RhodeCode Server ID'), val('server')['server_id'], state('server')), + (_('RhodeCode Configuration'), val('rhodecode_config')['path'], state('rhodecode_config')), + ('', '', ''), # spacer + + # Database + (_('Database'), val('database')['url'], state('database')), + (_('Database version'), val('database')['version'], state('database')), + ('', '', ''), # spacer - try: - c.system_memory = '%s/%s, %s%% (%s%%) used%s' % ( - h.format_byte_size_binary(c.memory['used']), - h.format_byte_size_binary(c.memory['total']), - c.memory['percent2'], - c.memory['percent'], - ' %s' % c.memory['error'] if 'error' in c.memory else '') - except TypeError: - c.system_memory = 'NOT AVAILABLE' + # Platform/Python + (_('Platform'), val('platform'), state('platform')), + (_('Python version'), val('python')['version'], state('python')), + (_('Python path'), val('python')['executable'], state('python')), + ('', '', ''), # spacer + + # Systems stats + (_('CPU'), val('cpu'), state('cpu')), + (_('Load'), val('load'), state('load')), + (_('Memory'), val('memory'), state('memory')), + (_('Uptime'), val('uptime')['uptime'], state('uptime')), + ('', '', ''), # spacer + + # Repo storage + (_('Storage location'), val('storage')['path'], state('storage')), + (_('Storage info'), val('storage')['text'], state('storage')), + (_('Storage inodes'), val('storage_inodes')['text'], state('storage_inodes')), - rhodecode_ini_safe = rhodecode.CONFIG.copy() - blacklist = [ - 'rhodecode_license_key', - 'routes.map', - 'pylons.h', - 'pylons.app_globals', - 'pylons.environ_config', - 'sqlalchemy.db1.url', - ('app_conf', 'sqlalchemy.db1.url') + (_('Gist storage location'), val('storage_gist')['path'], state('storage_gist')), + (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')), + + (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')), + (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')), + + (_('Search storage'), val('storage_search')['path'], state('storage_search')), + (_('Search info'), val('storage_search')['text'], state('storage_search')), + ('', '', ''), # spacer + + # VCS specific + (_('VCS Backends'), val('vcs_backends'), state('vcs_backends')), + (_('VCS Server'), val('vcs_server')['text'], state('vcs_server')), + (_('GIT'), val('git'), state('git')), + (_('HG'), val('hg'), state('hg')), + (_('SVN'), val('svn'), state('svn')), + ] - for k in blacklist: - if isinstance(k, tuple): - section, key = k - if section in rhodecode_ini_safe: - rhodecode_ini_safe[section].pop(key, None) - else: - rhodecode_ini_safe.pop(k, None) - - c.rhodecode_ini_safe = rhodecode_ini_safe # TODO: marcink, figure out how to allow only selected users to do this - c.allowed_to_snapshot = False + c.allowed_to_snapshot = c.rhodecode_user.admin if snapshot: if c.allowed_to_snapshot: diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/system_info.py @@ -0,0 +1,582 @@ +import os +import sys +import time +import platform +import pkg_resources +import logging + + +log = logging.getLogger(__name__) + + +psutil = None + +try: + # cygwin cannot have yet psutil support. + import psutil as psutil +except ImportError: + pass + + +_NA = 'NOT AVAILABLE' + +STATE_OK = 'ok' +STATE_ERR = 'error' +STATE_WARN = 'warning' + +STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK} + + +# HELPERS +def percentage(part, whole): + whole = float(whole) + if whole > 0: + return 100 * float(part) / whole + return 0 + + +def get_storage_size(storage_path): + sizes = [] + for file_ in os.listdir(storage_path): + storage_file = os.path.join(storage_path, file_) + if os.path.isfile(storage_file): + try: + sizes.append(os.path.getsize(storage_file)) + except OSError: + log.exception('Failed to get size of storage file %s', + storage_file) + pass + + return sum(sizes) + + +class SysInfoRes(object): + def __init__(self, value, state=STATE_OK_DEFAULT, human_value=None): + self.value = value + self.state = state + self.human_value = human_value or value + + def __json__(self): + return { + 'value': self.value, + 'state': self.state, + 'human_value': self.human_value, + } + + def __str__(self): + return ''.format(self.__json__()) + + +class SysInfo(object): + + def __init__(self, func_name, **kwargs): + self.func_name = func_name + self.value = _NA + self.state = None + self.kwargs = kwargs or {} + + def __call__(self): + computed = self.compute(**self.kwargs) + if not isinstance(computed, SysInfoRes): + raise ValueError( + 'computed value for {} is not instance of ' + '{}, got {} instead'.format( + self.func_name, SysInfoRes, type(computed))) + return computed.__json__() + + def __str__(self): + return ''.format(self.func_name) + + def compute(self, **kwargs): + return self.func_name(**kwargs) + + +# SysInfo functions +def python_info(): + value = { + 'version': ' '.join(platform._sys_version()), + 'executable': sys.executable + } + return SysInfoRes(value=value) + + +def py_modules(): + mods = dict([(p.project_name, p.version) + for p in pkg_resources.working_set]) + value = sorted(mods.items(), key=lambda k: k[0].lower()) + return SysInfoRes(value=value) + + +def platform_type(): + from rhodecode.lib.utils import safe_unicode + value = safe_unicode(platform.platform()) + return SysInfoRes(value=value) + + +def uptime(): + from rhodecode.lib.helpers import age, time_to_datetime + from rhodecode.translation import _ + + _uptime = _uptime_human = {'boot_time': 0, 'uptime': 0} + state = STATE_OK_DEFAULT + if not psutil: + return SysInfoRes(value=_uptime, state=state) + + boot_time = psutil.boot_time() + _uptime['boot_time'] = boot_time + _uptime['uptime'] = time.time() - boot_time + + _uptime_human['boot_time'] = time_to_datetime(boot_time) + _uptime_human['uptime'] = _('Server started {}').format( + age(time_to_datetime(boot_time))) + + return SysInfoRes(value=_uptime, human_value=_uptime_human) + + +def memory(): + from rhodecode.lib.helpers import format_byte_size_binary + _memory = {'available': 0, 'used': 0, 'cached': 0, 'percent': 0, + 'percent_used': 0, 'free': 0, 'inactive': 0, 'active': 0, + 'shared': 0, 'total': 0, 'buffers': 0} + state = STATE_OK_DEFAULT + if not psutil: + return SysInfoRes(value=_memory, state=state) + + # memory + _memory = dict(psutil.virtual_memory()._asdict()) + _memory['percent_used'] = psutil._common.usage_percent( + (_memory['total'] - _memory['free']), _memory['total'], 1) + + try: + human_value = '%s/%s, %s%% used' % ( + format_byte_size_binary(_memory['used']), + format_byte_size_binary(_memory['total']), + _memory['percent_used'],) + except TypeError: + human_value = 'NOT AVAILABLE' + + if state['type'] == STATE_OK and _memory['percent_used'] > 90: + msg = 'Critical: your available RAM memory is very low.' + state = {'message': msg, 'type': STATE_ERR} + + elif state['type'] == STATE_OK and _memory['percent_used'] > 70: + msg = 'Warning: your available RAM memory is running low.' + state = {'message': msg, 'type': STATE_WARN} + + return SysInfoRes(value=_memory, state=state, human_value=human_value) + + +def machine_load(): + _load = {'1_min': _NA, '5_min': _NA, '15_min': _NA} + + state = STATE_OK_DEFAULT + if not psutil: + return SysInfoRes(value=_load, state=state) + + # load averages + if hasattr(psutil.os, 'getloadavg'): + _load = dict(zip(['1_min', '5_min', '15_min'], psutil.os.getloadavg())) + + human_value = '1min: %s, 5min: %s, 15min: %s' % ( + _load['1_min'], _load['5_min'], _load['15_min']) + + # TODO: warn about too-much load 15 min + return SysInfoRes(value=_load, state=state, human_value=human_value) + + +def cpu(): + state = STATE_OK_DEFAULT + cpu_value = 0 + if not psutil: + return SysInfoRes(value=cpu_value, state=state) + + cpu_value = psutil.cpu_percent(0.5) + human_value = '{} %'.format(cpu_value) + return SysInfoRes(value=cpu_value, state=state, human_value=human_value) + + +def storage(): + from rhodecode.lib.helpers import format_byte_size_binary + from rhodecode.model.settings import VcsSettingsModel + path = VcsSettingsModel().get_repos_location() + + # disk storage + disk = {'percent': 0, 'used': 0, 'total': 0, 'path': path, 'text': ''} + state = STATE_OK_DEFAULT + if not psutil: + return SysInfoRes(value=disk, state=state) + + try: + disk.update(dict(psutil.disk_usage(path)._asdict())) + except Exception as e: + log.exception('Failed to fetch disk info') + state = {'message': str(e), 'type': STATE_ERR} + + human_value = disk + human_value['text'] = "{}/{}, {}% used".format( + format_byte_size_binary(disk['used']), + format_byte_size_binary(disk['total']), + (disk['percent'])) + + if state['type'] == STATE_OK and disk['percent'] > 90: + msg = 'Critical: your disk space is very low.' + state = {'message': msg, 'type': STATE_ERR} + + elif state['type'] == STATE_OK and disk['percent'] > 70: + msg = 'Warning: your disk space is running low.' + state = {'message': msg, 'type': STATE_WARN} + + return SysInfoRes(value=disk, state=state, human_value=human_value) + + +def storage_inodes(): + from rhodecode.model.settings import VcsSettingsModel + path = VcsSettingsModel().get_repos_location() + + _disk_inodes = dict(percent=0, free=0, used=0, total=0, path=path, text='') + state = STATE_OK_DEFAULT + if not psutil: + return SysInfoRes(value=_disk_inodes, state=state) + + try: + i_stat = os.statvfs(path) + + _disk_inodes['used'] = i_stat.f_ffree + _disk_inodes['free'] = i_stat.f_favail + _disk_inodes['total'] = i_stat.f_files + _disk_inodes['percent'] = percentage( + _disk_inodes['used'], _disk_inodes['total']) + except Exception as e: + log.exception('Failed to fetch disk inodes info') + state = {'message': str(e), 'type': STATE_ERR} + + human_value = _disk_inodes + human_value['text'] = "{}/{}, {}% used".format( + _disk_inodes['used'], _disk_inodes['total'], _disk_inodes['percent']) + + if state['type'] == STATE_OK and _disk_inodes['percent'] > 90: + msg = 'Critical: your disk free inodes are very low.' + state = {'message': msg, 'type': STATE_ERR} + + elif state['type'] == STATE_OK and _disk_inodes['percent'] > 70: + msg = 'Warning: your disk free inodes are running low.' + state = {'message': msg, 'type': STATE_WARN} + + return SysInfoRes(value=_disk_inodes, state=state) + + +def storage_archives(): + import rhodecode + from rhodecode.lib.utils import safe_str + from rhodecode.lib.helpers import format_byte_size_binary + + msg = 'Enable this by setting ' \ + 'archive_cache_dir=/path/to/cache option in the .ini file' + path = safe_str(rhodecode.CONFIG.get('archive_cache_dir', msg)) + + disk_archive = dict(percent=0, used=0, total=0, items=0, path=path, text='') + state = STATE_OK_DEFAULT + try: + items_count = 0 + used = 0 + for root, dirs, files in os.walk(path): + if root == path: + items_count = len(dirs) + + for f in files: + try: + used += os.path.getsize(os.path.join(root, f)) + except OSError: + pass + disk_archive.update({ + 'percent': 100, + 'used': used, + 'total': used, + 'items': items_count + }) + + except Exception as e: + log.exception('failed to fetch archive cache storage') + state = {'message': str(e), 'type': STATE_ERR} + + human_value = disk_archive + human_value['text'] = "{} ({} items)".format(format_byte_size_binary( + disk_archive['used']), disk_archive['total']) + + return SysInfoRes(value=disk_archive, state=state, human_value=human_value) + + +def storage_gist(): + from rhodecode.model.gist import GIST_STORE_LOC + from rhodecode.model.settings import VcsSettingsModel + from rhodecode.lib.utils import safe_str + from rhodecode.lib.helpers import format_byte_size_binary + path = safe_str(os.path.join( + VcsSettingsModel().get_repos_location(), GIST_STORE_LOC)) + + # gist storage + _disk_gist = dict(percent=0, used=0, total=0, items=0, path=path, text='') + state = STATE_OK_DEFAULT + + try: + items_count = 0 + used = 0 + for root, dirs, files in os.walk(path): + if root == path: + items_count = len(dirs) + + for f in files: + try: + used += os.path.getsize(os.path.join(root, f)) + except OSError: + pass + _disk_gist.update({ + 'percent': 100, + 'used': used, + 'total': used, + 'items': items_count + }) + except Exception as e: + log.exception('failed to fetch gist storage items') + state = {'message': str(e), 'type': STATE_ERR} + + human_value = _disk_gist + human_value['text'] = "{} ({} items)".format(format_byte_size_binary( + _disk_gist['used']), _disk_gist['items']) + return SysInfoRes(value=_disk_gist, state=state, human_value=human_value) + + +def storage_search(): + import rhodecode + path = rhodecode.CONFIG.get('search.location', '') + + # search index storage + _disk_index = dict(percent=0, used=0, total=0, path=path, text='') + state = STATE_OK_DEFAULT + try: + search_index_storage_path_exists = os.path.isdir(path) + if search_index_storage_path_exists: + used = get_storage_size(path) + _disk_index.update({ + 'percent': 100, + 'used': used, + 'total': used, + }) + except Exception as e: + log.exception('failed to fetch search index storage') + state = {'message': str(e), 'type': STATE_ERR} + + human_value = _disk_index + human_value['text'] = "{}/{}, {}% used".format( + _disk_index['used'], _disk_index['total'], _disk_index['percent']) + + return SysInfoRes(value=_disk_index, state=state, human_value=human_value) + + +def git_info(): + from rhodecode.lib.vcs.backends import git + state = STATE_OK_DEFAULT + value = human_value = '' + try: + value = git.discover_git_version(raise_on_exc=True) + human_value = 'version reported from VCSServer: {}'.format(value) + except Exception as e: + state = {'message': str(e), 'type': STATE_ERR} + + return SysInfoRes(value=value, state=state, human_value=human_value) + + +def hg_info(): + from rhodecode.lib.vcs.backends import hg + state = STATE_OK_DEFAULT + value = human_value = '' + try: + value = hg.discover_hg_version(raise_on_exc=True) + human_value = 'version reported from VCSServer: {}'.format(value) + except Exception as e: + state = {'message': str(e), 'type': STATE_ERR} + return SysInfoRes(value=value, state=state, human_value=human_value) + + +def svn_info(): + from rhodecode.lib.vcs.backends import svn + state = STATE_OK_DEFAULT + value = human_value = '' + try: + value = svn.discover_svn_version(raise_on_exc=True) + human_value = 'version reported from VCSServer: {}'.format(value) + except Exception as e: + state = {'message': str(e), 'type': STATE_ERR} + return SysInfoRes(value=value, state=state, human_value=human_value) + + +def vcs_backends(): + import rhodecode + value = rhodecode.CONFIG.get('vcs.backends', '').split(',') + human_value = 'Enabled backends in order: {}'.format(','.join(value)) + return SysInfoRes(value=value, human_value=human_value) + + +def vcs_server(): + import rhodecode + from rhodecode.lib.vcs.backends import get_vcsserver_version + + server_url = rhodecode.CONFIG.get('vcs.server') + enabled = rhodecode.CONFIG.get('vcs.server.enable') + protocol = rhodecode.CONFIG.get('vcs.server.protocol') + state = STATE_OK_DEFAULT + version = None + + try: + version = get_vcsserver_version() + connection = 'connected' + except Exception as e: + connection = 'failed' + state = {'message': str(e), 'type': STATE_ERR} + + value = dict( + url=server_url, + enabled=enabled, + protocol=protocol, + connection=connection, + version=version, + text='', + ) + + human_value = value + human_value['text'] = \ + '{url}@ver:{ver} via {mode} mode, connection:{conn}'.format( + url=server_url, ver=version, mode=protocol, conn=connection) + + return SysInfoRes(value='', state=state, human_value=human_value) + + +def rhodecode_app_info(): + import rhodecode + return SysInfoRes(value={'rhodecode_version': rhodecode.__version__}) + + +def rhodecode_config(): + import rhodecode + path = rhodecode.CONFIG.get('__file__') + rhodecode_ini_safe = rhodecode.CONFIG.copy() + + blacklist = [ + 'rhodecode_license_key', + 'routes.map', + 'pylons.h', + 'pylons.app_globals', + 'pylons.environ_config', + 'sqlalchemy.db1.url', + 'channelstream.secret', + 'beaker.session.secret', + 'rhodecode.encrypted_values.secret', + 'rhodecode_auth_github_consumer_key', + 'rhodecode_auth_github_consumer_secret', + 'rhodecode_auth_google_consumer_key', + 'rhodecode_auth_google_consumer_secret', + 'rhodecode_auth_bitbucket_consumer_secret', + 'rhodecode_auth_bitbucket_consumer_key', + 'rhodecode_auth_twitter_consumer_secret', + 'rhodecode_auth_twitter_consumer_key', + + 'rhodecode_auth_twitter_secret', + 'rhodecode_auth_github_secret', + 'rhodecode_auth_google_secret', + 'rhodecode_auth_bitbucket_secret', + + 'appenlight.api_key', + ('app_conf', 'sqlalchemy.db1.url') + ] + for k in blacklist: + if isinstance(k, tuple): + section, key = k + if section in rhodecode_ini_safe: + rhodecode_ini_safe[section] = '**OBFUSCATED**' + else: + rhodecode_ini_safe.pop(k, None) + + # TODO: maybe put some CONFIG checks here ? + return SysInfoRes(value={'config': rhodecode_ini_safe, 'path': path}) + + +def database_info(): + import rhodecode + from sqlalchemy.engine import url as engine_url + from rhodecode.model.meta import Base as sql_base, Session + from rhodecode.model.db import DbMigrateVersion + + state = STATE_OK_DEFAULT + + db_migrate = DbMigrateVersion.query().filter( + DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one() + + db_url_obj = engine_url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url']) + + try: + engine = sql_base.metadata.bind + db_server_info = engine.dialect._get_server_version_info( + Session.connection(bind=engine)) + db_version = '.'.join(map(str, db_server_info)) + except Exception: + log.exception('failed to fetch db version') + db_version = 'UNKNOWN' + + db_info = dict( + migrate_version=db_migrate.version, + type=db_url_obj.get_backend_name(), + version=db_version, + url=repr(db_url_obj) + ) + + human_value = db_info + human_value['url'] = "{} @ migration version: {}".format( + db_info['url'], db_info['migrate_version']) + human_value['version'] = "{} {}".format(db_info['type'], db_info['version']) + return SysInfoRes(value=db_info, state=state, human_value=human_value) + + +def server_info(environ): + import rhodecode + from rhodecode.lib.base import get_server_ip_addr, get_server_port + + value = { + 'server_ip': '%s:%s' % ( + get_server_ip_addr(environ, log_errors=False), + get_server_port(environ) + ), + 'server_id': rhodecode.CONFIG.get('instance_id'), + } + return SysInfoRes(value=value) + + +def get_system_info(environ): + environ = environ or {} + return { + 'rhodecode_app': SysInfo(rhodecode_app_info)(), + 'rhodecode_config': SysInfo(rhodecode_config)(), + 'python': SysInfo(python_info)(), + 'py_modules': SysInfo(py_modules)(), + + 'platform': SysInfo(platform_type)(), + 'server': SysInfo(server_info, environ=environ)(), + 'database': SysInfo(database_info)(), + + 'storage': SysInfo(storage)(), + 'storage_inodes': SysInfo(storage_inodes)(), + 'storage_archive': SysInfo(storage_archives)(), + 'storage_search': SysInfo(storage_search)(), + 'storage_gist': SysInfo(storage_gist)(), + + 'uptime': SysInfo(uptime)(), + 'load': SysInfo(machine_load)(), + 'cpu': SysInfo(cpu)(), + 'memory': SysInfo(memory)(), + + 'vcs_backends': SysInfo(vcs_backends)(), + 'vcs_server': SysInfo(vcs_server)(), + + 'git': SysInfo(git_info)(), + 'hg': SysInfo(hg_info)(), + 'svn': SysInfo(svn_info)(), + } diff --git a/rhodecode/lib/vcs/__init__.py b/rhodecode/lib/vcs/__init__.py --- a/rhodecode/lib/vcs/__init__.py +++ b/rhodecode/lib/vcs/__init__.py @@ -99,6 +99,7 @@ def connect_pyro4(server_and_port): connection.Git = None connection.Hg = None connection.Svn = None + connection.Service = None def connect_http(server_and_port): @@ -113,6 +114,8 @@ def connect_http(server_and_port): server_and_port, '/hg', session_factory) connection.Svn = client_http.RepoMaker( server_and_port, '/svn', session_factory) + connection.Service = client_http.ServiceConnection( + server_and_port, '/_service', session_factory) scm_app.HG_REMOTE_WSGI = client_http.VcsHttpProxy( server_and_port, '/proxy/hg') @@ -124,6 +127,7 @@ def connect_http(server_and_port): connection.Git = None connection.Hg = None connection.Svn = None + connection.Service = None def connect_vcs(server_and_port, protocol): diff --git a/rhodecode/lib/vcs/backends/__init__.py b/rhodecode/lib/vcs/backends/__init__.py --- a/rhodecode/lib/vcs/backends/__init__.py +++ b/rhodecode/lib/vcs/backends/__init__.py @@ -76,3 +76,12 @@ def get_supported_backends(): Returns list of aliases of supported backends. """ return settings.BACKENDS.keys() + + +def get_vcsserver_version(): + from rhodecode.lib.vcs import connection + data = connection.Service.get_vcsserver_service_data() + if data and 'version' in data: + return data['version'] + + return None diff --git a/rhodecode/lib/vcs/backends/git/__init__.py b/rhodecode/lib/vcs/backends/git/__init__.py --- a/rhodecode/lib/vcs/backends/git/__init__.py +++ b/rhodecode/lib/vcs/backends/git/__init__.py @@ -33,7 +33,7 @@ from rhodecode.lib.vcs.backends.git.inme log = logging.getLogger(__name__) -def discover_git_version(): +def discover_git_version(raise_on_exc=False): """ Returns the string as it was returned by running 'git --version' @@ -44,4 +44,6 @@ def discover_git_version(): return connection.Git.discover_git_version() except Exception: log.warning("Failed to discover the Git version", exc_info=True) + if raise_on_exc: + raise return '' diff --git a/rhodecode/lib/vcs/backends/hg/__init__.py b/rhodecode/lib/vcs/backends/hg/__init__.py --- a/rhodecode/lib/vcs/backends/hg/__init__.py +++ b/rhodecode/lib/vcs/backends/hg/__init__.py @@ -21,7 +21,28 @@ """ HG module """ +import logging +from rhodecode.lib.vcs import connection from rhodecode.lib.vcs.backends.hg.commit import MercurialCommit from rhodecode.lib.vcs.backends.hg.inmemory import MercurialInMemoryCommit from rhodecode.lib.vcs.backends.hg.repository import MercurialRepository + + +log = logging.getLogger(__name__) + + +def discover_hg_version(raise_on_exc=False): + """ + Returns the string as it was returned by running 'git --version' + + It will return an empty string in case the connection is not initialized + or no vcsserver is available. + """ + try: + return connection.Hg.discover_hg_version() + except Exception: + log.warning("Failed to discover the HG version", exc_info=True) + if raise_on_exc: + raise + return '' diff --git a/rhodecode/lib/vcs/backends/svn/__init__.py b/rhodecode/lib/vcs/backends/svn/__init__.py --- a/rhodecode/lib/vcs/backends/svn/__init__.py +++ b/rhodecode/lib/vcs/backends/svn/__init__.py @@ -21,6 +21,27 @@ """ SVN module """ +import logging +from rhodecode.lib.vcs import connection from rhodecode.lib.vcs.backends.svn.commit import SubversionCommit from rhodecode.lib.vcs.backends.svn.repository import SubversionRepository + + +log = logging.getLogger(__name__) + + +def discover_svn_version(raise_on_exc=False): + """ + Returns the string as it was returned by running 'git --version' + + It will return an empty string in case the connection is not initialized + or no vcsserver is available. + """ + try: + return connection.Svn.discover_svn_version() + except Exception: + log.warning("Failed to discover the SVN version", exc_info=True) + if raise_on_exc: + raise + return '' diff --git a/rhodecode/lib/vcs/connection.py b/rhodecode/lib/vcs/connection.py --- a/rhodecode/lib/vcs/connection.py +++ b/rhodecode/lib/vcs/connection.py @@ -30,6 +30,8 @@ def _not_initialized(*args, **kwargs): "Make sure `vcs.server` is enabled in your configuration.") # TODO: figure out a nice default value for these things +Service = _not_initialized + Git = _not_initialized Hg = _not_initialized Svn = _not_initialized diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -25,13 +25,11 @@ Scm model for RhodeCode import os.path import re import sys -import time import traceback import logging import cStringIO import pkg_resources -import pylons from pylons.i18n.translation import _ from sqlalchemy import func from zope.cachedescriptors.property import Lazy as LazyProperty @@ -50,12 +48,12 @@ from rhodecode.lib.exceptions import Non from rhodecode.lib import hooks_utils, caches from rhodecode.lib.utils import ( get_filesystem_repos, action_logger, make_db_config) -from rhodecode.lib.utils2 import ( - safe_str, safe_unicode, get_server_url, md5) +from rhodecode.lib.utils2 import (safe_str, safe_unicode) +from rhodecode.lib.system_info import get_system_info from rhodecode.model import BaseModel from rhodecode.model.db import ( Repository, CacheKey, UserFollowing, UserLog, User, RepoGroup, - PullRequest, DbMigrateVersion) + PullRequest) from rhodecode.model.settings import VcsSettingsModel log = logging.getLogger(__name__) @@ -882,218 +880,8 @@ class ScmModel(BaseModel): self.install_svn_hooks(repo) def get_server_info(self, environ=None): - import platform - import rhodecode - import pkg_resources - from rhodecode.model.meta import Base as sql_base, Session - from sqlalchemy.engine import url - from rhodecode.lib.base import get_server_ip_addr, get_server_port - from rhodecode.lib.vcs.backends.git import discover_git_version - from rhodecode.model.gist import GIST_STORE_LOC - - def percentage(part, whole): - whole = float(whole) - if whole > 0: - return 100 * float(part) / whole - return 0 - - try: - # cygwin cannot have yet psutil support. - import psutil - except ImportError: - psutil = None - - environ = environ or {} - _NA = 'NOT AVAILABLE' - _memory = _NA - _uptime = _NA - _boot_time = _NA - _cpu = _NA - _disk = dict(percent=0, used=0, total=0, error='') - _disk_inodes = dict(percent=0, free=0, used=0, total=0, error='') - _load = {'1_min': _NA, '5_min': _NA, '15_min': _NA} - - model = VcsSettingsModel() - storage_path = model.get_repos_location() - gist_storage_path = os.path.join(storage_path, GIST_STORE_LOC) - archive_storage_path = rhodecode.CONFIG.get('archive_cache_dir', '') - search_index_storage_path = rhodecode.CONFIG.get('search.location', '') - - if psutil: - # disk storage - try: - _disk = dict(psutil.disk_usage(storage_path)._asdict()) - except Exception as e: - log.exception('Failed to fetch disk info') - _disk = {'percent': 0, 'used': 0, 'total': 0, 'error': str(e)} - - # disk inodes usage - try: - i_stat = os.statvfs(storage_path) - - _disk_inodes['used'] = i_stat.f_ffree - _disk_inodes['free'] = i_stat.f_favail - _disk_inodes['total'] = i_stat.f_files - _disk_inodes['percent'] = percentage( - _disk_inodes['used'], _disk_inodes['total']) - except Exception as e: - log.exception('Failed to fetch disk inodes info') - _disk_inodes['error'] = str(e) - - # memory - _memory = dict(psutil.virtual_memory()._asdict()) - _memory['percent2'] = psutil._common.usage_percent( - (_memory['total'] - _memory['free']), - _memory['total'], 1) - - # load averages - if hasattr(psutil.os, 'getloadavg'): - _load = dict(zip( - ['1_min', '5_min', '15_min'], psutil.os.getloadavg())) - _uptime = time.time() - psutil.boot_time() - _boot_time = psutil.boot_time() - _cpu = psutil.cpu_percent(0.5) - - mods = dict([(p.project_name, p.version) - for p in pkg_resources.working_set]) - - def get_storage_size(storage_path): - sizes = [] - for file_ in os.listdir(storage_path): - storage_file = os.path.join(storage_path, file_) - if os.path.isfile(storage_file): - try: - sizes.append(os.path.getsize(storage_file)) - except OSError: - log.exception('Failed to get size of storage file %s', - storage_file) - pass - - return sum(sizes) - - # archive cache storage - _disk_archive = {'percent': 0, 'used': 0, 'total': 0} - try: - archive_storage_path_exists = os.path.isdir( - archive_storage_path) - if archive_storage_path and archive_storage_path_exists: - used = get_storage_size(archive_storage_path) - _disk_archive.update({ - 'used': used, - 'total': used, - }) - except Exception as e: - log.exception('failed to fetch archive cache storage') - _disk_archive['error'] = str(e) - - # search index storage - _disk_index = {'percent': 0, 'used': 0, 'total': 0} - try: - search_index_storage_path_exists = os.path.isdir( - search_index_storage_path) - if search_index_storage_path_exists: - used = get_storage_size(search_index_storage_path) - _disk_index.update({ - 'percent': 100, - 'used': used, - 'total': used, - }) - except Exception as e: - log.exception('failed to fetch search index storage') - _disk_index['error'] = str(e) - - # gist storage - _disk_gist = {'percent': 0, 'used': 0, 'total': 0, 'items': 0} - try: - items_count = 0 - used = 0 - for root, dirs, files in os.walk(safe_str(gist_storage_path)): - if root == gist_storage_path: - items_count = len(dirs) - - for f in files: - try: - used += os.path.getsize(os.path.join(root, f)) - except OSError: - pass - _disk_gist.update({ - 'percent': 100, - 'used': used, - 'total': used, - 'items': items_count - }) - except Exception as e: - log.exception('failed to fetch gist storage items') - _disk_gist['error'] = str(e) - - # GIT info - git_ver = discover_git_version() - - # SVN info - # TODO: johbo: Add discover_svn_version to replace this code. - try: - import svn.core - svn_ver = svn.core.SVN_VERSION - except ImportError: - svn_ver = None - - # DB stuff - db_info = url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url']) - db_type = db_info.__to_string__() - try: - engine = sql_base.metadata.bind - db_server_info = engine.dialect._get_server_version_info( - Session.connection(bind=engine)) - db_version = '%s %s' % (db_info.drivername, - '.'.join(map(str, db_server_info))) - except Exception: - log.exception('failed to fetch db version') - db_version = '%s %s' % (db_info.drivername, '?') - - db_migrate = DbMigrateVersion.query().filter( - DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one() - db_migrate_version = db_migrate.version - - info = { - 'py_version': ' '.join(platform._sys_version()), - 'py_path': sys.executable, - 'py_modules': sorted(mods.items(), key=lambda k: k[0].lower()), - - 'platform': safe_unicode(platform.platform()), - 'storage': storage_path, - 'archive_storage': archive_storage_path, - 'index_storage': search_index_storage_path, - 'gist_storage': gist_storage_path, - - - 'db_type': db_type, - 'db_version': db_version, - 'db_migrate_version': db_migrate_version, - - 'rhodecode_version': rhodecode.__version__, - 'rhodecode_config_ini': rhodecode.CONFIG.get('__file__'), - 'server_ip': '%s:%s' % ( - get_server_ip_addr(environ, log_errors=False), - get_server_port(environ) - ), - 'server_id': rhodecode.CONFIG.get('instance_id'), - - 'git_version': safe_unicode(git_ver), - 'hg_version': mods.get('mercurial'), - 'svn_version': svn_ver, - - 'uptime': _uptime, - 'boot_time': _boot_time, - 'load': _load, - 'cpu': _cpu, - 'memory': _memory, - 'disk': _disk, - 'disk_inodes': _disk_inodes, - 'disk_archive': _disk_archive, - 'disk_gist': _disk_gist, - 'disk_index': _disk_index, - } - return info + server_info = get_system_info(environ) + return server_info def _check_rhodecode_hook(hook_path): diff --git a/rhodecode/templates/admin/settings/settings_system.html b/rhodecode/templates/admin/settings/settings_system.html --- a/rhodecode/templates/admin/settings/settings_system.html +++ b/rhodecode/templates/admin/settings/settings_system.html @@ -1,42 +1,3 @@ -<% - elems = [ - ## general - (_('RhodeCode Enterprise version'), h.literal('%s ' % (c.rhodecode_version, _('check for updates'))), ''), - (_('Upgrade info endpoint'), h.literal('%s
%s.' % (c.rhodecode_update_url, _('Note: please make sure this server can access this url'))), ''), - (_('Configuration INI file'), c.rhodecode_config_ini, ''), - ## systems stats - (_('RhodeCode Enterprise Server IP'), c.server_ip, ''), - (_('RhodeCode Enterprise Server ID'), c.server_id, ''), - (_('Platform'), c.platform, ''), - (_('Uptime'), c.uptime_age, ''), - (_('Storage location'), c.storage, ''), - (_('Storage disk space'), "%s/%s, %s%% used%s" % (h.format_byte_size_binary(c.disk['used']), h.format_byte_size_binary(c.disk['total']),(c.disk['percent']), ' %s' % c.disk['error'] if 'error' in c.disk else ''), ''), - (_('Storage file limit (inodes)'), "%s/%s, %.1f%% used%s" % (c.disk_inodes['used'], c.disk_inodes['total'],(c.disk_inodes['percent']), ' %s' % c.disk_inodes['error'] if 'error' in c.disk_inodes else ''), ''), - - (_('Search index storage'), c.index_storage, ''), - (_('Search index size'), "%s %s" % (h.format_byte_size_binary(c.disk_index['used']), ' %s' % c.disk_index['error'] if 'error' in c.disk_index else ''), ''), - - (_('Gist storage'), c.gist_storage, ''), - (_('Gist storage size'), "%s (%s items)%s" % (h.format_byte_size_binary(c.disk_gist['used']),c.disk_gist['items'], ' %s' % c.disk_gist['error'] if 'error' in c.disk_gist else ''), ''), - - (_('Archive cache'), h.literal('%s
%s.' % (c.archive_storage, _('Enable this by setting archive_cache_dir=/path/to/cache option in the .ini file'))), ''), - (_('Archive cache size'), "%s%s" % (h.format_byte_size_binary(c.disk_archive['used']), ' %s' % c.disk_archive['error'] if 'error' in c.disk_archive else ''), ''), - - (_('System memory'), c.system_memory, ''), - (_('CPU'), '%s %%' %(c.cpu), ''), - (_('Load'), '1min: %s, 5min: %s, 15min: %s' %(c.load['1_min'],c.load['5_min'],c.load['15_min']), ''), - - ## rhodecode stuff - (_('Python version'), c.py_version, ''), - (_('Python path'), c.py_path, ''), - (_('GIT version'), c.git_version, ''), - (_('HG version'), c.hg_version, ''), - (_('SVN version'), c.svn_version, ''), - (_('Database'), "%s @ version: %s" % (c.db_type, c.db_migrate_version), ''), - (_('Database version'), c.db_version, ''), - - ] -%> @@ -71,12 +38,12 @@ - %for key, value in c.py_modules: + % for key, value in c.py_modules['human_value']: ${key} ${value} - %endfor + % endfor diff --git a/rhodecode/templates/admin/settings/settings_system_snapshot.html b/rhodecode/templates/admin/settings/settings_system_snapshot.html --- a/rhodecode/templates/admin/settings/settings_system_snapshot.html +++ b/rhodecode/templates/admin/settings/settings_system_snapshot.html @@ -1,61 +1,27 @@ -<% - elems = [ - ## general - (_('RhodeCode Enterprise version'), c.rhodecode_version, ''), - (_('Upgrade info endpoint'), c.rhodecode_update_url, ''), - (_('Configuration INI file'), c.rhodecode_config_ini, ''), - ## systems stats - (_('RhodeCode Enterprise Server IP'), c.server_ip, ''), - (_('RhodeCode Enterprise Server ID'), c.server_id, ''), - (_('Platform'), c.platform, ''), - (_('Uptime'), c.uptime_age, ''), - (_('Storage location'), c.storage, ''), - (_('Storage disk space'), "%s/%s, %s%% used%s" % (h.format_byte_size_binary(c.disk['used']), h.format_byte_size_binary(c.disk['total']),(c.disk['percent']), ' %s' % c.disk['error'] if 'error' in c.disk else ''), ''), - - (_('Search index storage'), c.index_storage, ''), - (_('Search index size'), "%s %s" % (h.format_byte_size_binary(c.disk_index['used']), ' %s' % c.disk_index['error'] if 'error' in c.disk_index else ''), ''), - - (_('Gist storage'), c.gist_storage, ''), - (_('Gist storage size'), "%s (%s items)%s" % (h.format_byte_size_binary(c.disk_gist['used']),c.disk_gist['items'], ' %s' % c.disk_gist['error'] if 'error' in c.disk_gist else ''), ''), - - (_('Archive cache'), c.archive_storage, ''), - (_('Archive cache size'), "%s%s" % (h.format_byte_size_binary(c.disk_archive['used']), ' %s' % c.disk_archive['error'] if 'error' in c.disk_archive else ''), ''), - - (_('System memory'), c.system_memory, ''), - (_('CPU'), '%s %%' %(c.cpu), ''), - (_('Load'), '1min: %s, 5min: %s, 15min: %s' %(c.load['1_min'],c.load['5_min'],c.load['15_min']), ''), - - ## rhodecode stuff - (_('Python version'), c.py_version, ''), - (_('Python path'), c.py_path, ''), - (_('GIT version'), c.git_version, ''), - (_('HG version'), c.hg_version, ''), - (_('SVN version'), c.svn_version, ''), - (_('Database'), "%s @ version: %s" % (c.db_type, c.db_migrate_version), ''), - (_('Database version'), c.db_version, ''), - - ] -%> 
 SYSTEM INFO
 -----------
 
-% for dt, dd, tt in elems:
-${dt}: ${dd}
+% for dt, dd, warn in c.data_items:
+  ${dt}${':' if dt else '---'}
+  ${dd}
+  % if warn and warn['message']:
+        ALERT_${warn['type'].upper()} ${warn['message']}
+  % endif
 % endfor
 
 PYTHON PACKAGES
 ---------------
 
-% for key, value in c.py_modules:
+% for key, value in c.py_modules['human_value']:
 ${key}: ${value}
 % endfor
 
 SYSTEM SETTINGS
 ---------------
 
-% for key, value in sorted(c.rhodecode_ini_safe.items()):
+% for key, value in sorted(c.rhodecode_config['human_value'].items()):
   % if isinstance(value, dict):
 
     % for key2, value2 in value.items():
diff --git a/rhodecode/tests/lib/test_system_info.py b/rhodecode/tests/lib/test_system_info.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/tests/lib/test_system_info.py
@@ -0,0 +1,15 @@
+import py.test
+
+from rhodecode.lib.system_info import get_system_info
+
+
+def test_system_info(app):
+    info = get_system_info({})
+    assert info['load']['value']['15_min'] != 'NOT AVAILABLE'
+
+
+def test_system_info_without_psutil(monkeypatch, app):
+    import rhodecode.lib.system_info
+    monkeypatch.setattr(rhodecode.lib.system_info, 'psutil', None)
+    info = get_system_info({})
+    assert info['load']['value']['15_min'] == 'NOT AVAILABLE'