# -*- coding: utf-8 -*- # Copyright 2010 - 2017 RhodeCode GmbH and the AppEnlight project authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import os import pkg_resources from datetime import datetime, timedelta import psutil import redis from pyramid.view import view_config from appenlight.models import DBSession from appenlight.models import Datastores from appenlight.lib.redis_keys import REDIS_KEYS def bytes2human(total): giga = 1024.0 ** 3 mega = 1024.0 ** 2 kilo = 1024.0 if giga <= total: return '{:0.1f}G'.format(total / giga) elif mega <= total: return '{:0.1f}M'.format(total / mega) else: return '{:0.1f}K'.format(total / kilo) log = logging.getLogger(__name__) @view_config(route_name='section_view', match_param=['section=admin_section', 'view=system'], renderer='json', permission='root_administration') def system(request): current_time = datetime.utcnow(). \ replace(second=0, microsecond=0) - timedelta(minutes=1) # global app counter processed_reports = request.registry.redis_conn.get( REDIS_KEYS['counters']['reports_per_minute'].format(current_time)) processed_reports = int(processed_reports) if processed_reports else 0 processed_logs = request.registry.redis_conn.get( REDIS_KEYS['counters']['logs_per_minute'].format(current_time)) processed_logs = int(processed_logs) if processed_logs else 0 processed_metrics = request.registry.redis_conn.get( REDIS_KEYS['counters']['metrics_per_minute'].format(current_time)) processed_metrics = int(processed_metrics) if processed_metrics else 0 waiting_reports = 0 waiting_logs = 0 waiting_metrics = 0 waiting_other = 0 if 'redis' in request.registry.settings['celery.broker_type']: redis_client = redis.StrictRedis.from_url( request.registry.settings['celery.broker_url']) waiting_reports = redis_client.llen('reports') waiting_logs = redis_client.llen('logs') waiting_metrics = redis_client.llen('metrics') waiting_other = redis_client.llen('default') # process def replace_inf(val): return val if val != psutil.RLIM_INFINITY else 'unlimited' p = psutil.Process() fd = p.rlimit(psutil.RLIMIT_NOFILE) memlock = p.rlimit(psutil.RLIMIT_MEMLOCK) self_info = { 'fds': {'soft': replace_inf(fd[0]), 'hard': replace_inf(fd[1])}, 'memlock': {'soft': replace_inf(memlock[0]), 'hard': replace_inf(memlock[1])}, } # disks disks = [] for part in psutil.disk_partitions(all=False): if os.name == 'nt': if 'cdrom' in part.opts or part.fstype == '': continue usage = psutil.disk_usage(part.mountpoint) disks.append({ 'device': part.device, 'total': bytes2human(usage.total), 'used': bytes2human(usage.used), 'free': bytes2human(usage.free), 'percentage': int(usage.percent), 'mountpoint': part.mountpoint, 'fstype': part.fstype }) # memory memory_v = psutil.virtual_memory() memory_s = psutil.swap_memory() memory = { 'total': bytes2human(memory_v.total), 'available': bytes2human(memory_v.available), 'percentage': memory_v.percent, 'used': bytes2human(memory_v.used), 'free': bytes2human(memory_v.free), 'active': bytes2human(memory_v.active), 'inactive': bytes2human(memory_v.inactive), 'buffers': bytes2human(memory_v.buffers), 'cached': bytes2human(memory_v.cached), 'swap_total': bytes2human(memory_s.total), 'swap_used': bytes2human(memory_s.used) } # load system_load = os.getloadavg() # processes min_mem = 1024 * 1024 * 40 # 40MB process_info = [] for p in psutil.process_iter(): mem_used = p.get_memory_info().rss if mem_used < min_mem: continue process_info.append({'owner': p.username(), 'pid': p.pid, 'cpu': round(p.get_cpu_percent(interval=0), 1), 'mem_percentage': round(p.get_memory_percent(), 1), 'mem_usage': bytes2human(mem_used), 'name': p.name(), 'command': ' '.join(p.cmdline()) }) process_info = sorted(process_info, key=lambda x: x['mem_percentage'], reverse=True) # pg tables db_size_query = ''' SELECT tablename, pg_total_relation_size(tablename::text) size FROM pg_tables WHERE tablename NOT LIKE 'pg_%' AND tablename NOT LIKE 'sql_%' ORDER BY size DESC;''' db_tables = [] for row in DBSession.execute(db_size_query): db_tables.append({"size_human": bytes2human(row.size), "table_name": row.tablename}) # es indices es_indices = [] result = Datastores.es.indices.stats(metric=['store, docs']) for ix, stats in result['indices'].items(): size = stats['primaries']['store']['size_in_bytes'] es_indices.append({'name': ix, 'size': size, 'size_human': bytes2human(size)}) # packages packages = ({'name': p.project_name, 'version': p.version} for p in pkg_resources.working_set) return {'db_tables': db_tables, 'es_indices': sorted(es_indices, key=lambda x: x['size'], reverse=True), 'process_info': process_info, 'system_load': system_load, 'disks': disks, 'memory': memory, 'packages': sorted(packages, key=lambda x: x['name'].lower()), 'current_time': current_time, 'queue_stats': { 'processed_reports': processed_reports, 'processed_logs': processed_logs, 'processed_metrics': processed_metrics, 'waiting_reports': waiting_reports, 'waiting_logs': waiting_logs, 'waiting_metrics': waiting_metrics, 'waiting_other': waiting_other }, 'self_info': self_info }