# HG changeset patch # User RhodeCode Admin # Date 2024-06-05 14:27:54 # Node ID 0968de67b7400601ab1dd41ce065b4289b920ea7 # Parent 9cce0276b2632c248b89b7fbd5a99cb0bcff8475 feat(system-info): properly report storage stats after diskcache removal diff --git a/rhodecode/apps/admin/views/system_info.py b/rhodecode/apps/admin/views/system_info.py --- a/rhodecode/apps/admin/views/system_info.py +++ b/rhodecode/apps/admin/views/system_info.py @@ -169,6 +169,7 @@ class AdminSystemInfoSettingsView(BaseAp (_('Gist storage location'), val('storage_gist')['path'], state('storage_gist')), (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')), + (_('Archive cache storage type'), val('storage_archive')['type'], state('storage_archive')), (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')), (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')), diff --git a/rhodecode/lib/rc_cache/archive_cache/utils.py b/rhodecode/lib/rc_cache/archive_cache/utils.py --- a/rhodecode/lib/rc_cache/archive_cache/utils.py +++ b/rhodecode/lib/rc_cache/archive_cache/utils.py @@ -15,6 +15,7 @@ # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ +import os class ArchiveCacheLock(Exception): @@ -28,3 +29,43 @@ def archive_iterator(_reader, block_size if not data: break yield data + + +def get_directory_statistics(start_path): + """ + total_files, total_size, directory_stats = get_directory_statistics(start_path) + + print(f"Directory statistics for: {start_path}\n") + print(f"Total files: {total_files}") + print(f"Total size: {format_size(total_size)}\n") + + :param start_path: + :return: + """ + + total_files = 0 + total_size = 0 + directory_stats = {} + + for dir_path, dir_names, file_names in os.walk(start_path): + dir_size = 0 + file_count = len(file_names) + + for file in file_names: + filepath = os.path.join(dir_path, file) + file_size = os.path.getsize(filepath) + dir_size += file_size + + directory_stats[dir_path] = {'file_count': file_count, 'size': dir_size} + total_files += file_count + total_size += dir_size + + return total_files, total_size, directory_stats + + +def format_size(size): + # Convert size in bytes to a human-readable format (e.g., KB, MB, GB) + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if size < 1024: + return f"{size:.2f} {unit}" + size /= 1024 diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py --- a/rhodecode/lib/system_info.py +++ b/rhodecode/lib/system_info.py @@ -100,7 +100,7 @@ def get_cert_path(ini_path): default = '/etc/ssl/certs/ca-certificates.crt' control_ca_bundle = os.path.join( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(ini_path)))), - '.rccontrol-profile/etc/ca-bundle.crt') + '/etc/ssl/certs/ca-certificates.crt') if os.path.isfile(control_ca_bundle): default = control_ca_bundle @@ -323,7 +323,7 @@ def cpu(): value['cpu_count'] = psutil.cpu_count() human_value = value.copy() - human_value['text'] = '{} cores at {} %'.format(value['cpu_count'], value['cpu']) + human_value['text'] = f'{value["cpu_count"]} cores at {value["cpu"]} %' return SysInfoRes(value=value, state=state, human_value=human_value) @@ -398,8 +398,8 @@ def storage_inodes(): @register_sysinfo def storage_archives(): import rhodecode - from rhodecode.lib.utils import safe_str from rhodecode.lib.helpers import format_byte_size_binary + from rhodecode.lib.rc_cache.archive_cache.utils import get_directory_statistics storage_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type') storage_key = 'archive_cache.filesystem.store_dir' @@ -408,29 +408,20 @@ def storage_archives(): f'{storage_key}=/path/to/cache option in the .ini file' path = rhodecode.ConfigGet().get_str(storage_key, missing=default_msg) - value = dict(percent=0, used=0, total=0, items=0, path=path, text='') + value = dict(percent=0, used=0, total=0, items=0, path=path, text='', type=storage_type) state = STATE_OK_DEFAULT try: if storage_type != 'filesystem': # raise Exc to stop reporting on different type raise ValueError('Storage type must be "filesystem"') - items_count = 0 - used = 0 - for root, dirs, files in os.walk(path): - if root == path: - items_count = len(dirs) + total_files, total_size, _directory_stats = get_directory_statistics(path) - for f in files: - try: - used += os.path.getsize(os.path.join(root, f)) - except OSError: - pass value.update({ 'percent': 100, - 'used': used, - 'total': used, - 'items': items_count + 'used': total_size, + 'total': total_size, + 'items': total_files }) except Exception as e: @@ -451,6 +442,8 @@ def storage_gist(): from rhodecode.model.gist import GIST_STORE_LOC from rhodecode.lib.utils import safe_str, get_rhodecode_repo_store_path from rhodecode.lib.helpers import format_byte_size_binary + from rhodecode.lib.rc_cache.archive_cache.utils import get_directory_statistics + path = safe_str(os.path.join( get_rhodecode_repo_store_path(), GIST_STORE_LOC)) @@ -459,22 +452,12 @@ def storage_gist(): state = STATE_OK_DEFAULT try: - items_count = 0 - used = 0 - for root, dirs, files in os.walk(path): - if root == path: - items_count = len(dirs) - - for f in files: - try: - used += os.path.getsize(os.path.join(root, f)) - except OSError: - pass + total_files, total_size, _directory_stats = get_directory_statistics(path) value.update({ 'percent': 100, - 'used': used, - 'total': used, - 'items': items_count + 'used': total_size, + 'total': total_size, + 'items': total_files }) except Exception as e: log.exception('failed to fetch gist storage items')