# -*- coding: utf-8 -*- # Copyright (C) 2010-2016 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 # (only), as published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ """ Scm model for RhodeCode """ import os.path import re import sys import time import traceback import logging import cStringIO import pkg_resources import pylons from pylons.i18n.translation import _ from sqlalchemy import func from zope.cachedescriptors.property import Lazy as LazyProperty import rhodecode from rhodecode.lib.vcs import get_backend from rhodecode.lib.vcs.exceptions import RepositoryError, NodeNotChangedError from rhodecode.lib.vcs.nodes import FileNode from rhodecode.lib.vcs.backends.base import EmptyCommit from rhodecode.lib import helpers as h from rhodecode.lib.auth import ( HasRepoPermissionAny, HasRepoGroupPermissionAny, HasUserGroupPermissionAny) from rhodecode.lib.exceptions import NonRelativePathError, IMCCommitError from rhodecode.lib import hooks_utils, caches from rhodecode.lib.utils import ( get_filesystem_repos, action_logger, make_db_config) from rhodecode.lib.utils2 import ( safe_str, safe_unicode, get_server_url, md5) from rhodecode.model import BaseModel from rhodecode.model.db import ( Repository, CacheKey, UserFollowing, UserLog, User, RepoGroup, PullRequest, DbMigrateVersion) from rhodecode.model.settings import VcsSettingsModel log = logging.getLogger(__name__) class UserTemp(object): def __init__(self, user_id): self.user_id = user_id def __repr__(self): return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id) class RepoTemp(object): def __init__(self, repo_id): self.repo_id = repo_id def __repr__(self): return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id) class SimpleCachedRepoList(object): """ Lighter version of of iteration of repos without the scm initialisation, and with cache usage """ def __init__(self, db_repo_list, repos_path, order_by=None, perm_set=None): self.db_repo_list = db_repo_list self.repos_path = repos_path self.order_by = order_by self.reversed = (order_by or '').startswith('-') if not perm_set: perm_set = ['repository.read', 'repository.write', 'repository.admin'] self.perm_set = perm_set def __len__(self): return len(self.db_repo_list) def __repr__(self): return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) def __iter__(self): for dbr in self.db_repo_list: # check permission at this level has_perm = HasRepoPermissionAny(*self.perm_set)( dbr.repo_name, 'SimpleCachedRepoList check') if not has_perm: continue tmp_d = { 'name': dbr.repo_name, 'dbrepo': dbr.get_dict(), 'dbrepo_fork': dbr.fork.get_dict() if dbr.fork else {} } yield tmp_d class _PermCheckIterator(object): def __init__( self, obj_list, obj_attr, perm_set, perm_checker, extra_kwargs=None): """ Creates iterator from given list of objects, additionally checking permission for them from perm_set var :param obj_list: list of db objects :param obj_attr: attribute of object to pass into perm_checker :param perm_set: list of permissions to check :param perm_checker: callable to check permissions against """ self.obj_list = obj_list self.obj_attr = obj_attr self.perm_set = perm_set self.perm_checker = perm_checker self.extra_kwargs = extra_kwargs or {} def __len__(self): return len(self.obj_list) def __repr__(self): return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) def __iter__(self): checker = self.perm_checker(*self.perm_set) for db_obj in self.obj_list: # check permission at this level name = getattr(db_obj, self.obj_attr, None) if not checker(name, self.__class__.__name__, **self.extra_kwargs): continue yield db_obj class RepoList(_PermCheckIterator): def __init__(self, db_repo_list, perm_set=None, extra_kwargs=None): if not perm_set: perm_set = [ 'repository.read', 'repository.write', 'repository.admin'] super(RepoList, self).__init__( obj_list=db_repo_list, obj_attr='repo_name', perm_set=perm_set, perm_checker=HasRepoPermissionAny, extra_kwargs=extra_kwargs) class RepoGroupList(_PermCheckIterator): def __init__(self, db_repo_group_list, perm_set=None, extra_kwargs=None): if not perm_set: perm_set = ['group.read', 'group.write', 'group.admin'] super(RepoGroupList, self).__init__( obj_list=db_repo_group_list, obj_attr='group_name', perm_set=perm_set, perm_checker=HasRepoGroupPermissionAny, extra_kwargs=extra_kwargs) class UserGroupList(_PermCheckIterator): def __init__(self, db_user_group_list, perm_set=None, extra_kwargs=None): if not perm_set: perm_set = ['usergroup.read', 'usergroup.write', 'usergroup.admin'] super(UserGroupList, self).__init__( obj_list=db_user_group_list, obj_attr='users_group_name', perm_set=perm_set, perm_checker=HasUserGroupPermissionAny, extra_kwargs=extra_kwargs) class ScmModel(BaseModel): """ Generic Scm Model """ @LazyProperty def repos_path(self): """ Gets the repositories root path from database """ settings_model = VcsSettingsModel(sa=self.sa) return settings_model.get_repos_location() def repo_scan(self, repos_path=None): """ Listing of repositories in given path. This path should not be a repository itself. Return a dictionary of repository objects :param repos_path: path to directory containing repositories """ if repos_path is None: repos_path = self.repos_path log.info('scanning for repositories in %s', repos_path) config = make_db_config() config.set('extensions', 'largefiles', '') repos = {} for name, path in get_filesystem_repos(repos_path, recursive=True): # name need to be decomposed and put back together using the / # since this is internal storage separator for rhodecode name = Repository.normalize_repo_name(name) try: if name in repos: raise RepositoryError('Duplicate repository name %s ' 'found in %s' % (name, path)) elif path[0] in rhodecode.BACKENDS: klass = get_backend(path[0]) repos[name] = klass(path[1], config=config) except OSError: continue log.debug('found %s paths with repositories', len(repos)) return repos def get_repos(self, all_repos=None, sort_key=None): """ Get all repositories from db and for each repo create it's backend instance and fill that backed with information from database :param all_repos: list of repository names as strings give specific repositories list, good for filtering :param sort_key: initial sorting of repositories """ if all_repos is None: all_repos = self.sa.query(Repository)\ .filter(Repository.group_id == None)\ .order_by(func.lower(Repository.repo_name)).all() repo_iter = SimpleCachedRepoList( all_repos, repos_path=self.repos_path, order_by=sort_key) return repo_iter def get_repo_groups(self, all_groups=None): if all_groups is None: all_groups = RepoGroup.query()\ .filter(RepoGroup.group_parent_id == None).all() return [x for x in RepoGroupList(all_groups)] def mark_for_invalidation(self, repo_name, delete=False): """ Mark caches of this repo invalid in the database. `delete` flag removes the cache entries :param repo_name: the repo_name for which caches should be marked invalid, or deleted :param delete: delete the entry keys instead of setting bool flag on them """ CacheKey.set_invalidate(repo_name, delete=delete) repo = Repository.get_by_repo_name(repo_name) if repo: config = repo._config config.set('extensions', 'largefiles', '') cs_cache = None if delete: # if we do a hard clear, reset last-commit to Empty cs_cache = EmptyCommit() repo.update_commit_cache(config=config, cs_cache=cs_cache) caches.clear_repo_caches(repo_name) def toggle_following_repo(self, follow_repo_id, user_id): f = self.sa.query(UserFollowing)\ .filter(UserFollowing.follows_repo_id == follow_repo_id)\ .filter(UserFollowing.user_id == user_id).scalar() if f is not None: try: self.sa.delete(f) action_logger(UserTemp(user_id), 'stopped_following_repo', RepoTemp(follow_repo_id)) return except Exception: log.error(traceback.format_exc()) raise try: f = UserFollowing() f.user_id = user_id f.follows_repo_id = follow_repo_id self.sa.add(f) action_logger(UserTemp(user_id), 'started_following_repo', RepoTemp(follow_repo_id)) except Exception: log.error(traceback.format_exc()) raise def toggle_following_user(self, follow_user_id, user_id): f = self.sa.query(UserFollowing)\ .filter(UserFollowing.follows_user_id == follow_user_id)\ .filter(UserFollowing.user_id == user_id).scalar() if f is not None: try: self.sa.delete(f) return except Exception: log.error(traceback.format_exc()) raise try: f = UserFollowing() f.user_id = user_id f.follows_user_id = follow_user_id self.sa.add(f) except Exception: log.error(traceback.format_exc()) raise def is_following_repo(self, repo_name, user_id, cache=False): r = self.sa.query(Repository)\ .filter(Repository.repo_name == repo_name).scalar() f = self.sa.query(UserFollowing)\ .filter(UserFollowing.follows_repository == r)\ .filter(UserFollowing.user_id == user_id).scalar() return f is not None def is_following_user(self, username, user_id, cache=False): u = User.get_by_username(username) f = self.sa.query(UserFollowing)\ .filter(UserFollowing.follows_user == u)\ .filter(UserFollowing.user_id == user_id).scalar() return f is not None def get_followers(self, repo): repo = self._get_repo(repo) return self.sa.query(UserFollowing)\ .filter(UserFollowing.follows_repository == repo).count() def get_forks(self, repo): repo = self._get_repo(repo) return self.sa.query(Repository)\ .filter(Repository.fork == repo).count() def get_pull_requests(self, repo): repo = self._get_repo(repo) return self.sa.query(PullRequest)\ .filter(PullRequest.target_repo == repo)\ .filter(PullRequest.status != PullRequest.STATUS_CLOSED).count() def mark_as_fork(self, repo, fork, user): repo = self._get_repo(repo) fork = self._get_repo(fork) if fork and repo.repo_id == fork.repo_id: raise Exception("Cannot set repository as fork of itself") if fork and repo.repo_type != fork.repo_type: raise RepositoryError( "Cannot set repository as fork of repository with other type") repo.fork = fork self.sa.add(repo) return repo def pull_changes(self, repo, username): dbrepo = self._get_repo(repo) clone_uri = dbrepo.clone_uri if not clone_uri: raise Exception("This repository doesn't have a clone uri") repo = dbrepo.scm_instance(cache=False) # TODO: marcink fix this an re-enable since we need common logic # for hg/git remove hooks so we don't trigger them on fetching # commits from remote repo.config.clear_section('hooks') repo_name = dbrepo.repo_name try: # TODO: we need to make sure those operations call proper hooks ! repo.pull(clone_uri) self.mark_for_invalidation(repo_name) except Exception: log.error(traceback.format_exc()) raise def commit_change(self, repo, repo_name, commit, user, author, message, content, f_path): """ Commits changes :param repo: SCM instance """ user = self._get_user(user) # decoding here will force that we have proper encoded values # in any other case this will throw exceptions and deny commit content = safe_str(content) path = safe_str(f_path) # message and author needs to be unicode # proper backend should then translate that into required type message = safe_unicode(message) author = safe_unicode(author) imc = repo.in_memory_commit imc.change(FileNode(path, content, mode=commit.get_file_mode(f_path))) try: # TODO: handle pre-push action ! tip = imc.commit( message=message, author=author, parents=[commit], branch=commit.branch) except Exception as e: log.error(traceback.format_exc()) raise IMCCommitError(str(e)) finally: # always clear caches, if commit fails we want fresh object also self.mark_for_invalidation(repo_name) # We trigger the post-push action hooks_utils.trigger_post_push_hook( username=user.username, action='push_local', repo_name=repo_name, repo_alias=repo.alias, commit_ids=[tip.raw_id]) return tip def _sanitize_path(self, f_path): if f_path.startswith('/') or f_path.startswith('.') or '../' in f_path: raise NonRelativePathError('%s is not an relative path' % f_path) if f_path: f_path = os.path.normpath(f_path) return f_path def get_dirnode_metadata(self, commit, dir_node): if not dir_node.is_dir(): return [] data = [] for node in dir_node: if not node.is_file(): # we skip file-nodes continue last_commit = node.last_commit last_commit_date = last_commit.date data.append({ 'name': node.name, 'size': h.format_byte_size_binary(node.size), 'modified_at': h.format_date(last_commit_date), 'modified_ts': last_commit_date.isoformat(), 'revision': last_commit.revision, 'short_id': last_commit.short_id, 'message': h.escape(last_commit.message), 'author': h.escape(last_commit.author), 'user_profile': h.gravatar_with_user(last_commit.author), }) return data def get_nodes(self, repo_name, commit_id, root_path='/', flat=True, extended_info=False, content=False): """ recursive walk in root dir and return a set of all path in that dir based on repository walk function :param repo_name: name of repository :param commit_id: commit id for which to list nodes :param root_path: root path to list :param flat: return as a list, if False returns a dict with decription """ _files = list() _dirs = list() try: _repo = self._get_repo(repo_name) commit = _repo.scm_instance().get_commit(commit_id=commit_id) root_path = root_path.lstrip('/') for __, dirs, files in commit.walk(root_path): for f in files: _content = None _data = f.unicode_path if not flat: _data = { "name": f.unicode_path, "type": "file", } if extended_info: _content = safe_str(f.content) _data.update({ "md5": md5(_content), "binary": f.is_binary, "size": f.size, "extension": f.extension, "mimetype": f.mimetype, "lines": f.lines()[0] }) if content: full_content = None if not f.is_binary: # in case we loaded the _content already # re-use it, or load from f[ile] full_content = _content or safe_str(f.content) _data.update({ "content": full_content }) _files.append(_data) for d in dirs: _data = d.unicode_path if not flat: _data = { "name": d.unicode_path, "type": "dir", } if extended_info: _data.update({ "md5": None, "binary": None, "size": None, "extension": None, }) if content: _data.update({ "content": None }) _dirs.append(_data) except RepositoryError: log.debug("Exception in get_nodes", exc_info=True) raise return _dirs, _files def create_nodes(self, user, repo, message, nodes, parent_commit=None, author=None, trigger_push_hook=True): """ Commits given multiple nodes into repo :param user: RhodeCode User object or user_id, the commiter :param repo: RhodeCode Repository object :param message: commit message :param nodes: mapping {filename:{'content':content},...} :param parent_commit: parent commit, can be empty than it's initial commit :param author: author of commit, cna be different that commiter only for git :param trigger_push_hook: trigger push hooks :returns: new commited commit """ user = self._get_user(user) scm_instance = repo.scm_instance(cache=False) processed_nodes = [] for f_path in nodes: f_path = self._sanitize_path(f_path) content = nodes[f_path]['content'] f_path = safe_str(f_path) # decoding here will force that we have proper encoded values # in any other case this will throw exceptions and deny commit if isinstance(content, (basestring,)): content = safe_str(content) elif isinstance(content, (file, cStringIO.OutputType,)): content = content.read() else: raise Exception('Content is of unrecognized type %s' % ( type(content) )) processed_nodes.append((f_path, content)) message = safe_unicode(message) commiter = user.full_contact author = safe_unicode(author) if author else commiter imc = scm_instance.in_memory_commit if not parent_commit: parent_commit = EmptyCommit(alias=scm_instance.alias) if isinstance(parent_commit, EmptyCommit): # EmptyCommit means we we're editing empty repository parents = None else: parents = [parent_commit] # add multiple nodes for path, content in processed_nodes: imc.add(FileNode(path, content=content)) # TODO: handle pre push scenario tip = imc.commit(message=message, author=author, parents=parents, branch=parent_commit.branch) self.mark_for_invalidation(repo.repo_name) if trigger_push_hook: hooks_utils.trigger_post_push_hook( username=user.username, action='push_local', repo_name=repo.repo_name, repo_alias=scm_instance.alias, commit_ids=[tip.raw_id]) return tip def update_nodes(self, user, repo, message, nodes, parent_commit=None, author=None, trigger_push_hook=True): user = self._get_user(user) scm_instance = repo.scm_instance(cache=False) message = safe_unicode(message) commiter = user.full_contact author = safe_unicode(author) if author else commiter imc = scm_instance.in_memory_commit if not parent_commit: parent_commit = EmptyCommit(alias=scm_instance.alias) if isinstance(parent_commit, EmptyCommit): # EmptyCommit means we we're editing empty repository parents = None else: parents = [parent_commit] # add multiple nodes for _filename, data in nodes.items(): # new filename, can be renamed from the old one, also sanitaze # the path for any hack around relative paths like ../../ etc. filename = self._sanitize_path(data['filename']) old_filename = self._sanitize_path(_filename) content = data['content'] filenode = FileNode(old_filename, content=content) op = data['op'] if op == 'add': imc.add(filenode) elif op == 'del': imc.remove(filenode) elif op == 'mod': if filename != old_filename: # TODO: handle renames more efficient, needs vcs lib # changes imc.remove(filenode) imc.add(FileNode(filename, content=content)) else: imc.change(filenode) try: # TODO: handle pre push scenario # commit changes tip = imc.commit(message=message, author=author, parents=parents, branch=parent_commit.branch) except NodeNotChangedError: raise except Exception as e: log.exception("Unexpected exception during call to imc.commit") raise IMCCommitError(str(e)) finally: # always clear caches, if commit fails we want fresh object also self.mark_for_invalidation(repo.repo_name) if trigger_push_hook: hooks_utils.trigger_post_push_hook( username=user.username, action='push_local', repo_name=repo.repo_name, repo_alias=scm_instance.alias, commit_ids=[tip.raw_id]) def delete_nodes(self, user, repo, message, nodes, parent_commit=None, author=None, trigger_push_hook=True): """ Deletes given multiple nodes into `repo` :param user: RhodeCode User object or user_id, the committer :param repo: RhodeCode Repository object :param message: commit message :param nodes: mapping {filename:{'content':content},...} :param parent_commit: parent commit, can be empty than it's initial commit :param author: author of commit, cna be different that commiter only for git :param trigger_push_hook: trigger push hooks :returns: new commit after deletion """ user = self._get_user(user) scm_instance = repo.scm_instance(cache=False) processed_nodes = [] for f_path in nodes: f_path = self._sanitize_path(f_path) # content can be empty but for compatabilty it allows same dicts # structure as add_nodes content = nodes[f_path].get('content') processed_nodes.append((f_path, content)) message = safe_unicode(message) commiter = user.full_contact author = safe_unicode(author) if author else commiter imc = scm_instance.in_memory_commit if not parent_commit: parent_commit = EmptyCommit(alias=scm_instance.alias) if isinstance(parent_commit, EmptyCommit): # EmptyCommit means we we're editing empty repository parents = None else: parents = [parent_commit] # add multiple nodes for path, content in processed_nodes: imc.remove(FileNode(path, content=content)) # TODO: handle pre push scenario tip = imc.commit(message=message, author=author, parents=parents, branch=parent_commit.branch) self.mark_for_invalidation(repo.repo_name) if trigger_push_hook: hooks_utils.trigger_post_push_hook( username=user.username, action='push_local', repo_name=repo.repo_name, repo_alias=scm_instance.alias, commit_ids=[tip.raw_id]) return tip def strip(self, repo, commit_id, branch): scm_instance = repo.scm_instance(cache=False) scm_instance.config.clear_section('hooks') scm_instance.strip(commit_id, branch) self.mark_for_invalidation(repo.repo_name) def get_unread_journal(self): return self.sa.query(UserLog).count() def get_repo_landing_revs(self, repo=None): """ Generates select option with tags branches and bookmarks (for hg only) grouped by type :param repo: """ hist_l = [] choices = [] repo = self._get_repo(repo) hist_l.append(['rev:tip', _('latest tip')]) choices.append('rev:tip') if not repo: return choices, hist_l repo = repo.scm_instance() branches_group = ( [(u'branch:%s' % safe_unicode(b), safe_unicode(b)) for b in repo.branches], _("Branches")) hist_l.append(branches_group) choices.extend([x[0] for x in branches_group[0]]) if repo.alias == 'hg': bookmarks_group = ( [(u'book:%s' % safe_unicode(b), safe_unicode(b)) for b in repo.bookmarks], _("Bookmarks")) hist_l.append(bookmarks_group) choices.extend([x[0] for x in bookmarks_group[0]]) tags_group = ( [(u'tag:%s' % safe_unicode(t), safe_unicode(t)) for t in repo.tags], _("Tags")) hist_l.append(tags_group) choices.extend([x[0] for x in tags_group[0]]) return choices, hist_l def install_git_hook(self, repo, force_create=False): """ Creates a rhodecode hook inside a git repository :param repo: Instance of VCS repo :param force_create: Create even if same name hook exists """ loc = os.path.join(repo.path, 'hooks') if not repo.bare: loc = os.path.join(repo.path, '.git', 'hooks') if not os.path.isdir(loc): os.makedirs(loc, mode=0777) tmpl_post = pkg_resources.resource_string( 'rhodecode', '/'.join( ('config', 'hook_templates', 'git_post_receive.py.tmpl'))) tmpl_pre = pkg_resources.resource_string( 'rhodecode', '/'.join( ('config', 'hook_templates', 'git_pre_receive.py.tmpl'))) for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]: _hook_file = os.path.join(loc, '%s-receive' % h_type) log.debug('Installing git hook in repo %s', repo) _rhodecode_hook = _check_rhodecode_hook(_hook_file) if _rhodecode_hook or force_create: log.debug('writing %s hook file !', h_type) try: with open(_hook_file, 'wb') as f: tmpl = tmpl.replace('_TMPL_', rhodecode.__version__) tmpl = tmpl.replace('_ENV_', sys.executable) f.write(tmpl) os.chmod(_hook_file, 0755) except IOError: log.exception('error writing hook file %s', _hook_file) else: log.debug('skipping writing hook file') def install_svn_hooks(self, repo, force_create=False): """ Creates rhodecode hooks inside a svn repository :param repo: Instance of VCS repo :param force_create: Create even if same name hook exists """ hooks_path = os.path.join(repo.path, 'hooks') if not os.path.isdir(hooks_path): os.makedirs(hooks_path) post_commit_tmpl = pkg_resources.resource_string( 'rhodecode', '/'.join( ('config', 'hook_templates', 'svn_post_commit_hook.py.tmpl'))) pre_commit_template = pkg_resources.resource_string( 'rhodecode', '/'.join( ('config', 'hook_templates', 'svn_pre_commit_hook.py.tmpl'))) templates = { 'post-commit': post_commit_tmpl, 'pre-commit': pre_commit_template } for filename in templates: _hook_file = os.path.join(hooks_path, filename) _rhodecode_hook = _check_rhodecode_hook(_hook_file) if _rhodecode_hook or force_create: log.debug('writing %s hook file !', filename) template = templates[filename] try: with open(_hook_file, 'wb') as f: template = template.replace( '_TMPL_', rhodecode.__version__) template = template.replace('_ENV_', sys.executable) f.write(template) os.chmod(_hook_file, 0755) except IOError: log.exception('error writing hook file %s', filename) else: log.debug('skipping writing hook file') def install_hooks(self, repo, repo_type): if repo_type == 'git': self.install_git_hook(repo) elif repo_type == 'svn': self.install_svn_hooks(repo) def get_server_info(self, environ=None): import platform import rhodecode import pkg_resources from rhodecode.model.meta import Base as sql_base, Session from sqlalchemy.engine import url from rhodecode.lib.base import get_server_ip_addr, get_server_port from rhodecode.lib.vcs.backends.git import discover_git_version from rhodecode.model.gist import GIST_STORE_LOC try: # cygwin cannot have yet psutil support. import psutil except ImportError: psutil = None environ = environ or {} _NA = 'NOT AVAILABLE' _memory = _NA _uptime = _NA _boot_time = _NA _cpu = _NA _disk = dict(percent=0, used=0, total=0, error='') _load = {'1_min': _NA, '5_min': _NA, '15_min': _NA} model = VcsSettingsModel() storage_path = model.get_repos_location() gist_storage_path = os.path.join(storage_path, GIST_STORE_LOC) archive_storage_path = rhodecode.CONFIG.get('archive_cache_dir', '') search_index_storage_path = rhodecode.CONFIG.get('search.location', '') if psutil: # disk storage try: _disk = dict(psutil.disk_usage(storage_path)._asdict()) except Exception as e: log.exception('Failed to fetch disk info') _disk = {'percent': 0, 'used': 0, 'total': 0, 'error': str(e)} # memory _memory = dict(psutil.virtual_memory()._asdict()) _memory['percent2'] = psutil._common.usage_percent( (_memory['total'] - _memory['free']), _memory['total'], 1) # load averages if hasattr(psutil.os, 'getloadavg'): _load = dict(zip( ['1_min', '5_min', '15_min'], psutil.os.getloadavg())) _uptime = time.time() - psutil.boot_time() _boot_time = psutil.boot_time() _cpu = psutil.cpu_percent(0.5) mods = dict([(p.project_name, p.version) for p in pkg_resources.working_set]) def get_storage_size(storage_path): sizes = [] for file_ in os.listdir(storage_path): storage_file = os.path.join(storage_path, file_) if os.path.isfile(storage_file): try: sizes.append(os.path.getsize(storage_file)) except OSError: log.exception('Failed to get size of storage file %s', storage_file) pass return sum(sizes) # archive cache storage _disk_archive = {'percent': 0, 'used': 0, 'total': 0} try: archive_storage_path_exists = os.path.isdir( archive_storage_path) if archive_storage_path and archive_storage_path_exists: used = get_storage_size(archive_storage_path) _disk_archive.update({ 'used': used, 'total': used, }) except Exception as e: log.exception('failed to fetch archive cache storage') _disk_archive['error'] = str(e) # search index storage _disk_index = {'percent': 0, 'used': 0, 'total': 0} try: search_index_storage_path_exists = os.path.isdir( search_index_storage_path) if search_index_storage_path_exists: used = get_storage_size(search_index_storage_path) _disk_index.update({ 'percent': 100, 'used': used, 'total': used, }) except Exception as e: log.exception('failed to fetch search index storage') _disk_index['error'] = str(e) # gist storage _disk_gist = {'percent': 0, 'used': 0, 'total': 0, 'items': 0} try: items_count = 0 used = 0 for root, dirs, files in os.walk(safe_str(gist_storage_path)): if root == gist_storage_path: items_count = len(dirs) for f in files: try: used += os.path.getsize(os.path.join(root, f)) except OSError: pass _disk_gist.update({ 'percent': 100, 'used': used, 'total': used, 'items': items_count }) except Exception as e: log.exception('failed to fetch gist storage items') _disk_gist['error'] = str(e) # GIT info git_ver = discover_git_version() # SVN info # TODO: johbo: Add discover_svn_version to replace this code. try: import svn.core svn_ver = svn.core.SVN_VERSION except ImportError: svn_ver = None # DB stuff db_info = url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url']) db_type = db_info.__to_string__() try: engine = sql_base.metadata.bind db_server_info = engine.dialect._get_server_version_info( Session.connection(bind=engine)) db_version = '%s %s' % (db_info.drivername, '.'.join(map(str, db_server_info))) except Exception: log.exception('failed to fetch db version') db_version = '%s %s' % (db_info.drivername, '?') db_migrate = DbMigrateVersion.query().filter( DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one() db_migrate_version = db_migrate.version info = { 'py_version': ' '.join(platform._sys_version()), 'py_path': sys.executable, 'py_modules': sorted(mods.items(), key=lambda k: k[0].lower()), 'platform': safe_unicode(platform.platform()), 'storage': storage_path, 'archive_storage': archive_storage_path, 'index_storage': search_index_storage_path, 'gist_storage': gist_storage_path, 'db_type': db_type, 'db_version': db_version, 'db_migrate_version': db_migrate_version, 'rhodecode_version': rhodecode.__version__, 'rhodecode_config_ini': rhodecode.CONFIG.get('__file__'), 'server_ip': '%s:%s' % ( get_server_ip_addr(environ, log_errors=False), get_server_port(environ) ), 'server_id': rhodecode.CONFIG.get('instance_id'), 'git_version': safe_unicode(git_ver), 'hg_version': mods.get('mercurial'), 'svn_version': svn_ver, 'uptime': _uptime, 'boot_time': _boot_time, 'load': _load, 'cpu': _cpu, 'memory': _memory, 'disk': _disk, 'disk_archive': _disk_archive, 'disk_gist': _disk_gist, 'disk_index': _disk_index, } return info def _check_rhodecode_hook(hook_path): """ Check if the hook was created by RhodeCode """ if not os.path.exists(hook_path): return True log.debug('hook exists, checking if it is from rhodecode') hook_content = _read_hook(hook_path) matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content) if matches: try: version = matches.groups()[0] log.debug('got %s, it is rhodecode', version) return True except Exception: log.exception("Exception while reading the hook version.") return False def _read_hook(hook_path): with open(hook_path, 'rb') as f: content = f.read() return content