##// END OF EJS Templates
tests: Add a ``db`` fixture that initializes the database....
tests: Add a ``db`` fixture that initializes the database. This is quite useful if tests only need the database and not the whole app. Then only this fixture is needed instead of the full blown pylonsapp/app fixtures.

File last commit:

r502:8bb872ec default
r914:cf699af2 default
Show More
scm.py
1102 lines | 39.2 KiB | text/x-python | PythonLexer
# -*- coding: utf-8 -*-
# Copyright (C) 2010-2016 RhodeCode GmbH
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License, version 3
# (only), as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
"""
Scm model for RhodeCode
"""
import os.path
import re
import sys
import time
import traceback
import logging
import cStringIO
import pkg_resources
import pylons
from pylons.i18n.translation import _
from sqlalchemy import func
from zope.cachedescriptors.property import Lazy as LazyProperty
import rhodecode
from rhodecode.lib.vcs import get_backend
from rhodecode.lib.vcs.exceptions import RepositoryError, NodeNotChangedError
from rhodecode.lib.vcs.nodes import FileNode
from rhodecode.lib.vcs.backends.base import EmptyCommit
from rhodecode.lib import helpers as h
from rhodecode.lib.auth import (
HasRepoPermissionAny, HasRepoGroupPermissionAny,
HasUserGroupPermissionAny)
from rhodecode.lib.exceptions import NonRelativePathError, IMCCommitError
from rhodecode.lib import hooks_utils, caches
from rhodecode.lib.utils import (
get_filesystem_repos, action_logger, make_db_config)
from rhodecode.lib.utils2 import (
safe_str, safe_unicode, get_server_url, md5)
from rhodecode.model import BaseModel
from rhodecode.model.db import (
Repository, CacheKey, UserFollowing, UserLog, User, RepoGroup,
PullRequest, DbMigrateVersion)
from rhodecode.model.settings import VcsSettingsModel
log = logging.getLogger(__name__)
class UserTemp(object):
def __init__(self, user_id):
self.user_id = user_id
def __repr__(self):
return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
class RepoTemp(object):
def __init__(self, repo_id):
self.repo_id = repo_id
def __repr__(self):
return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
class SimpleCachedRepoList(object):
"""
Lighter version of of iteration of repos without the scm initialisation,
and with cache usage
"""
def __init__(self, db_repo_list, repos_path, order_by=None, perm_set=None):
self.db_repo_list = db_repo_list
self.repos_path = repos_path
self.order_by = order_by
self.reversed = (order_by or '').startswith('-')
if not perm_set:
perm_set = ['repository.read', 'repository.write',
'repository.admin']
self.perm_set = perm_set
def __len__(self):
return len(self.db_repo_list)
def __repr__(self):
return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
def __iter__(self):
for dbr in self.db_repo_list:
# check permission at this level
has_perm = HasRepoPermissionAny(*self.perm_set)(
dbr.repo_name, 'SimpleCachedRepoList check')
if not has_perm:
continue
tmp_d = {
'name': dbr.repo_name,
'dbrepo': dbr.get_dict(),
'dbrepo_fork': dbr.fork.get_dict() if dbr.fork else {}
}
yield tmp_d
class _PermCheckIterator(object):
def __init__(
self, obj_list, obj_attr, perm_set, perm_checker,
extra_kwargs=None):
"""
Creates iterator from given list of objects, additionally
checking permission for them from perm_set var
:param obj_list: list of db objects
:param obj_attr: attribute of object to pass into perm_checker
:param perm_set: list of permissions to check
:param perm_checker: callable to check permissions against
"""
self.obj_list = obj_list
self.obj_attr = obj_attr
self.perm_set = perm_set
self.perm_checker = perm_checker
self.extra_kwargs = extra_kwargs or {}
def __len__(self):
return len(self.obj_list)
def __repr__(self):
return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
def __iter__(self):
checker = self.perm_checker(*self.perm_set)
for db_obj in self.obj_list:
# check permission at this level
name = getattr(db_obj, self.obj_attr, None)
if not checker(name, self.__class__.__name__, **self.extra_kwargs):
continue
yield db_obj
class RepoList(_PermCheckIterator):
def __init__(self, db_repo_list, perm_set=None, extra_kwargs=None):
if not perm_set:
perm_set = [
'repository.read', 'repository.write', 'repository.admin']
super(RepoList, self).__init__(
obj_list=db_repo_list,
obj_attr='repo_name', perm_set=perm_set,
perm_checker=HasRepoPermissionAny,
extra_kwargs=extra_kwargs)
class RepoGroupList(_PermCheckIterator):
def __init__(self, db_repo_group_list, perm_set=None, extra_kwargs=None):
if not perm_set:
perm_set = ['group.read', 'group.write', 'group.admin']
super(RepoGroupList, self).__init__(
obj_list=db_repo_group_list,
obj_attr='group_name', perm_set=perm_set,
perm_checker=HasRepoGroupPermissionAny,
extra_kwargs=extra_kwargs)
class UserGroupList(_PermCheckIterator):
def __init__(self, db_user_group_list, perm_set=None, extra_kwargs=None):
if not perm_set:
perm_set = ['usergroup.read', 'usergroup.write', 'usergroup.admin']
super(UserGroupList, self).__init__(
obj_list=db_user_group_list,
obj_attr='users_group_name', perm_set=perm_set,
perm_checker=HasUserGroupPermissionAny,
extra_kwargs=extra_kwargs)
class ScmModel(BaseModel):
"""
Generic Scm Model
"""
@LazyProperty
def repos_path(self):
"""
Gets the repositories root path from database
"""
settings_model = VcsSettingsModel(sa=self.sa)
return settings_model.get_repos_location()
def repo_scan(self, repos_path=None):
"""
Listing of repositories in given path. This path should not be a
repository itself. Return a dictionary of repository objects
:param repos_path: path to directory containing repositories
"""
if repos_path is None:
repos_path = self.repos_path
log.info('scanning for repositories in %s', repos_path)
config = make_db_config()
config.set('extensions', 'largefiles', '')
repos = {}
for name, path in get_filesystem_repos(repos_path, recursive=True):
# name need to be decomposed and put back together using the /
# since this is internal storage separator for rhodecode
name = Repository.normalize_repo_name(name)
try:
if name in repos:
raise RepositoryError('Duplicate repository name %s '
'found in %s' % (name, path))
elif path[0] in rhodecode.BACKENDS:
klass = get_backend(path[0])
repos[name] = klass(path[1], config=config)
except OSError:
continue
log.debug('found %s paths with repositories', len(repos))
return repos
def get_repos(self, all_repos=None, sort_key=None):
"""
Get all repositories from db and for each repo create it's
backend instance and fill that backed with information from database
:param all_repos: list of repository names as strings
give specific repositories list, good for filtering
:param sort_key: initial sorting of repositories
"""
if all_repos is None:
all_repos = self.sa.query(Repository)\
.filter(Repository.group_id == None)\
.order_by(func.lower(Repository.repo_name)).all()
repo_iter = SimpleCachedRepoList(
all_repos, repos_path=self.repos_path, order_by=sort_key)
return repo_iter
def get_repo_groups(self, all_groups=None):
if all_groups is None:
all_groups = RepoGroup.query()\
.filter(RepoGroup.group_parent_id == None).all()
return [x for x in RepoGroupList(all_groups)]
def mark_for_invalidation(self, repo_name, delete=False):
"""
Mark caches of this repo invalid in the database. `delete` flag
removes the cache entries
:param repo_name: the repo_name for which caches should be marked
invalid, or deleted
:param delete: delete the entry keys instead of setting bool
flag on them
"""
CacheKey.set_invalidate(repo_name, delete=delete)
repo = Repository.get_by_repo_name(repo_name)
if repo:
config = repo._config
config.set('extensions', 'largefiles', '')
repo.update_commit_cache(config=config, cs_cache=None)
caches.clear_repo_caches(repo_name)
def toggle_following_repo(self, follow_repo_id, user_id):
f = self.sa.query(UserFollowing)\
.filter(UserFollowing.follows_repo_id == follow_repo_id)\
.filter(UserFollowing.user_id == user_id).scalar()
if f is not None:
try:
self.sa.delete(f)
action_logger(UserTemp(user_id),
'stopped_following_repo',
RepoTemp(follow_repo_id))
return
except Exception:
log.error(traceback.format_exc())
raise
try:
f = UserFollowing()
f.user_id = user_id
f.follows_repo_id = follow_repo_id
self.sa.add(f)
action_logger(UserTemp(user_id),
'started_following_repo',
RepoTemp(follow_repo_id))
except Exception:
log.error(traceback.format_exc())
raise
def toggle_following_user(self, follow_user_id, user_id):
f = self.sa.query(UserFollowing)\
.filter(UserFollowing.follows_user_id == follow_user_id)\
.filter(UserFollowing.user_id == user_id).scalar()
if f is not None:
try:
self.sa.delete(f)
return
except Exception:
log.error(traceback.format_exc())
raise
try:
f = UserFollowing()
f.user_id = user_id
f.follows_user_id = follow_user_id
self.sa.add(f)
except Exception:
log.error(traceback.format_exc())
raise
def is_following_repo(self, repo_name, user_id, cache=False):
r = self.sa.query(Repository)\
.filter(Repository.repo_name == repo_name).scalar()
f = self.sa.query(UserFollowing)\
.filter(UserFollowing.follows_repository == r)\
.filter(UserFollowing.user_id == user_id).scalar()
return f is not None
def is_following_user(self, username, user_id, cache=False):
u = User.get_by_username(username)
f = self.sa.query(UserFollowing)\
.filter(UserFollowing.follows_user == u)\
.filter(UserFollowing.user_id == user_id).scalar()
return f is not None
def get_followers(self, repo):
repo = self._get_repo(repo)
return self.sa.query(UserFollowing)\
.filter(UserFollowing.follows_repository == repo).count()
def get_forks(self, repo):
repo = self._get_repo(repo)
return self.sa.query(Repository)\
.filter(Repository.fork == repo).count()
def get_pull_requests(self, repo):
repo = self._get_repo(repo)
return self.sa.query(PullRequest)\
.filter(PullRequest.target_repo == repo)\
.filter(PullRequest.status != PullRequest.STATUS_CLOSED).count()
def mark_as_fork(self, repo, fork, user):
repo = self._get_repo(repo)
fork = self._get_repo(fork)
if fork and repo.repo_id == fork.repo_id:
raise Exception("Cannot set repository as fork of itself")
if fork and repo.repo_type != fork.repo_type:
raise RepositoryError(
"Cannot set repository as fork of repository with other type")
repo.fork = fork
self.sa.add(repo)
return repo
def pull_changes(self, repo, username):
dbrepo = self._get_repo(repo)
clone_uri = dbrepo.clone_uri
if not clone_uri:
raise Exception("This repository doesn't have a clone uri")
repo = dbrepo.scm_instance(cache=False)
# TODO: marcink fix this an re-enable since we need common logic
# for hg/git remove hooks so we don't trigger them on fetching
# commits from remote
repo.config.clear_section('hooks')
repo_name = dbrepo.repo_name
try:
# TODO: we need to make sure those operations call proper hooks !
repo.pull(clone_uri)
self.mark_for_invalidation(repo_name)
except Exception:
log.error(traceback.format_exc())
raise
def commit_change(self, repo, repo_name, commit, user, author, message,
content, f_path):
"""
Commits changes
:param repo: SCM instance
"""
user = self._get_user(user)
# decoding here will force that we have proper encoded values
# in any other case this will throw exceptions and deny commit
content = safe_str(content)
path = safe_str(f_path)
# message and author needs to be unicode
# proper backend should then translate that into required type
message = safe_unicode(message)
author = safe_unicode(author)
imc = repo.in_memory_commit
imc.change(FileNode(path, content, mode=commit.get_file_mode(f_path)))
try:
# TODO: handle pre-push action !
tip = imc.commit(
message=message, author=author, parents=[commit],
branch=commit.branch)
except Exception as e:
log.error(traceback.format_exc())
raise IMCCommitError(str(e))
finally:
# always clear caches, if commit fails we want fresh object also
self.mark_for_invalidation(repo_name)
# We trigger the post-push action
hooks_utils.trigger_post_push_hook(
username=user.username, action='push_local', repo_name=repo_name,
repo_alias=repo.alias, commit_ids=[tip.raw_id])
return tip
def _sanitize_path(self, f_path):
if f_path.startswith('/') or f_path.startswith('./') or '../' in f_path:
raise NonRelativePathError('%s is not an relative path' % f_path)
if f_path:
f_path = os.path.normpath(f_path)
return f_path
def get_dirnode_metadata(self, commit, dir_node):
if not dir_node.is_dir():
return []
data = []
for node in dir_node:
if not node.is_file():
# we skip file-nodes
continue
last_commit = node.last_commit
last_commit_date = last_commit.date
data.append({
'name': node.name,
'size': h.format_byte_size_binary(node.size),
'modified_at': h.format_date(last_commit_date),
'modified_ts': last_commit_date.isoformat(),
'revision': last_commit.revision,
'short_id': last_commit.short_id,
'message': h.escape(last_commit.message),
'author': h.escape(last_commit.author),
'user_profile': h.gravatar_with_user(last_commit.author),
})
return data
def get_nodes(self, repo_name, commit_id, root_path='/', flat=True,
extended_info=False, content=False, max_file_bytes=None):
"""
recursive walk in root dir and return a set of all path in that dir
based on repository walk function
:param repo_name: name of repository
:param commit_id: commit id for which to list nodes
:param root_path: root path to list
:param flat: return as a list, if False returns a dict with description
:param max_file_bytes: will not return file contents over this limit
"""
_files = list()
_dirs = list()
try:
_repo = self._get_repo(repo_name)
commit = _repo.scm_instance().get_commit(commit_id=commit_id)
root_path = root_path.lstrip('/')
for __, dirs, files in commit.walk(root_path):
for f in files:
_content = None
_data = f.unicode_path
over_size_limit = (max_file_bytes is not None
and f.size > max_file_bytes)
if not flat:
_data = {
"name": f.unicode_path,
"type": "file",
}
if extended_info:
_data.update({
"md5": f.md5,
"binary": f.is_binary,
"size": f.size,
"extension": f.extension,
"mimetype": f.mimetype,
"lines": f.lines()[0]
})
if content:
full_content = None
if not f.is_binary and not over_size_limit:
full_content = safe_str(f.content)
_data.update({
"content": full_content,
})
_files.append(_data)
for d in dirs:
_data = d.unicode_path
if not flat:
_data = {
"name": d.unicode_path,
"type": "dir",
}
if extended_info:
_data.update({
"md5": None,
"binary": None,
"size": None,
"extension": None,
})
if content:
_data.update({
"content": None
})
_dirs.append(_data)
except RepositoryError:
log.debug("Exception in get_nodes", exc_info=True)
raise
return _dirs, _files
def create_nodes(self, user, repo, message, nodes, parent_commit=None,
author=None, trigger_push_hook=True):
"""
Commits given multiple nodes into repo
:param user: RhodeCode User object or user_id, the commiter
:param repo: RhodeCode Repository object
:param message: commit message
:param nodes: mapping {filename:{'content':content},...}
:param parent_commit: parent commit, can be empty than it's
initial commit
:param author: author of commit, cna be different that commiter
only for git
:param trigger_push_hook: trigger push hooks
:returns: new commited commit
"""
user = self._get_user(user)
scm_instance = repo.scm_instance(cache=False)
processed_nodes = []
for f_path in nodes:
f_path = self._sanitize_path(f_path)
content = nodes[f_path]['content']
f_path = safe_str(f_path)
# decoding here will force that we have proper encoded values
# in any other case this will throw exceptions and deny commit
if isinstance(content, (basestring,)):
content = safe_str(content)
elif isinstance(content, (file, cStringIO.OutputType,)):
content = content.read()
else:
raise Exception('Content is of unrecognized type %s' % (
type(content)
))
processed_nodes.append((f_path, content))
message = safe_unicode(message)
commiter = user.full_contact
author = safe_unicode(author) if author else commiter
imc = scm_instance.in_memory_commit
if not parent_commit:
parent_commit = EmptyCommit(alias=scm_instance.alias)
if isinstance(parent_commit, EmptyCommit):
# EmptyCommit means we we're editing empty repository
parents = None
else:
parents = [parent_commit]
# add multiple nodes
for path, content in processed_nodes:
imc.add(FileNode(path, content=content))
# TODO: handle pre push scenario
tip = imc.commit(message=message,
author=author,
parents=parents,
branch=parent_commit.branch)
self.mark_for_invalidation(repo.repo_name)
if trigger_push_hook:
hooks_utils.trigger_post_push_hook(
username=user.username, action='push_local',
repo_name=repo.repo_name, repo_alias=scm_instance.alias,
commit_ids=[tip.raw_id])
return tip
def update_nodes(self, user, repo, message, nodes, parent_commit=None,
author=None, trigger_push_hook=True):
user = self._get_user(user)
scm_instance = repo.scm_instance(cache=False)
message = safe_unicode(message)
commiter = user.full_contact
author = safe_unicode(author) if author else commiter
imc = scm_instance.in_memory_commit
if not parent_commit:
parent_commit = EmptyCommit(alias=scm_instance.alias)
if isinstance(parent_commit, EmptyCommit):
# EmptyCommit means we we're editing empty repository
parents = None
else:
parents = [parent_commit]
# add multiple nodes
for _filename, data in nodes.items():
# new filename, can be renamed from the old one, also sanitaze
# the path for any hack around relative paths like ../../ etc.
filename = self._sanitize_path(data['filename'])
old_filename = self._sanitize_path(_filename)
content = data['content']
filenode = FileNode(old_filename, content=content)
op = data['op']
if op == 'add':
imc.add(filenode)
elif op == 'del':
imc.remove(filenode)
elif op == 'mod':
if filename != old_filename:
# TODO: handle renames more efficient, needs vcs lib
# changes
imc.remove(filenode)
imc.add(FileNode(filename, content=content))
else:
imc.change(filenode)
try:
# TODO: handle pre push scenario
# commit changes
tip = imc.commit(message=message,
author=author,
parents=parents,
branch=parent_commit.branch)
except NodeNotChangedError:
raise
except Exception as e:
log.exception("Unexpected exception during call to imc.commit")
raise IMCCommitError(str(e))
finally:
# always clear caches, if commit fails we want fresh object also
self.mark_for_invalidation(repo.repo_name)
if trigger_push_hook:
hooks_utils.trigger_post_push_hook(
username=user.username, action='push_local',
repo_name=repo.repo_name, repo_alias=scm_instance.alias,
commit_ids=[tip.raw_id])
def delete_nodes(self, user, repo, message, nodes, parent_commit=None,
author=None, trigger_push_hook=True):
"""
Deletes given multiple nodes into `repo`
:param user: RhodeCode User object or user_id, the committer
:param repo: RhodeCode Repository object
:param message: commit message
:param nodes: mapping {filename:{'content':content},...}
:param parent_commit: parent commit, can be empty than it's initial
commit
:param author: author of commit, cna be different that commiter only
for git
:param trigger_push_hook: trigger push hooks
:returns: new commit after deletion
"""
user = self._get_user(user)
scm_instance = repo.scm_instance(cache=False)
processed_nodes = []
for f_path in nodes:
f_path = self._sanitize_path(f_path)
# content can be empty but for compatabilty it allows same dicts
# structure as add_nodes
content = nodes[f_path].get('content')
processed_nodes.append((f_path, content))
message = safe_unicode(message)
commiter = user.full_contact
author = safe_unicode(author) if author else commiter
imc = scm_instance.in_memory_commit
if not parent_commit:
parent_commit = EmptyCommit(alias=scm_instance.alias)
if isinstance(parent_commit, EmptyCommit):
# EmptyCommit means we we're editing empty repository
parents = None
else:
parents = [parent_commit]
# add multiple nodes
for path, content in processed_nodes:
imc.remove(FileNode(path, content=content))
# TODO: handle pre push scenario
tip = imc.commit(message=message,
author=author,
parents=parents,
branch=parent_commit.branch)
self.mark_for_invalidation(repo.repo_name)
if trigger_push_hook:
hooks_utils.trigger_post_push_hook(
username=user.username, action='push_local',
repo_name=repo.repo_name, repo_alias=scm_instance.alias,
commit_ids=[tip.raw_id])
return tip
def strip(self, repo, commit_id, branch):
scm_instance = repo.scm_instance(cache=False)
scm_instance.config.clear_section('hooks')
scm_instance.strip(commit_id, branch)
self.mark_for_invalidation(repo.repo_name)
def get_unread_journal(self):
return self.sa.query(UserLog).count()
def get_repo_landing_revs(self, repo=None):
"""
Generates select option with tags branches and bookmarks (for hg only)
grouped by type
:param repo:
"""
hist_l = []
choices = []
repo = self._get_repo(repo)
hist_l.append(['rev:tip', _('latest tip')])
choices.append('rev:tip')
if not repo:
return choices, hist_l
repo = repo.scm_instance()
branches_group = (
[(u'branch:%s' % safe_unicode(b), safe_unicode(b))
for b in repo.branches],
_("Branches"))
hist_l.append(branches_group)
choices.extend([x[0] for x in branches_group[0]])
if repo.alias == 'hg':
bookmarks_group = (
[(u'book:%s' % safe_unicode(b), safe_unicode(b))
for b in repo.bookmarks],
_("Bookmarks"))
hist_l.append(bookmarks_group)
choices.extend([x[0] for x in bookmarks_group[0]])
tags_group = (
[(u'tag:%s' % safe_unicode(t), safe_unicode(t))
for t in repo.tags],
_("Tags"))
hist_l.append(tags_group)
choices.extend([x[0] for x in tags_group[0]])
return choices, hist_l
def install_git_hook(self, repo, force_create=False):
"""
Creates a rhodecode hook inside a git repository
:param repo: Instance of VCS repo
:param force_create: Create even if same name hook exists
"""
loc = os.path.join(repo.path, 'hooks')
if not repo.bare:
loc = os.path.join(repo.path, '.git', 'hooks')
if not os.path.isdir(loc):
os.makedirs(loc, mode=0777)
tmpl_post = pkg_resources.resource_string(
'rhodecode', '/'.join(
('config', 'hook_templates', 'git_post_receive.py.tmpl')))
tmpl_pre = pkg_resources.resource_string(
'rhodecode', '/'.join(
('config', 'hook_templates', 'git_pre_receive.py.tmpl')))
for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:
_hook_file = os.path.join(loc, '%s-receive' % h_type)
log.debug('Installing git hook in repo %s', repo)
_rhodecode_hook = _check_rhodecode_hook(_hook_file)
if _rhodecode_hook or force_create:
log.debug('writing %s hook file !', h_type)
try:
with open(_hook_file, 'wb') as f:
tmpl = tmpl.replace('_TMPL_', rhodecode.__version__)
tmpl = tmpl.replace('_ENV_', sys.executable)
f.write(tmpl)
os.chmod(_hook_file, 0755)
except IOError:
log.exception('error writing hook file %s', _hook_file)
else:
log.debug('skipping writing hook file')
def install_svn_hooks(self, repo, force_create=False):
"""
Creates rhodecode hooks inside a svn repository
:param repo: Instance of VCS repo
:param force_create: Create even if same name hook exists
"""
hooks_path = os.path.join(repo.path, 'hooks')
if not os.path.isdir(hooks_path):
os.makedirs(hooks_path)
post_commit_tmpl = pkg_resources.resource_string(
'rhodecode', '/'.join(
('config', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
pre_commit_template = pkg_resources.resource_string(
'rhodecode', '/'.join(
('config', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
templates = {
'post-commit': post_commit_tmpl,
'pre-commit': pre_commit_template
}
for filename in templates:
_hook_file = os.path.join(hooks_path, filename)
_rhodecode_hook = _check_rhodecode_hook(_hook_file)
if _rhodecode_hook or force_create:
log.debug('writing %s hook file !', filename)
template = templates[filename]
try:
with open(_hook_file, 'wb') as f:
template = template.replace(
'_TMPL_', rhodecode.__version__)
template = template.replace('_ENV_', sys.executable)
f.write(template)
os.chmod(_hook_file, 0755)
except IOError:
log.exception('error writing hook file %s', filename)
else:
log.debug('skipping writing hook file')
def install_hooks(self, repo, repo_type):
if repo_type == 'git':
self.install_git_hook(repo)
elif repo_type == 'svn':
self.install_svn_hooks(repo)
def get_server_info(self, environ=None):
import platform
import rhodecode
import pkg_resources
from rhodecode.model.meta import Base as sql_base, Session
from sqlalchemy.engine import url
from rhodecode.lib.base import get_server_ip_addr, get_server_port
from rhodecode.lib.vcs.backends.git import discover_git_version
from rhodecode.model.gist import GIST_STORE_LOC
try:
# cygwin cannot have yet psutil support.
import psutil
except ImportError:
psutil = None
environ = environ or {}
_NA = 'NOT AVAILABLE'
_memory = _NA
_uptime = _NA
_boot_time = _NA
_cpu = _NA
_disk = dict(percent=0, used=0, total=0, error='')
_load = {'1_min': _NA, '5_min': _NA, '15_min': _NA}
model = VcsSettingsModel()
storage_path = model.get_repos_location()
gist_storage_path = os.path.join(storage_path, GIST_STORE_LOC)
archive_storage_path = rhodecode.CONFIG.get('archive_cache_dir', '')
search_index_storage_path = rhodecode.CONFIG.get('search.location', '')
if psutil:
# disk storage
try:
_disk = dict(psutil.disk_usage(storage_path)._asdict())
except Exception as e:
log.exception('Failed to fetch disk info')
_disk = {'percent': 0, 'used': 0, 'total': 0, 'error': str(e)}
# memory
_memory = dict(psutil.virtual_memory()._asdict())
_memory['percent2'] = psutil._common.usage_percent(
(_memory['total'] - _memory['free']),
_memory['total'], 1)
# load averages
if hasattr(psutil.os, 'getloadavg'):
_load = dict(zip(
['1_min', '5_min', '15_min'], psutil.os.getloadavg()))
_uptime = time.time() - psutil.boot_time()
_boot_time = psutil.boot_time()
_cpu = psutil.cpu_percent(0.5)
mods = dict([(p.project_name, p.version)
for p in pkg_resources.working_set])
def get_storage_size(storage_path):
sizes = []
for file_ in os.listdir(storage_path):
storage_file = os.path.join(storage_path, file_)
if os.path.isfile(storage_file):
try:
sizes.append(os.path.getsize(storage_file))
except OSError:
log.exception('Failed to get size of storage file %s',
storage_file)
pass
return sum(sizes)
# archive cache storage
_disk_archive = {'percent': 0, 'used': 0, 'total': 0}
try:
archive_storage_path_exists = os.path.isdir(
archive_storage_path)
if archive_storage_path and archive_storage_path_exists:
used = get_storage_size(archive_storage_path)
_disk_archive.update({
'used': used,
'total': used,
})
except Exception as e:
log.exception('failed to fetch archive cache storage')
_disk_archive['error'] = str(e)
# search index storage
_disk_index = {'percent': 0, 'used': 0, 'total': 0}
try:
search_index_storage_path_exists = os.path.isdir(
search_index_storage_path)
if search_index_storage_path_exists:
used = get_storage_size(search_index_storage_path)
_disk_index.update({
'percent': 100,
'used': used,
'total': used,
})
except Exception as e:
log.exception('failed to fetch search index storage')
_disk_index['error'] = str(e)
# gist storage
_disk_gist = {'percent': 0, 'used': 0, 'total': 0, 'items': 0}
try:
items_count = 0
used = 0
for root, dirs, files in os.walk(safe_str(gist_storage_path)):
if root == gist_storage_path:
items_count = len(dirs)
for f in files:
try:
used += os.path.getsize(os.path.join(root, f))
except OSError:
pass
_disk_gist.update({
'percent': 100,
'used': used,
'total': used,
'items': items_count
})
except Exception as e:
log.exception('failed to fetch gist storage items')
_disk_gist['error'] = str(e)
# GIT info
git_ver = discover_git_version()
# SVN info
# TODO: johbo: Add discover_svn_version to replace this code.
try:
import svn.core
svn_ver = svn.core.SVN_VERSION
except ImportError:
svn_ver = None
# DB stuff
db_info = url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
db_type = db_info.__to_string__()
try:
engine = sql_base.metadata.bind
db_server_info = engine.dialect._get_server_version_info(
Session.connection(bind=engine))
db_version = '%s %s' % (db_info.drivername,
'.'.join(map(str, db_server_info)))
except Exception:
log.exception('failed to fetch db version')
db_version = '%s %s' % (db_info.drivername, '?')
db_migrate = DbMigrateVersion.query().filter(
DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
db_migrate_version = db_migrate.version
info = {
'py_version': ' '.join(platform._sys_version()),
'py_path': sys.executable,
'py_modules': sorted(mods.items(), key=lambda k: k[0].lower()),
'platform': safe_unicode(platform.platform()),
'storage': storage_path,
'archive_storage': archive_storage_path,
'index_storage': search_index_storage_path,
'gist_storage': gist_storage_path,
'db_type': db_type,
'db_version': db_version,
'db_migrate_version': db_migrate_version,
'rhodecode_version': rhodecode.__version__,
'rhodecode_config_ini': rhodecode.CONFIG.get('__file__'),
'server_ip': '%s:%s' % (
get_server_ip_addr(environ, log_errors=False),
get_server_port(environ)
),
'server_id': rhodecode.CONFIG.get('instance_id'),
'git_version': safe_unicode(git_ver),
'hg_version': mods.get('mercurial'),
'svn_version': svn_ver,
'uptime': _uptime,
'boot_time': _boot_time,
'load': _load,
'cpu': _cpu,
'memory': _memory,
'disk': _disk,
'disk_archive': _disk_archive,
'disk_gist': _disk_gist,
'disk_index': _disk_index,
}
return info
def _check_rhodecode_hook(hook_path):
"""
Check if the hook was created by RhodeCode
"""
if not os.path.exists(hook_path):
return True
log.debug('hook exists, checking if it is from rhodecode')
hook_content = _read_hook(hook_path)
matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content)
if matches:
try:
version = matches.groups()[0]
log.debug('got %s, it is rhodecode', version)
return True
except Exception:
log.exception("Exception while reading the hook version.")
return False
def _read_hook(hook_path):
with open(hook_path, 'rb') as f:
content = f.read()
return content