scm.py
1098 lines
| 38.9 KiB
| text/x-python
|
PythonLexer
r1 | # -*- coding: utf-8 -*- | ||
# Copyright (C) 2010-2016 RhodeCode GmbH | |||
# | |||
# This program is free software: you can redistribute it and/or modify | |||
# it under the terms of the GNU Affero General Public License, version 3 | |||
# (only), as published by the Free Software Foundation. | |||
# | |||
# This program is distributed in the hope that it will be useful, | |||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
# GNU General Public License for more details. | |||
# | |||
# You should have received a copy of the GNU Affero General Public License | |||
# along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
# | |||
# This program is dual-licensed. If you wish to learn more about the | |||
# RhodeCode Enterprise Edition, including its added features, Support services, | |||
# and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
""" | |||
Scm model for RhodeCode | |||
""" | |||
import os.path | |||
import re | |||
import sys | |||
import time | |||
import traceback | |||
import logging | |||
import cStringIO | |||
import pkg_resources | |||
import pylons | |||
from pylons.i18n.translation import _ | |||
from sqlalchemy import func | |||
from zope.cachedescriptors.property import Lazy as LazyProperty | |||
import rhodecode | |||
from rhodecode.lib.vcs import get_backend | |||
from rhodecode.lib.vcs.exceptions import RepositoryError, NodeNotChangedError | |||
from rhodecode.lib.vcs.nodes import FileNode | |||
from rhodecode.lib.vcs.backends.base import EmptyCommit | |||
from rhodecode.lib import helpers as h | |||
from rhodecode.lib.auth import ( | |||
HasRepoPermissionAny, HasRepoGroupPermissionAny, | |||
HasUserGroupPermissionAny) | |||
from rhodecode.lib.exceptions import NonRelativePathError, IMCCommitError | |||
from rhodecode.lib import hooks_utils, caches | |||
from rhodecode.lib.utils import ( | |||
get_filesystem_repos, action_logger, make_db_config) | |||
from rhodecode.lib.utils2 import ( | |||
safe_str, safe_unicode, get_server_url, md5) | |||
from rhodecode.model import BaseModel | |||
from rhodecode.model.db import ( | |||
Repository, CacheKey, UserFollowing, UserLog, User, RepoGroup, | |||
PullRequest, DbMigrateVersion) | |||
from rhodecode.model.settings import VcsSettingsModel | |||
log = logging.getLogger(__name__) | |||
class UserTemp(object): | |||
def __init__(self, user_id): | |||
self.user_id = user_id | |||
def __repr__(self): | |||
return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id) | |||
class RepoTemp(object): | |||
def __init__(self, repo_id): | |||
self.repo_id = repo_id | |||
def __repr__(self): | |||
return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id) | |||
class SimpleCachedRepoList(object): | |||
""" | |||
Lighter version of of iteration of repos without the scm initialisation, | |||
and with cache usage | |||
""" | |||
def __init__(self, db_repo_list, repos_path, order_by=None, perm_set=None): | |||
self.db_repo_list = db_repo_list | |||
self.repos_path = repos_path | |||
self.order_by = order_by | |||
self.reversed = (order_by or '').startswith('-') | |||
if not perm_set: | |||
perm_set = ['repository.read', 'repository.write', | |||
'repository.admin'] | |||
self.perm_set = perm_set | |||
def __len__(self): | |||
return len(self.db_repo_list) | |||
def __repr__(self): | |||
return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) | |||
def __iter__(self): | |||
for dbr in self.db_repo_list: | |||
# check permission at this level | |||
has_perm = HasRepoPermissionAny(*self.perm_set)( | |||
dbr.repo_name, 'SimpleCachedRepoList check') | |||
if not has_perm: | |||
continue | |||
tmp_d = { | |||
'name': dbr.repo_name, | |||
'dbrepo': dbr.get_dict(), | |||
'dbrepo_fork': dbr.fork.get_dict() if dbr.fork else {} | |||
} | |||
yield tmp_d | |||
class _PermCheckIterator(object): | |||
def __init__( | |||
self, obj_list, obj_attr, perm_set, perm_checker, | |||
extra_kwargs=None): | |||
""" | |||
Creates iterator from given list of objects, additionally | |||
checking permission for them from perm_set var | |||
:param obj_list: list of db objects | |||
:param obj_attr: attribute of object to pass into perm_checker | |||
:param perm_set: list of permissions to check | |||
:param perm_checker: callable to check permissions against | |||
""" | |||
self.obj_list = obj_list | |||
self.obj_attr = obj_attr | |||
self.perm_set = perm_set | |||
self.perm_checker = perm_checker | |||
self.extra_kwargs = extra_kwargs or {} | |||
def __len__(self): | |||
return len(self.obj_list) | |||
def __repr__(self): | |||
return '<%s (%s)>' % (self.__class__.__name__, self.__len__()) | |||
def __iter__(self): | |||
checker = self.perm_checker(*self.perm_set) | |||
for db_obj in self.obj_list: | |||
# check permission at this level | |||
name = getattr(db_obj, self.obj_attr, None) | |||
if not checker(name, self.__class__.__name__, **self.extra_kwargs): | |||
continue | |||
yield db_obj | |||
class RepoList(_PermCheckIterator): | |||
def __init__(self, db_repo_list, perm_set=None, extra_kwargs=None): | |||
if not perm_set: | |||
perm_set = [ | |||
'repository.read', 'repository.write', 'repository.admin'] | |||
super(RepoList, self).__init__( | |||
obj_list=db_repo_list, | |||
obj_attr='repo_name', perm_set=perm_set, | |||
perm_checker=HasRepoPermissionAny, | |||
extra_kwargs=extra_kwargs) | |||
class RepoGroupList(_PermCheckIterator): | |||
def __init__(self, db_repo_group_list, perm_set=None, extra_kwargs=None): | |||
if not perm_set: | |||
perm_set = ['group.read', 'group.write', 'group.admin'] | |||
super(RepoGroupList, self).__init__( | |||
obj_list=db_repo_group_list, | |||
obj_attr='group_name', perm_set=perm_set, | |||
perm_checker=HasRepoGroupPermissionAny, | |||
extra_kwargs=extra_kwargs) | |||
class UserGroupList(_PermCheckIterator): | |||
def __init__(self, db_user_group_list, perm_set=None, extra_kwargs=None): | |||
if not perm_set: | |||
perm_set = ['usergroup.read', 'usergroup.write', 'usergroup.admin'] | |||
super(UserGroupList, self).__init__( | |||
obj_list=db_user_group_list, | |||
obj_attr='users_group_name', perm_set=perm_set, | |||
perm_checker=HasUserGroupPermissionAny, | |||
extra_kwargs=extra_kwargs) | |||
class ScmModel(BaseModel): | |||
""" | |||
Generic Scm Model | |||
""" | |||
@LazyProperty | |||
def repos_path(self): | |||
""" | |||
Gets the repositories root path from database | |||
""" | |||
settings_model = VcsSettingsModel(sa=self.sa) | |||
return settings_model.get_repos_location() | |||
def repo_scan(self, repos_path=None): | |||
""" | |||
Listing of repositories in given path. This path should not be a | |||
repository itself. Return a dictionary of repository objects | |||
:param repos_path: path to directory containing repositories | |||
""" | |||
if repos_path is None: | |||
repos_path = self.repos_path | |||
log.info('scanning for repositories in %s', repos_path) | |||
config = make_db_config() | |||
config.set('extensions', 'largefiles', '') | |||
repos = {} | |||
for name, path in get_filesystem_repos(repos_path, recursive=True): | |||
# name need to be decomposed and put back together using the / | |||
# since this is internal storage separator for rhodecode | |||
name = Repository.normalize_repo_name(name) | |||
try: | |||
if name in repos: | |||
raise RepositoryError('Duplicate repository name %s ' | |||
'found in %s' % (name, path)) | |||
elif path[0] in rhodecode.BACKENDS: | |||
klass = get_backend(path[0]) | |||
repos[name] = klass(path[1], config=config) | |||
except OSError: | |||
continue | |||
log.debug('found %s paths with repositories', len(repos)) | |||
return repos | |||
def get_repos(self, all_repos=None, sort_key=None): | |||
""" | |||
Get all repositories from db and for each repo create it's | |||
backend instance and fill that backed with information from database | |||
:param all_repos: list of repository names as strings | |||
give specific repositories list, good for filtering | |||
:param sort_key: initial sorting of repositories | |||
""" | |||
if all_repos is None: | |||
all_repos = self.sa.query(Repository)\ | |||
.filter(Repository.group_id == None)\ | |||
.order_by(func.lower(Repository.repo_name)).all() | |||
repo_iter = SimpleCachedRepoList( | |||
all_repos, repos_path=self.repos_path, order_by=sort_key) | |||
return repo_iter | |||
def get_repo_groups(self, all_groups=None): | |||
if all_groups is None: | |||
all_groups = RepoGroup.query()\ | |||
.filter(RepoGroup.group_parent_id == None).all() | |||
return [x for x in RepoGroupList(all_groups)] | |||
def mark_for_invalidation(self, repo_name, delete=False): | |||
""" | |||
Mark caches of this repo invalid in the database. `delete` flag | |||
removes the cache entries | |||
:param repo_name: the repo_name for which caches should be marked | |||
invalid, or deleted | |||
:param delete: delete the entry keys instead of setting bool | |||
flag on them | |||
""" | |||
CacheKey.set_invalidate(repo_name, delete=delete) | |||
repo = Repository.get_by_repo_name(repo_name) | |||
if repo: | |||
config = repo._config | |||
config.set('extensions', 'largefiles', '') | |||
r337 | repo.update_commit_cache(config=config, cs_cache=None) | ||
r1 | caches.clear_repo_caches(repo_name) | ||
def toggle_following_repo(self, follow_repo_id, user_id): | |||
f = self.sa.query(UserFollowing)\ | |||
.filter(UserFollowing.follows_repo_id == follow_repo_id)\ | |||
.filter(UserFollowing.user_id == user_id).scalar() | |||
if f is not None: | |||
try: | |||
self.sa.delete(f) | |||
action_logger(UserTemp(user_id), | |||
'stopped_following_repo', | |||
RepoTemp(follow_repo_id)) | |||
return | |||
except Exception: | |||
log.error(traceback.format_exc()) | |||
raise | |||
try: | |||
f = UserFollowing() | |||
f.user_id = user_id | |||
f.follows_repo_id = follow_repo_id | |||
self.sa.add(f) | |||
action_logger(UserTemp(user_id), | |||
'started_following_repo', | |||
RepoTemp(follow_repo_id)) | |||
except Exception: | |||
log.error(traceback.format_exc()) | |||
raise | |||
def toggle_following_user(self, follow_user_id, user_id): | |||
f = self.sa.query(UserFollowing)\ | |||
.filter(UserFollowing.follows_user_id == follow_user_id)\ | |||
.filter(UserFollowing.user_id == user_id).scalar() | |||
if f is not None: | |||
try: | |||
self.sa.delete(f) | |||
return | |||
except Exception: | |||
log.error(traceback.format_exc()) | |||
raise | |||
try: | |||
f = UserFollowing() | |||
f.user_id = user_id | |||
f.follows_user_id = follow_user_id | |||
self.sa.add(f) | |||
except Exception: | |||
log.error(traceback.format_exc()) | |||
raise | |||
def is_following_repo(self, repo_name, user_id, cache=False): | |||
r = self.sa.query(Repository)\ | |||
.filter(Repository.repo_name == repo_name).scalar() | |||
f = self.sa.query(UserFollowing)\ | |||
.filter(UserFollowing.follows_repository == r)\ | |||
.filter(UserFollowing.user_id == user_id).scalar() | |||
return f is not None | |||
def is_following_user(self, username, user_id, cache=False): | |||
u = User.get_by_username(username) | |||
f = self.sa.query(UserFollowing)\ | |||
.filter(UserFollowing.follows_user == u)\ | |||
.filter(UserFollowing.user_id == user_id).scalar() | |||
return f is not None | |||
def get_followers(self, repo): | |||
repo = self._get_repo(repo) | |||
return self.sa.query(UserFollowing)\ | |||
.filter(UserFollowing.follows_repository == repo).count() | |||
def get_forks(self, repo): | |||
repo = self._get_repo(repo) | |||
return self.sa.query(Repository)\ | |||
.filter(Repository.fork == repo).count() | |||
def get_pull_requests(self, repo): | |||
repo = self._get_repo(repo) | |||
return self.sa.query(PullRequest)\ | |||
.filter(PullRequest.target_repo == repo)\ | |||
.filter(PullRequest.status != PullRequest.STATUS_CLOSED).count() | |||
def mark_as_fork(self, repo, fork, user): | |||
repo = self._get_repo(repo) | |||
fork = self._get_repo(fork) | |||
if fork and repo.repo_id == fork.repo_id: | |||
raise Exception("Cannot set repository as fork of itself") | |||
if fork and repo.repo_type != fork.repo_type: | |||
raise RepositoryError( | |||
"Cannot set repository as fork of repository with other type") | |||
repo.fork = fork | |||
self.sa.add(repo) | |||
return repo | |||
def pull_changes(self, repo, username): | |||
dbrepo = self._get_repo(repo) | |||
clone_uri = dbrepo.clone_uri | |||
if not clone_uri: | |||
raise Exception("This repository doesn't have a clone uri") | |||
repo = dbrepo.scm_instance(cache=False) | |||
# TODO: marcink fix this an re-enable since we need common logic | |||
# for hg/git remove hooks so we don't trigger them on fetching | |||
# commits from remote | |||
repo.config.clear_section('hooks') | |||
repo_name = dbrepo.repo_name | |||
try: | |||
# TODO: we need to make sure those operations call proper hooks ! | |||
repo.pull(clone_uri) | |||
self.mark_for_invalidation(repo_name) | |||
except Exception: | |||
log.error(traceback.format_exc()) | |||
raise | |||
def commit_change(self, repo, repo_name, commit, user, author, message, | |||
content, f_path): | |||
""" | |||
Commits changes | |||
:param repo: SCM instance | |||
""" | |||
user = self._get_user(user) | |||
# decoding here will force that we have proper encoded values | |||
# in any other case this will throw exceptions and deny commit | |||
content = safe_str(content) | |||
path = safe_str(f_path) | |||
# message and author needs to be unicode | |||
# proper backend should then translate that into required type | |||
message = safe_unicode(message) | |||
author = safe_unicode(author) | |||
imc = repo.in_memory_commit | |||
imc.change(FileNode(path, content, mode=commit.get_file_mode(f_path))) | |||
try: | |||
# TODO: handle pre-push action ! | |||
tip = imc.commit( | |||
message=message, author=author, parents=[commit], | |||
branch=commit.branch) | |||
except Exception as e: | |||
log.error(traceback.format_exc()) | |||
raise IMCCommitError(str(e)) | |||
finally: | |||
# always clear caches, if commit fails we want fresh object also | |||
self.mark_for_invalidation(repo_name) | |||
# We trigger the post-push action | |||
hooks_utils.trigger_post_push_hook( | |||
username=user.username, action='push_local', repo_name=repo_name, | |||
repo_alias=repo.alias, commit_ids=[tip.raw_id]) | |||
return tip | |||
def _sanitize_path(self, f_path): | |||
r91 | if f_path.startswith('/') or f_path.startswith('./') or '../' in f_path: | ||
r1 | raise NonRelativePathError('%s is not an relative path' % f_path) | ||
if f_path: | |||
f_path = os.path.normpath(f_path) | |||
return f_path | |||
def get_dirnode_metadata(self, commit, dir_node): | |||
if not dir_node.is_dir(): | |||
return [] | |||
data = [] | |||
for node in dir_node: | |||
if not node.is_file(): | |||
# we skip file-nodes | |||
continue | |||
last_commit = node.last_commit | |||
last_commit_date = last_commit.date | |||
data.append({ | |||
'name': node.name, | |||
'size': h.format_byte_size_binary(node.size), | |||
'modified_at': h.format_date(last_commit_date), | |||
'modified_ts': last_commit_date.isoformat(), | |||
'revision': last_commit.revision, | |||
'short_id': last_commit.short_id, | |||
'message': h.escape(last_commit.message), | |||
'author': h.escape(last_commit.author), | |||
'user_profile': h.gravatar_with_user(last_commit.author), | |||
}) | |||
return data | |||
def get_nodes(self, repo_name, commit_id, root_path='/', flat=True, | |||
extended_info=False, content=False): | |||
""" | |||
recursive walk in root dir and return a set of all path in that dir | |||
based on repository walk function | |||
:param repo_name: name of repository | |||
:param commit_id: commit id for which to list nodes | |||
:param root_path: root path to list | |||
r501 | :param flat: return as a list, if False returns a dict with description | ||
r1 | |||
""" | |||
_files = list() | |||
_dirs = list() | |||
try: | |||
_repo = self._get_repo(repo_name) | |||
commit = _repo.scm_instance().get_commit(commit_id=commit_id) | |||
root_path = root_path.lstrip('/') | |||
for __, dirs, files in commit.walk(root_path): | |||
for f in files: | |||
_content = None | |||
_data = f.unicode_path | |||
r501 | |||
r1 | if not flat: | ||
_data = { | |||
"name": f.unicode_path, | |||
"type": "file", | |||
} | |||
if extended_info: | |||
_data.update({ | |||
r501 | "md5": f.md5, | ||
r1 | "binary": f.is_binary, | ||
"size": f.size, | |||
"extension": f.extension, | |||
"mimetype": f.mimetype, | |||
"lines": f.lines()[0] | |||
}) | |||
r501 | |||
r1 | if content: | ||
full_content = None | |||
if not f.is_binary: | |||
r501 | full_content = safe_str(f.content) | ||
r1 | |||
_data.update({ | |||
r501 | "content": full_content, | ||
r1 | }) | ||
_files.append(_data) | |||
for d in dirs: | |||
_data = d.unicode_path | |||
if not flat: | |||
_data = { | |||
"name": d.unicode_path, | |||
"type": "dir", | |||
} | |||
if extended_info: | |||
_data.update({ | |||
"md5": None, | |||
"binary": None, | |||
"size": None, | |||
"extension": None, | |||
}) | |||
if content: | |||
_data.update({ | |||
"content": None | |||
}) | |||
_dirs.append(_data) | |||
except RepositoryError: | |||
log.debug("Exception in get_nodes", exc_info=True) | |||
raise | |||
return _dirs, _files | |||
def create_nodes(self, user, repo, message, nodes, parent_commit=None, | |||
author=None, trigger_push_hook=True): | |||
""" | |||
Commits given multiple nodes into repo | |||
:param user: RhodeCode User object or user_id, the commiter | |||
:param repo: RhodeCode Repository object | |||
:param message: commit message | |||
:param nodes: mapping {filename:{'content':content},...} | |||
:param parent_commit: parent commit, can be empty than it's | |||
initial commit | |||
:param author: author of commit, cna be different that commiter | |||
only for git | |||
:param trigger_push_hook: trigger push hooks | |||
:returns: new commited commit | |||
""" | |||
user = self._get_user(user) | |||
scm_instance = repo.scm_instance(cache=False) | |||
processed_nodes = [] | |||
for f_path in nodes: | |||
f_path = self._sanitize_path(f_path) | |||
content = nodes[f_path]['content'] | |||
f_path = safe_str(f_path) | |||
# decoding here will force that we have proper encoded values | |||
# in any other case this will throw exceptions and deny commit | |||
if isinstance(content, (basestring,)): | |||
content = safe_str(content) | |||
elif isinstance(content, (file, cStringIO.OutputType,)): | |||
content = content.read() | |||
else: | |||
raise Exception('Content is of unrecognized type %s' % ( | |||
type(content) | |||
)) | |||
processed_nodes.append((f_path, content)) | |||
message = safe_unicode(message) | |||
commiter = user.full_contact | |||
author = safe_unicode(author) if author else commiter | |||
imc = scm_instance.in_memory_commit | |||
if not parent_commit: | |||
parent_commit = EmptyCommit(alias=scm_instance.alias) | |||
if isinstance(parent_commit, EmptyCommit): | |||
# EmptyCommit means we we're editing empty repository | |||
parents = None | |||
else: | |||
parents = [parent_commit] | |||
# add multiple nodes | |||
for path, content in processed_nodes: | |||
imc.add(FileNode(path, content=content)) | |||
# TODO: handle pre push scenario | |||
tip = imc.commit(message=message, | |||
author=author, | |||
parents=parents, | |||
branch=parent_commit.branch) | |||
self.mark_for_invalidation(repo.repo_name) | |||
if trigger_push_hook: | |||
hooks_utils.trigger_post_push_hook( | |||
username=user.username, action='push_local', | |||
repo_name=repo.repo_name, repo_alias=scm_instance.alias, | |||
commit_ids=[tip.raw_id]) | |||
return tip | |||
def update_nodes(self, user, repo, message, nodes, parent_commit=None, | |||
author=None, trigger_push_hook=True): | |||
user = self._get_user(user) | |||
scm_instance = repo.scm_instance(cache=False) | |||
message = safe_unicode(message) | |||
commiter = user.full_contact | |||
author = safe_unicode(author) if author else commiter | |||
imc = scm_instance.in_memory_commit | |||
if not parent_commit: | |||
parent_commit = EmptyCommit(alias=scm_instance.alias) | |||
if isinstance(parent_commit, EmptyCommit): | |||
# EmptyCommit means we we're editing empty repository | |||
parents = None | |||
else: | |||
parents = [parent_commit] | |||
# add multiple nodes | |||
for _filename, data in nodes.items(): | |||
# new filename, can be renamed from the old one, also sanitaze | |||
# the path for any hack around relative paths like ../../ etc. | |||
filename = self._sanitize_path(data['filename']) | |||
old_filename = self._sanitize_path(_filename) | |||
content = data['content'] | |||
filenode = FileNode(old_filename, content=content) | |||
op = data['op'] | |||
if op == 'add': | |||
imc.add(filenode) | |||
elif op == 'del': | |||
imc.remove(filenode) | |||
elif op == 'mod': | |||
if filename != old_filename: | |||
# TODO: handle renames more efficient, needs vcs lib | |||
# changes | |||
imc.remove(filenode) | |||
imc.add(FileNode(filename, content=content)) | |||
else: | |||
imc.change(filenode) | |||
try: | |||
# TODO: handle pre push scenario | |||
# commit changes | |||
tip = imc.commit(message=message, | |||
author=author, | |||
parents=parents, | |||
branch=parent_commit.branch) | |||
except NodeNotChangedError: | |||
raise | |||
except Exception as e: | |||
log.exception("Unexpected exception during call to imc.commit") | |||
raise IMCCommitError(str(e)) | |||
finally: | |||
# always clear caches, if commit fails we want fresh object also | |||
self.mark_for_invalidation(repo.repo_name) | |||
if trigger_push_hook: | |||
hooks_utils.trigger_post_push_hook( | |||
username=user.username, action='push_local', | |||
repo_name=repo.repo_name, repo_alias=scm_instance.alias, | |||
commit_ids=[tip.raw_id]) | |||
def delete_nodes(self, user, repo, message, nodes, parent_commit=None, | |||
author=None, trigger_push_hook=True): | |||
""" | |||
Deletes given multiple nodes into `repo` | |||
:param user: RhodeCode User object or user_id, the committer | |||
:param repo: RhodeCode Repository object | |||
:param message: commit message | |||
:param nodes: mapping {filename:{'content':content},...} | |||
:param parent_commit: parent commit, can be empty than it's initial | |||
commit | |||
:param author: author of commit, cna be different that commiter only | |||
for git | |||
:param trigger_push_hook: trigger push hooks | |||
:returns: new commit after deletion | |||
""" | |||
user = self._get_user(user) | |||
scm_instance = repo.scm_instance(cache=False) | |||
processed_nodes = [] | |||
for f_path in nodes: | |||
f_path = self._sanitize_path(f_path) | |||
# content can be empty but for compatabilty it allows same dicts | |||
# structure as add_nodes | |||
content = nodes[f_path].get('content') | |||
processed_nodes.append((f_path, content)) | |||
message = safe_unicode(message) | |||
commiter = user.full_contact | |||
author = safe_unicode(author) if author else commiter | |||
imc = scm_instance.in_memory_commit | |||
if not parent_commit: | |||
parent_commit = EmptyCommit(alias=scm_instance.alias) | |||
if isinstance(parent_commit, EmptyCommit): | |||
# EmptyCommit means we we're editing empty repository | |||
parents = None | |||
else: | |||
parents = [parent_commit] | |||
# add multiple nodes | |||
for path, content in processed_nodes: | |||
imc.remove(FileNode(path, content=content)) | |||
# TODO: handle pre push scenario | |||
tip = imc.commit(message=message, | |||
author=author, | |||
parents=parents, | |||
branch=parent_commit.branch) | |||
self.mark_for_invalidation(repo.repo_name) | |||
if trigger_push_hook: | |||
hooks_utils.trigger_post_push_hook( | |||
username=user.username, action='push_local', | |||
repo_name=repo.repo_name, repo_alias=scm_instance.alias, | |||
commit_ids=[tip.raw_id]) | |||
return tip | |||
def strip(self, repo, commit_id, branch): | |||
scm_instance = repo.scm_instance(cache=False) | |||
scm_instance.config.clear_section('hooks') | |||
scm_instance.strip(commit_id, branch) | |||
self.mark_for_invalidation(repo.repo_name) | |||
def get_unread_journal(self): | |||
return self.sa.query(UserLog).count() | |||
def get_repo_landing_revs(self, repo=None): | |||
""" | |||
Generates select option with tags branches and bookmarks (for hg only) | |||
grouped by type | |||
:param repo: | |||
""" | |||
hist_l = [] | |||
choices = [] | |||
repo = self._get_repo(repo) | |||
hist_l.append(['rev:tip', _('latest tip')]) | |||
choices.append('rev:tip') | |||
if not repo: | |||
return choices, hist_l | |||
repo = repo.scm_instance() | |||
branches_group = ( | |||
[(u'branch:%s' % safe_unicode(b), safe_unicode(b)) | |||
for b in repo.branches], | |||
_("Branches")) | |||
hist_l.append(branches_group) | |||
choices.extend([x[0] for x in branches_group[0]]) | |||
if repo.alias == 'hg': | |||
bookmarks_group = ( | |||
[(u'book:%s' % safe_unicode(b), safe_unicode(b)) | |||
for b in repo.bookmarks], | |||
_("Bookmarks")) | |||
hist_l.append(bookmarks_group) | |||
choices.extend([x[0] for x in bookmarks_group[0]]) | |||
tags_group = ( | |||
[(u'tag:%s' % safe_unicode(t), safe_unicode(t)) | |||
for t in repo.tags], | |||
_("Tags")) | |||
hist_l.append(tags_group) | |||
choices.extend([x[0] for x in tags_group[0]]) | |||
return choices, hist_l | |||
def install_git_hook(self, repo, force_create=False): | |||
""" | |||
Creates a rhodecode hook inside a git repository | |||
:param repo: Instance of VCS repo | |||
:param force_create: Create even if same name hook exists | |||
""" | |||
loc = os.path.join(repo.path, 'hooks') | |||
if not repo.bare: | |||
loc = os.path.join(repo.path, '.git', 'hooks') | |||
if not os.path.isdir(loc): | |||
os.makedirs(loc, mode=0777) | |||
tmpl_post = pkg_resources.resource_string( | |||
'rhodecode', '/'.join( | |||
('config', 'hook_templates', 'git_post_receive.py.tmpl'))) | |||
tmpl_pre = pkg_resources.resource_string( | |||
'rhodecode', '/'.join( | |||
('config', 'hook_templates', 'git_pre_receive.py.tmpl'))) | |||
for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]: | |||
_hook_file = os.path.join(loc, '%s-receive' % h_type) | |||
log.debug('Installing git hook in repo %s', repo) | |||
_rhodecode_hook = _check_rhodecode_hook(_hook_file) | |||
if _rhodecode_hook or force_create: | |||
log.debug('writing %s hook file !', h_type) | |||
try: | |||
with open(_hook_file, 'wb') as f: | |||
tmpl = tmpl.replace('_TMPL_', rhodecode.__version__) | |||
tmpl = tmpl.replace('_ENV_', sys.executable) | |||
f.write(tmpl) | |||
os.chmod(_hook_file, 0755) | |||
except IOError: | |||
log.exception('error writing hook file %s', _hook_file) | |||
else: | |||
log.debug('skipping writing hook file') | |||
def install_svn_hooks(self, repo, force_create=False): | |||
""" | |||
Creates rhodecode hooks inside a svn repository | |||
:param repo: Instance of VCS repo | |||
:param force_create: Create even if same name hook exists | |||
""" | |||
hooks_path = os.path.join(repo.path, 'hooks') | |||
if not os.path.isdir(hooks_path): | |||
os.makedirs(hooks_path) | |||
post_commit_tmpl = pkg_resources.resource_string( | |||
'rhodecode', '/'.join( | |||
('config', 'hook_templates', 'svn_post_commit_hook.py.tmpl'))) | |||
pre_commit_template = pkg_resources.resource_string( | |||
'rhodecode', '/'.join( | |||
('config', 'hook_templates', 'svn_pre_commit_hook.py.tmpl'))) | |||
templates = { | |||
'post-commit': post_commit_tmpl, | |||
'pre-commit': pre_commit_template | |||
} | |||
for filename in templates: | |||
_hook_file = os.path.join(hooks_path, filename) | |||
_rhodecode_hook = _check_rhodecode_hook(_hook_file) | |||
if _rhodecode_hook or force_create: | |||
log.debug('writing %s hook file !', filename) | |||
template = templates[filename] | |||
try: | |||
with open(_hook_file, 'wb') as f: | |||
template = template.replace( | |||
'_TMPL_', rhodecode.__version__) | |||
template = template.replace('_ENV_', sys.executable) | |||
f.write(template) | |||
os.chmod(_hook_file, 0755) | |||
except IOError: | |||
log.exception('error writing hook file %s', filename) | |||
else: | |||
log.debug('skipping writing hook file') | |||
def install_hooks(self, repo, repo_type): | |||
if repo_type == 'git': | |||
self.install_git_hook(repo) | |||
elif repo_type == 'svn': | |||
self.install_svn_hooks(repo) | |||
def get_server_info(self, environ=None): | |||
import platform | |||
import rhodecode | |||
import pkg_resources | |||
from rhodecode.model.meta import Base as sql_base, Session | |||
from sqlalchemy.engine import url | |||
from rhodecode.lib.base import get_server_ip_addr, get_server_port | |||
from rhodecode.lib.vcs.backends.git import discover_git_version | |||
from rhodecode.model.gist import GIST_STORE_LOC | |||
try: | |||
# cygwin cannot have yet psutil support. | |||
import psutil | |||
except ImportError: | |||
psutil = None | |||
environ = environ or {} | |||
_NA = 'NOT AVAILABLE' | |||
_memory = _NA | |||
_uptime = _NA | |||
_boot_time = _NA | |||
_cpu = _NA | |||
_disk = dict(percent=0, used=0, total=0, error='') | |||
_load = {'1_min': _NA, '5_min': _NA, '15_min': _NA} | |||
model = VcsSettingsModel() | |||
storage_path = model.get_repos_location() | |||
gist_storage_path = os.path.join(storage_path, GIST_STORE_LOC) | |||
archive_storage_path = rhodecode.CONFIG.get('archive_cache_dir', '') | |||
search_index_storage_path = rhodecode.CONFIG.get('search.location', '') | |||
if psutil: | |||
# disk storage | |||
try: | |||
_disk = dict(psutil.disk_usage(storage_path)._asdict()) | |||
except Exception as e: | |||
log.exception('Failed to fetch disk info') | |||
_disk = {'percent': 0, 'used': 0, 'total': 0, 'error': str(e)} | |||
# memory | |||
_memory = dict(psutil.virtual_memory()._asdict()) | |||
_memory['percent2'] = psutil._common.usage_percent( | |||
(_memory['total'] - _memory['free']), | |||
_memory['total'], 1) | |||
# load averages | |||
if hasattr(psutil.os, 'getloadavg'): | |||
_load = dict(zip( | |||
['1_min', '5_min', '15_min'], psutil.os.getloadavg())) | |||
_uptime = time.time() - psutil.boot_time() | |||
_boot_time = psutil.boot_time() | |||
_cpu = psutil.cpu_percent(0.5) | |||
mods = dict([(p.project_name, p.version) | |||
for p in pkg_resources.working_set]) | |||
def get_storage_size(storage_path): | |||
sizes = [] | |||
for file_ in os.listdir(storage_path): | |||
storage_file = os.path.join(storage_path, file_) | |||
if os.path.isfile(storage_file): | |||
try: | |||
sizes.append(os.path.getsize(storage_file)) | |||
except OSError: | |||
log.exception('Failed to get size of storage file %s', | |||
storage_file) | |||
pass | |||
return sum(sizes) | |||
# archive cache storage | |||
_disk_archive = {'percent': 0, 'used': 0, 'total': 0} | |||
try: | |||
archive_storage_path_exists = os.path.isdir( | |||
archive_storage_path) | |||
if archive_storage_path and archive_storage_path_exists: | |||
used = get_storage_size(archive_storage_path) | |||
_disk_archive.update({ | |||
'used': used, | |||
'total': used, | |||
}) | |||
except Exception as e: | |||
log.exception('failed to fetch archive cache storage') | |||
_disk_archive['error'] = str(e) | |||
# search index storage | |||
_disk_index = {'percent': 0, 'used': 0, 'total': 0} | |||
try: | |||
search_index_storage_path_exists = os.path.isdir( | |||
search_index_storage_path) | |||
if search_index_storage_path_exists: | |||
used = get_storage_size(search_index_storage_path) | |||
_disk_index.update({ | |||
'percent': 100, | |||
'used': used, | |||
'total': used, | |||
}) | |||
except Exception as e: | |||
log.exception('failed to fetch search index storage') | |||
_disk_index['error'] = str(e) | |||
# gist storage | |||
_disk_gist = {'percent': 0, 'used': 0, 'total': 0, 'items': 0} | |||
try: | |||
items_count = 0 | |||
used = 0 | |||
for root, dirs, files in os.walk(safe_str(gist_storage_path)): | |||
if root == gist_storage_path: | |||
items_count = len(dirs) | |||
for f in files: | |||
try: | |||
used += os.path.getsize(os.path.join(root, f)) | |||
except OSError: | |||
pass | |||
_disk_gist.update({ | |||
'percent': 100, | |||
'used': used, | |||
'total': used, | |||
'items': items_count | |||
}) | |||
except Exception as e: | |||
log.exception('failed to fetch gist storage items') | |||
_disk_gist['error'] = str(e) | |||
# GIT info | |||
git_ver = discover_git_version() | |||
# SVN info | |||
# TODO: johbo: Add discover_svn_version to replace this code. | |||
try: | |||
import svn.core | |||
svn_ver = svn.core.SVN_VERSION | |||
except ImportError: | |||
svn_ver = None | |||
# DB stuff | |||
db_info = url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url']) | |||
db_type = db_info.__to_string__() | |||
try: | |||
engine = sql_base.metadata.bind | |||
db_server_info = engine.dialect._get_server_version_info( | |||
Session.connection(bind=engine)) | |||
db_version = '%s %s' % (db_info.drivername, | |||
'.'.join(map(str, db_server_info))) | |||
except Exception: | |||
log.exception('failed to fetch db version') | |||
db_version = '%s %s' % (db_info.drivername, '?') | |||
db_migrate = DbMigrateVersion.query().filter( | |||
DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one() | |||
db_migrate_version = db_migrate.version | |||
info = { | |||
'py_version': ' '.join(platform._sys_version()), | |||
'py_path': sys.executable, | |||
'py_modules': sorted(mods.items(), key=lambda k: k[0].lower()), | |||
'platform': safe_unicode(platform.platform()), | |||
'storage': storage_path, | |||
'archive_storage': archive_storage_path, | |||
'index_storage': search_index_storage_path, | |||
'gist_storage': gist_storage_path, | |||
'db_type': db_type, | |||
'db_version': db_version, | |||
'db_migrate_version': db_migrate_version, | |||
'rhodecode_version': rhodecode.__version__, | |||
'rhodecode_config_ini': rhodecode.CONFIG.get('__file__'), | |||
'server_ip': '%s:%s' % ( | |||
get_server_ip_addr(environ, log_errors=False), | |||
get_server_port(environ) | |||
), | |||
'server_id': rhodecode.CONFIG.get('instance_id'), | |||
'git_version': safe_unicode(git_ver), | |||
'hg_version': mods.get('mercurial'), | |||
'svn_version': svn_ver, | |||
'uptime': _uptime, | |||
'boot_time': _boot_time, | |||
'load': _load, | |||
'cpu': _cpu, | |||
'memory': _memory, | |||
'disk': _disk, | |||
'disk_archive': _disk_archive, | |||
'disk_gist': _disk_gist, | |||
'disk_index': _disk_index, | |||
} | |||
return info | |||
def _check_rhodecode_hook(hook_path): | |||
""" | |||
Check if the hook was created by RhodeCode | |||
""" | |||
if not os.path.exists(hook_path): | |||
return True | |||
log.debug('hook exists, checking if it is from rhodecode') | |||
hook_content = _read_hook(hook_path) | |||
matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content) | |||
if matches: | |||
try: | |||
version = matches.groups()[0] | |||
log.debug('got %s, it is rhodecode', version) | |||
return True | |||
except Exception: | |||
log.exception("Exception while reading the hook version.") | |||
return False | |||
def _read_hook(hook_path): | |||
with open(hook_path, 'rb') as f: | |||
content = f.read() | |||
r501 | return content |