# -*- coding: utf-8 -*- # Copyright (C) 2010-2018 RhodeCode GmbH # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License, version 3 # (only), as published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ import os import re import shutil import time import logging import traceback import datetime from pyramid.threadlocal import get_current_request from zope.cachedescriptors.property import Lazy as LazyProperty from rhodecode import events from rhodecode.lib.auth import HasUserGroupPermissionAny from rhodecode.lib.caching_query import FromCache from rhodecode.lib.exceptions import AttachedForksError from rhodecode.lib.hooks_base import log_delete_repository from rhodecode.lib.user_log_filter import user_log_filter from rhodecode.lib.utils import make_db_config from rhodecode.lib.utils2 import ( safe_str, safe_unicode, remove_prefix, obfuscate_url_pw, get_current_rhodecode_user, safe_int, datetime_to_time, action_logger_generic) from rhodecode.lib.vcs.backends import get_backend from rhodecode.model import BaseModel from rhodecode.model.db import ( _hash_key, joinedload, or_, Repository, UserRepoToPerm, UserGroupRepoToPerm, UserRepoGroupToPerm, UserGroupRepoGroupToPerm, User, Permission, Statistics, UserGroup, RepoGroup, RepositoryField, UserLog) from rhodecode.model.settings import VcsSettingsModel log = logging.getLogger(__name__) class RepoModel(BaseModel): cls = Repository def _get_user_group(self, users_group): return self._get_instance(UserGroup, users_group, callback=UserGroup.get_by_group_name) def _get_repo_group(self, repo_group): return self._get_instance(RepoGroup, repo_group, callback=RepoGroup.get_by_group_name) def _create_default_perms(self, repository, private): # create default permission default = 'repository.read' def_user = User.get_default_user() for p in def_user.user_perms: if p.permission.permission_name.startswith('repository.'): default = p.permission.permission_name break default_perm = 'repository.none' if private else default repo_to_perm = UserRepoToPerm() repo_to_perm.permission = Permission.get_by_key(default_perm) repo_to_perm.repository = repository repo_to_perm.user_id = def_user.user_id return repo_to_perm @LazyProperty def repos_path(self): """ Gets the repositories root path from database """ settings_model = VcsSettingsModel(sa=self.sa) return settings_model.get_repos_location() def get(self, repo_id, cache=False): repo = self.sa.query(Repository) \ .filter(Repository.repo_id == repo_id) if cache: repo = repo.options( FromCache("sql_cache_short", "get_repo_%s" % repo_id)) return repo.scalar() def get_repo(self, repository): return self._get_repo(repository) def get_by_repo_name(self, repo_name, cache=False): repo = self.sa.query(Repository) \ .filter(Repository.repo_name == repo_name) if cache: name_key = _hash_key(repo_name) repo = repo.options( FromCache("sql_cache_short", "get_repo_%s" % name_key)) return repo.scalar() def _extract_id_from_repo_name(self, repo_name): if repo_name.startswith('/'): repo_name = repo_name.lstrip('/') by_id_match = re.match(r'^_(\d{1,})', repo_name) if by_id_match: return by_id_match.groups()[0] def get_repo_by_id(self, repo_name): """ Extracts repo_name by id from special urls. Example url is _11/repo_name :param repo_name: :return: repo object if matched else None """ try: _repo_id = self._extract_id_from_repo_name(repo_name) if _repo_id: return self.get(_repo_id) except Exception: log.exception('Failed to extract repo_name from URL') return None def get_repos_for_root(self, root, traverse=False): if traverse: like_expression = u'{}%'.format(safe_unicode(root)) repos = Repository.query().filter( Repository.repo_name.like(like_expression)).all() else: if root and not isinstance(root, RepoGroup): raise ValueError( 'Root must be an instance ' 'of RepoGroup, got:{} instead'.format(type(root))) repos = Repository.query().filter(Repository.group == root).all() return repos def get_url(self, repo, request=None, permalink=False): if not request: request = get_current_request() if not request: return if permalink: return request.route_url( 'repo_summary', repo_name='_{}'.format(safe_str(repo.repo_id))) else: return request.route_url( 'repo_summary', repo_name=safe_str(repo.repo_name)) def get_commit_url(self, repo, commit_id, request=None, permalink=False): if not request: request = get_current_request() if not request: return if permalink: return request.route_url( 'repo_commit', repo_name=safe_str(repo.repo_id), commit_id=commit_id) else: return request.route_url( 'repo_commit', repo_name=safe_str(repo.repo_name), commit_id=commit_id) def get_repo_log(self, repo, filter_term): repo_log = UserLog.query()\ .filter(or_(UserLog.repository_id == repo.repo_id, UserLog.repository_name == repo.repo_name))\ .options(joinedload(UserLog.user))\ .options(joinedload(UserLog.repository))\ .order_by(UserLog.action_date.desc()) repo_log = user_log_filter(repo_log, filter_term) return repo_log @classmethod def update_repoinfo(cls, repositories=None): if not repositories: repositories = Repository.getAll() for repo in repositories: repo.update_commit_cache() def get_repos_as_dict(self, repo_list=None, admin=False, super_user_actions=False): _render = get_current_request().get_partial_renderer( 'rhodecode:templates/data_table/_dt_elements.mako') c = _render.get_call_context() def quick_menu(repo_name): return _render('quick_menu', repo_name) def repo_lnk(name, rtype, rstate, private, fork_of): return _render('repo_name', name, rtype, rstate, private, fork_of, short_name=not admin, admin=False) def last_change(last_change): if admin and isinstance(last_change, datetime.datetime) and not last_change.tzinfo: last_change = last_change + datetime.timedelta(seconds= (datetime.datetime.now() - datetime.datetime.utcnow()).seconds) return _render("last_change", last_change) def rss_lnk(repo_name): return _render("rss", repo_name) def atom_lnk(repo_name): return _render("atom", repo_name) def last_rev(repo_name, cs_cache): return _render('revision', repo_name, cs_cache.get('revision'), cs_cache.get('raw_id'), cs_cache.get('author'), cs_cache.get('message'), cs_cache.get('date')) def desc(desc): return _render('repo_desc', desc, c.visual.stylify_metatags) def state(repo_state): return _render("repo_state", repo_state) def repo_actions(repo_name): return _render('repo_actions', repo_name, super_user_actions) def user_profile(username): return _render('user_profile', username) repos_data = [] for repo in repo_list: cs_cache = repo.changeset_cache row = { "menu": quick_menu(repo.repo_name), "name": repo_lnk(repo.repo_name, repo.repo_type, repo.repo_state, repo.private, repo.fork), "name_raw": repo.repo_name.lower(), "last_change": last_change(repo.last_db_change), "last_change_raw": datetime_to_time(repo.last_db_change), "last_changeset": last_rev(repo.repo_name, cs_cache), "last_changeset_raw": cs_cache.get('revision'), "desc": desc(repo.description_safe), "owner": user_profile(repo.user.username), "state": state(repo.repo_state), "rss": rss_lnk(repo.repo_name), "atom": atom_lnk(repo.repo_name), } if admin: row.update({ "action": repo_actions(repo.repo_name), }) repos_data.append(row) return repos_data def _get_defaults(self, repo_name): """ Gets information about repository, and returns a dict for usage in forms :param repo_name: """ repo_info = Repository.get_by_repo_name(repo_name) if repo_info is None: return None defaults = repo_info.get_dict() defaults['repo_name'] = repo_info.just_name groups = repo_info.groups_with_parents parent_group = groups[-1] if groups else None # we use -1 as this is how in HTML, we mark an empty group defaults['repo_group'] = getattr(parent_group, 'group_id', -1) keys_to_process = ( {'k': 'repo_type', 'strip': False}, {'k': 'repo_enable_downloads', 'strip': True}, {'k': 'repo_description', 'strip': True}, {'k': 'repo_enable_locking', 'strip': True}, {'k': 'repo_landing_rev', 'strip': True}, {'k': 'clone_uri', 'strip': False}, {'k': 'push_uri', 'strip': False}, {'k': 'repo_private', 'strip': True}, {'k': 'repo_enable_statistics', 'strip': True} ) for item in keys_to_process: attr = item['k'] if item['strip']: attr = remove_prefix(item['k'], 'repo_') val = defaults[attr] if item['k'] == 'repo_landing_rev': val = ':'.join(defaults[attr]) defaults[item['k']] = val if item['k'] == 'clone_uri': defaults['clone_uri_hidden'] = repo_info.clone_uri_hidden if item['k'] == 'push_uri': defaults['push_uri_hidden'] = repo_info.push_uri_hidden # fill owner if repo_info.user: defaults.update({'user': repo_info.user.username}) else: replacement_user = User.get_first_super_admin().username defaults.update({'user': replacement_user}) return defaults def update(self, repo, **kwargs): try: cur_repo = self._get_repo(repo) source_repo_name = cur_repo.repo_name if 'user' in kwargs: cur_repo.user = User.get_by_username(kwargs['user']) if 'repo_group' in kwargs: cur_repo.group = RepoGroup.get(kwargs['repo_group']) log.debug('Updating repo %s with params:%s', cur_repo, kwargs) update_keys = [ (1, 'repo_description'), (1, 'repo_landing_rev'), (1, 'repo_private'), (1, 'repo_enable_downloads'), (1, 'repo_enable_locking'), (1, 'repo_enable_statistics'), (0, 'clone_uri'), (0, 'push_uri'), (0, 'fork_id') ] for strip, k in update_keys: if k in kwargs: val = kwargs[k] if strip: k = remove_prefix(k, 'repo_') setattr(cur_repo, k, val) new_name = cur_repo.get_new_name(kwargs['repo_name']) cur_repo.repo_name = new_name # if private flag is set, reset default permission to NONE if kwargs.get('repo_private'): EMPTY_PERM = 'repository.none' RepoModel().grant_user_permission( repo=cur_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM ) # handle extra fields for field in filter(lambda k: k.startswith(RepositoryField.PREFIX), kwargs): k = RepositoryField.un_prefix_key(field) ex_field = RepositoryField.get_by_key_name( key=k, repo=cur_repo) if ex_field: ex_field.field_value = kwargs[field] self.sa.add(ex_field) cur_repo.updated_on = datetime.datetime.now() self.sa.add(cur_repo) if source_repo_name != new_name: # rename repository self._rename_filesystem_repo( old=source_repo_name, new=new_name) return cur_repo except Exception: log.error(traceback.format_exc()) raise def _create_repo(self, repo_name, repo_type, description, owner, private=False, clone_uri=None, repo_group=None, landing_rev='rev:tip', fork_of=None, copy_fork_permissions=False, enable_statistics=False, enable_locking=False, enable_downloads=False, copy_group_permissions=False, state=Repository.STATE_PENDING): """ Create repository inside database with PENDING state, this should be only executed by create() repo. With exception of importing existing repos """ from rhodecode.model.scm import ScmModel owner = self._get_user(owner) fork_of = self._get_repo(fork_of) repo_group = self._get_repo_group(safe_int(repo_group)) try: repo_name = safe_unicode(repo_name) description = safe_unicode(description) # repo name is just a name of repository # while repo_name_full is a full qualified name that is combined # with name and path of group repo_name_full = repo_name repo_name = repo_name.split(Repository.NAME_SEP)[-1] new_repo = Repository() new_repo.repo_state = state new_repo.enable_statistics = False new_repo.repo_name = repo_name_full new_repo.repo_type = repo_type new_repo.user = owner new_repo.group = repo_group new_repo.description = description or repo_name new_repo.private = private new_repo.clone_uri = clone_uri new_repo.landing_rev = landing_rev new_repo.enable_statistics = enable_statistics new_repo.enable_locking = enable_locking new_repo.enable_downloads = enable_downloads if repo_group: new_repo.enable_locking = repo_group.enable_locking if fork_of: parent_repo = fork_of new_repo.fork = parent_repo events.trigger(events.RepoPreCreateEvent(new_repo)) self.sa.add(new_repo) EMPTY_PERM = 'repository.none' if fork_of and copy_fork_permissions: repo = fork_of user_perms = UserRepoToPerm.query() \ .filter(UserRepoToPerm.repository == repo).all() group_perms = UserGroupRepoToPerm.query() \ .filter(UserGroupRepoToPerm.repository == repo).all() for perm in user_perms: UserRepoToPerm.create( perm.user, new_repo, perm.permission) for perm in group_perms: UserGroupRepoToPerm.create( perm.users_group, new_repo, perm.permission) # in case we copy permissions and also set this repo to private # override the default user permission to make it a private # repo if private: RepoModel(self.sa).grant_user_permission( repo=new_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM) elif repo_group and copy_group_permissions: user_perms = UserRepoGroupToPerm.query() \ .filter(UserRepoGroupToPerm.group == repo_group).all() group_perms = UserGroupRepoGroupToPerm.query() \ .filter(UserGroupRepoGroupToPerm.group == repo_group).all() for perm in user_perms: perm_name = perm.permission.permission_name.replace( 'group.', 'repository.') perm_obj = Permission.get_by_key(perm_name) UserRepoToPerm.create(perm.user, new_repo, perm_obj) for perm in group_perms: perm_name = perm.permission.permission_name.replace( 'group.', 'repository.') perm_obj = Permission.get_by_key(perm_name) UserGroupRepoToPerm.create( perm.users_group, new_repo, perm_obj) if private: RepoModel(self.sa).grant_user_permission( repo=new_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM) else: perm_obj = self._create_default_perms(new_repo, private) self.sa.add(perm_obj) # now automatically start following this repository as owner ScmModel(self.sa).toggle_following_repo(new_repo.repo_id, owner.user_id) # we need to flush here, in order to check if database won't # throw any exceptions, create filesystem dirs at the very end self.sa.flush() events.trigger(events.RepoCreateEvent(new_repo)) return new_repo except Exception: log.error(traceback.format_exc()) raise def create(self, form_data, cur_user): """ Create repository using celery tasks :param form_data: :param cur_user: """ from rhodecode.lib.celerylib import tasks, run_task return run_task(tasks.create_repo, form_data, cur_user) def update_permissions(self, repo, perm_additions=None, perm_updates=None, perm_deletions=None, check_perms=True, cur_user=None): if not perm_additions: perm_additions = [] if not perm_updates: perm_updates = [] if not perm_deletions: perm_deletions = [] req_perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin') changes = { 'added': [], 'updated': [], 'deleted': [] } # update permissions for member_id, perm, member_type in perm_updates: member_id = int(member_id) if member_type == 'user': member_name = User.get(member_id).username # this updates also current one if found self.grant_user_permission( repo=repo, user=member_id, perm=perm) else: # set for user group # check if we have permissions to alter this usergroup member_name = UserGroup.get(member_id).users_group_name if not check_perms or HasUserGroupPermissionAny( *req_perms)(member_name, user=cur_user): self.grant_user_group_permission( repo=repo, group_name=member_id, perm=perm) changes['updated'].append({'type': member_type, 'id': member_id, 'name': member_name, 'new_perm': perm}) # set new permissions for member_id, perm, member_type in perm_additions: member_id = int(member_id) if member_type == 'user': member_name = User.get(member_id).username self.grant_user_permission( repo=repo, user=member_id, perm=perm) else: # set for user group # check if we have permissions to alter this usergroup member_name = UserGroup.get(member_id).users_group_name if not check_perms or HasUserGroupPermissionAny( *req_perms)(member_name, user=cur_user): self.grant_user_group_permission( repo=repo, group_name=member_id, perm=perm) changes['added'].append({'type': member_type, 'id': member_id, 'name': member_name, 'new_perm': perm}) # delete permissions for member_id, perm, member_type in perm_deletions: member_id = int(member_id) if member_type == 'user': member_name = User.get(member_id).username self.revoke_user_permission(repo=repo, user=member_id) else: # set for user group # check if we have permissions to alter this usergroup member_name = UserGroup.get(member_id).users_group_name if not check_perms or HasUserGroupPermissionAny( *req_perms)(member_name, user=cur_user): self.revoke_user_group_permission( repo=repo, group_name=member_id) changes['deleted'].append({'type': member_type, 'id': member_id, 'name': member_name, 'new_perm': perm}) return changes def create_fork(self, form_data, cur_user): """ Simple wrapper into executing celery task for fork creation :param form_data: :param cur_user: """ from rhodecode.lib.celerylib import tasks, run_task return run_task(tasks.create_repo_fork, form_data, cur_user) def delete(self, repo, forks=None, fs_remove=True, cur_user=None): """ Delete given repository, forks parameter defines what do do with attached forks. Throws AttachedForksError if deleted repo has attached forks :param repo: :param forks: str 'delete' or 'detach' :param fs_remove: remove(archive) repo from filesystem """ if not cur_user: cur_user = getattr(get_current_rhodecode_user(), 'username', None) repo = self._get_repo(repo) if repo: if forks == 'detach': for r in repo.forks: r.fork = None self.sa.add(r) elif forks == 'delete': for r in repo.forks: self.delete(r, forks='delete') elif [f for f in repo.forks]: raise AttachedForksError() old_repo_dict = repo.get_dict() events.trigger(events.RepoPreDeleteEvent(repo)) try: self.sa.delete(repo) if fs_remove: self._delete_filesystem_repo(repo) else: log.debug('skipping removal from filesystem') old_repo_dict.update({ 'deleted_by': cur_user, 'deleted_on': time.time(), }) log_delete_repository(**old_repo_dict) events.trigger(events.RepoDeleteEvent(repo)) except Exception: log.error(traceback.format_exc()) raise def grant_user_permission(self, repo, user, perm): """ Grant permission for user on given repository, or update existing one if found :param repo: Instance of Repository, repository_id, or repository name :param user: Instance of User, user_id or username :param perm: Instance of Permission, or permission_name """ user = self._get_user(user) repo = self._get_repo(repo) permission = self._get_perm(perm) # check if we have that permission already obj = self.sa.query(UserRepoToPerm) \ .filter(UserRepoToPerm.user == user) \ .filter(UserRepoToPerm.repository == repo) \ .scalar() if obj is None: # create new ! obj = UserRepoToPerm() obj.repository = repo obj.user = user obj.permission = permission self.sa.add(obj) log.debug('Granted perm %s to %s on %s', perm, user, repo) action_logger_generic( 'granted permission: {} to user: {} on repo: {}'.format( perm, user, repo), namespace='security.repo') return obj def revoke_user_permission(self, repo, user): """ Revoke permission for user on given repository :param repo: Instance of Repository, repository_id, or repository name :param user: Instance of User, user_id or username """ user = self._get_user(user) repo = self._get_repo(repo) obj = self.sa.query(UserRepoToPerm) \ .filter(UserRepoToPerm.repository == repo) \ .filter(UserRepoToPerm.user == user) \ .scalar() if obj: self.sa.delete(obj) log.debug('Revoked perm on %s on %s', repo, user) action_logger_generic( 'revoked permission from user: {} on repo: {}'.format( user, repo), namespace='security.repo') def grant_user_group_permission(self, repo, group_name, perm): """ Grant permission for user group on given repository, or update existing one if found :param repo: Instance of Repository, repository_id, or repository name :param group_name: Instance of UserGroup, users_group_id, or user group name :param perm: Instance of Permission, or permission_name """ repo = self._get_repo(repo) group_name = self._get_user_group(group_name) permission = self._get_perm(perm) # check if we have that permission already obj = self.sa.query(UserGroupRepoToPerm) \ .filter(UserGroupRepoToPerm.users_group == group_name) \ .filter(UserGroupRepoToPerm.repository == repo) \ .scalar() if obj is None: # create new obj = UserGroupRepoToPerm() obj.repository = repo obj.users_group = group_name obj.permission = permission self.sa.add(obj) log.debug('Granted perm %s to %s on %s', perm, group_name, repo) action_logger_generic( 'granted permission: {} to usergroup: {} on repo: {}'.format( perm, group_name, repo), namespace='security.repo') return obj def revoke_user_group_permission(self, repo, group_name): """ Revoke permission for user group on given repository :param repo: Instance of Repository, repository_id, or repository name :param group_name: Instance of UserGroup, users_group_id, or user group name """ repo = self._get_repo(repo) group_name = self._get_user_group(group_name) obj = self.sa.query(UserGroupRepoToPerm) \ .filter(UserGroupRepoToPerm.repository == repo) \ .filter(UserGroupRepoToPerm.users_group == group_name) \ .scalar() if obj: self.sa.delete(obj) log.debug('Revoked perm to %s on %s', repo, group_name) action_logger_generic( 'revoked permission from usergroup: {} on repo: {}'.format( group_name, repo), namespace='security.repo') def delete_stats(self, repo_name): """ removes stats for given repo :param repo_name: """ repo = self._get_repo(repo_name) try: obj = self.sa.query(Statistics) \ .filter(Statistics.repository == repo).scalar() if obj: self.sa.delete(obj) except Exception: log.error(traceback.format_exc()) raise def add_repo_field(self, repo_name, field_key, field_label, field_value='', field_type='str', field_desc=''): repo = self._get_repo(repo_name) new_field = RepositoryField() new_field.repository = repo new_field.field_key = field_key new_field.field_type = field_type # python type new_field.field_value = field_value new_field.field_desc = field_desc new_field.field_label = field_label self.sa.add(new_field) return new_field def delete_repo_field(self, repo_name, field_key): repo = self._get_repo(repo_name) field = RepositoryField.get_by_key_name(field_key, repo) if field: self.sa.delete(field) def _create_filesystem_repo(self, repo_name, repo_type, repo_group, clone_uri=None, repo_store_location=None, use_global_config=False): """ makes repository on filesystem. It's group aware means it'll create a repository within a group, and alter the paths accordingly of group location :param repo_name: :param alias: :param parent: :param clone_uri: :param repo_store_location: """ from rhodecode.lib.utils import is_valid_repo, is_valid_repo_group from rhodecode.model.scm import ScmModel if Repository.NAME_SEP in repo_name: raise ValueError( 'repo_name must not contain groups got `%s`' % repo_name) if isinstance(repo_group, RepoGroup): new_parent_path = os.sep.join(repo_group.full_path_splitted) else: new_parent_path = repo_group or '' if repo_store_location: _paths = [repo_store_location] else: _paths = [self.repos_path, new_parent_path, repo_name] # we need to make it str for mercurial repo_path = os.path.join(*map(lambda x: safe_str(x), _paths)) # check if this path is not a repository if is_valid_repo(repo_path, self.repos_path): raise Exception('This path %s is a valid repository' % repo_path) # check if this path is a group if is_valid_repo_group(repo_path, self.repos_path): raise Exception('This path %s is a valid group' % repo_path) log.info('creating repo %s in %s from url: `%s`', repo_name, safe_unicode(repo_path), obfuscate_url_pw(clone_uri)) backend = get_backend(repo_type) config_repo = None if use_global_config else repo_name if config_repo and new_parent_path: config_repo = Repository.NAME_SEP.join( (new_parent_path, config_repo)) config = make_db_config(clear_session=False, repo=config_repo) config.set('extensions', 'largefiles', '') # patch and reset hooks section of UI config to not run any # hooks on creating remote repo config.clear_section('hooks') # TODO: johbo: Unify this, hardcoded "bare=True" does not look nice if repo_type == 'git': repo = backend( repo_path, config=config, create=True, src_url=clone_uri, bare=True) else: repo = backend( repo_path, config=config, create=True, src_url=clone_uri) repo.install_hooks() log.debug('Created repo %s with %s backend', safe_unicode(repo_name), safe_unicode(repo_type)) return repo def _rename_filesystem_repo(self, old, new): """ renames repository on filesystem :param old: old name :param new: new name """ log.info('renaming repo from %s to %s', old, new) old_path = os.path.join(self.repos_path, old) new_path = os.path.join(self.repos_path, new) if os.path.isdir(new_path): raise Exception( 'Was trying to rename to already existing dir %s' % new_path ) shutil.move(old_path, new_path) def _delete_filesystem_repo(self, repo): """ removes repo from filesystem, the removal is acctually made by added rm__ prefix into dir, and rename internat .hg/.git dirs so this repository is no longer valid for rhodecode, can be undeleted later on by reverting the renames on this repository :param repo: repo object """ rm_path = os.path.join(self.repos_path, repo.repo_name) repo_group = repo.group log.info("Removing repository %s", rm_path) # disable hg/git internal that it doesn't get detected as repo alias = repo.repo_type config = make_db_config(clear_session=False) config.set('extensions', 'largefiles', '') bare = getattr(repo.scm_instance(config=config), 'bare', False) # skip this for bare git repos if not bare: # disable VCS repo vcs_path = os.path.join(rm_path, '.%s' % alias) if os.path.exists(vcs_path): shutil.move(vcs_path, os.path.join(rm_path, 'rm__.%s' % alias)) _now = datetime.datetime.now() _ms = str(_now.microsecond).rjust(6, '0') _d = 'rm__%s__%s' % (_now.strftime('%Y%m%d_%H%M%S_' + _ms), repo.just_name) if repo_group: # if repository is in group, prefix the removal path with the group args = repo_group.full_path_splitted + [_d] _d = os.path.join(*args) if os.path.isdir(rm_path): shutil.move(rm_path, os.path.join(self.repos_path, _d)) # finally cleanup diff-cache if it exists cached_diffs_dir = repo.cached_diffs_dir if os.path.isdir(cached_diffs_dir): shutil.rmtree(cached_diffs_dir) class ReadmeFinder: """ Utility which knows how to find a readme for a specific commit. The main idea is that this is a configurable algorithm. When creating an instance you can define parameters, currently only the `default_renderer`. Based on this configuration the method :meth:`search` behaves slightly different. """ readme_re = re.compile(r'^readme(\.[^\.]+)?$', re.IGNORECASE) path_re = re.compile(r'^docs?', re.IGNORECASE) default_priorities = { None: 0, '.text': 2, '.txt': 3, '.rst': 1, '.rest': 2, '.md': 1, '.mkdn': 2, '.mdown': 3, '.markdown': 4, } path_priority = { 'doc': 0, 'docs': 1, } FALLBACK_PRIORITY = 99 RENDERER_TO_EXTENSION = { 'rst': ['.rst', '.rest'], 'markdown': ['.md', 'mkdn', '.mdown', '.markdown'], } def __init__(self, default_renderer=None): self._default_renderer = default_renderer self._renderer_extensions = self.RENDERER_TO_EXTENSION.get( default_renderer, []) def search(self, commit, path='/'): """ Find a readme in the given `commit`. """ nodes = commit.get_nodes(path) matches = self._match_readmes(nodes) matches = self._sort_according_to_priority(matches) if matches: return matches[0].node paths = self._match_paths(nodes) paths = self._sort_paths_according_to_priority(paths) for path in paths: match = self.search(commit, path=path) if match: return match return None def _match_readmes(self, nodes): for node in nodes: if not node.is_file(): continue path = node.path.rsplit('/', 1)[-1] match = self.readme_re.match(path) if match: extension = match.group(1) yield ReadmeMatch(node, match, self._priority(extension)) def _match_paths(self, nodes): for node in nodes: if not node.is_dir(): continue match = self.path_re.match(node.path) if match: yield node.path def _priority(self, extension): renderer_priority = ( 0 if extension in self._renderer_extensions else 1) extension_priority = self.default_priorities.get( extension, self.FALLBACK_PRIORITY) return (renderer_priority, extension_priority) def _sort_according_to_priority(self, matches): def priority_and_path(match): return (match.priority, match.path) return sorted(matches, key=priority_and_path) def _sort_paths_according_to_priority(self, paths): def priority_and_path(path): return (self.path_priority.get(path, self.FALLBACK_PRIORITY), path) return sorted(paths, key=priority_and_path) class ReadmeMatch: def __init__(self, node, match, priority): self.node = node self._match = match self.priority = priority @property def path(self): return self.node.path def __repr__(self): return '