bugfix: santize wouldnt allow files starting with . - fixes #3936

rhodecode/model/scm.py

@@ -1,1105 +1,1105 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Scm model for RhodeCode
 """
 
 import os.path
 import re
 import sys
 import time
 import traceback
 import logging
 import cStringIO
 import pkg_resources
 
 import pylons
 from pylons.i18n.translation import _
 from sqlalchemy import func
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
 from rhodecode.lib.vcs import get_backend
 from rhodecode.lib.vcs.exceptions import RepositoryError, NodeNotChangedError
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib import helpers as h
 
 from rhodecode.lib.auth import (
     HasRepoPermissionAny, HasRepoGroupPermissionAny,
     HasUserGroupPermissionAny)
 from rhodecode.lib.exceptions import NonRelativePathError, IMCCommitError
 from rhodecode.lib import hooks_utils, caches
 from rhodecode.lib.utils import (
     get_filesystem_repos, action_logger, make_db_config)
 from rhodecode.lib.utils2 import (
     safe_str, safe_unicode, get_server_url, md5)
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     Repository, CacheKey, UserFollowing, UserLog, User, RepoGroup,
     PullRequest, DbMigrateVersion)
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 class UserTemp(object):
     def __init__(self, user_id):
         self.user_id = user_id
 
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
 
 
 class RepoTemp(object):
     def __init__(self, repo_id):
         self.repo_id = repo_id
 
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
 
 
 class SimpleCachedRepoList(object):
     """
     Lighter version of of iteration of repos without the scm initialisation,
     and with cache usage
     """
     def __init__(self, db_repo_list, repos_path, order_by=None, perm_set=None):
         self.db_repo_list = db_repo_list
         self.repos_path = repos_path
         self.order_by = order_by
         self.reversed = (order_by or '').startswith('-')
         if not perm_set:
             perm_set = ['repository.read', 'repository.write',
                         'repository.admin']
         self.perm_set = perm_set
 
     def __len__(self):
         return len(self.db_repo_list)
 
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
 
     def __iter__(self):
         for dbr in self.db_repo_list:
             # check permission at this level
             has_perm = HasRepoPermissionAny(*self.perm_set)(
                 dbr.repo_name, 'SimpleCachedRepoList check')
             if not has_perm:
                 continue
 
             tmp_d = {
                 'name': dbr.repo_name,
                 'dbrepo': dbr.get_dict(),
                 'dbrepo_fork': dbr.fork.get_dict() if dbr.fork else {}
             }
             yield tmp_d
 
 
 class _PermCheckIterator(object):
 
     def __init__(
             self, obj_list, obj_attr, perm_set, perm_checker,
             extra_kwargs=None):
         """
         Creates iterator from given list of objects, additionally
         checking permission for them from perm_set var
 
         :param obj_list: list of db objects
         :param obj_attr: attribute of object to pass into perm_checker
         :param perm_set: list of permissions to check
         :param perm_checker: callable to check permissions against
         """
         self.obj_list = obj_list
         self.obj_attr = obj_attr
         self.perm_set = perm_set
         self.perm_checker = perm_checker
         self.extra_kwargs = extra_kwargs or {}
 
     def __len__(self):
         return len(self.obj_list)
 
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
 
     def __iter__(self):
         checker = self.perm_checker(*self.perm_set)
         for db_obj in self.obj_list:
             # check permission at this level
             name = getattr(db_obj, self.obj_attr, None)
             if not checker(name, self.__class__.__name__, **self.extra_kwargs):
                 continue
 
             yield db_obj
 
 
 class RepoList(_PermCheckIterator):
 
     def __init__(self, db_repo_list, perm_set=None, extra_kwargs=None):
         if not perm_set:
             perm_set = [
                 'repository.read', 'repository.write', 'repository.admin']
 
         super(RepoList, self).__init__(
             obj_list=db_repo_list,
             obj_attr='repo_name', perm_set=perm_set,
             perm_checker=HasRepoPermissionAny,
             extra_kwargs=extra_kwargs)
 
 
 class RepoGroupList(_PermCheckIterator):
 
     def __init__(self, db_repo_group_list, perm_set=None, extra_kwargs=None):
         if not perm_set:
             perm_set = ['group.read', 'group.write', 'group.admin']
 
         super(RepoGroupList, self).__init__(
             obj_list=db_repo_group_list,
             obj_attr='group_name', perm_set=perm_set,
             perm_checker=HasRepoGroupPermissionAny,
             extra_kwargs=extra_kwargs)
 
 
 class UserGroupList(_PermCheckIterator):
 
     def __init__(self, db_user_group_list, perm_set=None, extra_kwargs=None):
         if not perm_set:
             perm_set = ['usergroup.read', 'usergroup.write', 'usergroup.admin']
 
         super(UserGroupList, self).__init__(
             obj_list=db_user_group_list,
             obj_attr='users_group_name', perm_set=perm_set,
             perm_checker=HasUserGroupPermissionAny,
             extra_kwargs=extra_kwargs)
 
 
 class ScmModel(BaseModel):
     """
     Generic Scm Model
     """
 
     @LazyProperty
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
 
         settings_model = VcsSettingsModel(sa=self.sa)
         return settings_model.get_repos_location()
 
     def repo_scan(self, repos_path=None):
         """
         Listing of repositories in given path. This path should not be a
         repository itself. Return a dictionary of repository objects
 
         :param repos_path: path to directory containing repositories
         """
 
         if repos_path is None:
             repos_path = self.repos_path
 
         log.info('scanning for repositories in %s', repos_path)
 
         config = make_db_config()
         config.set('extensions', 'largefiles', '')
         repos = {}
 
         for name, path in get_filesystem_repos(repos_path, recursive=True):
             # name need to be decomposed and put back together using the /
             # since this is internal storage separator for rhodecode
             name = Repository.normalize_repo_name(name)
 
             try:
                 if name in repos:
                     raise RepositoryError('Duplicate repository name %s '
                                           'found in %s' % (name, path))
                 elif path[0] in rhodecode.BACKENDS:
                     klass = get_backend(path[0])
                     repos[name] = klass(path[1], config=config)
             except OSError:
                 continue
         log.debug('found %s paths with repositories', len(repos))
         return repos
 
     def get_repos(self, all_repos=None, sort_key=None):
         """
         Get all repositories from db and for each repo create it's
         backend instance and fill that backed with information from database
 
         :param all_repos: list of repository names as strings
             give specific repositories list, good for filtering
 
         :param sort_key: initial sorting of repositories
         """
         if all_repos is None:
             all_repos = self.sa.query(Repository)\
                 .filter(Repository.group_id == None)\
                 .order_by(func.lower(Repository.repo_name)).all()
         repo_iter = SimpleCachedRepoList(
             all_repos, repos_path=self.repos_path, order_by=sort_key)
         return repo_iter
 
     def get_repo_groups(self, all_groups=None):
         if all_groups is None:
             all_groups = RepoGroup.query()\
                 .filter(RepoGroup.group_parent_id == None).all()
         return [x for x in RepoGroupList(all_groups)]
 
     def mark_for_invalidation(self, repo_name, delete=False):
         """
         Mark caches of this repo invalid in the database. `delete` flag
         removes the cache entries
 
         :param repo_name: the repo_name for which caches should be marked
             invalid, or deleted
         :param delete: delete the entry keys instead of setting bool
             flag on them
         """
         CacheKey.set_invalidate(repo_name, delete=delete)
         repo = Repository.get_by_repo_name(repo_name)
 
         if repo:
             config = repo._config
             config.set('extensions', 'largefiles', '')
             cs_cache = None
             if delete:
                 # if we do a hard clear, reset last-commit to Empty
                 cs_cache = EmptyCommit()
             repo.update_commit_cache(config=config, cs_cache=cs_cache)
             caches.clear_repo_caches(repo_name)
 
     def toggle_following_repo(self, follow_repo_id, user_id):
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repo_id == follow_repo_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 self.sa.delete(f)
                 action_logger(UserTemp(user_id),
                               'stopped_following_repo',
                               RepoTemp(follow_repo_id))
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_repo_id = follow_repo_id
             self.sa.add(f)
 
             action_logger(UserTemp(user_id),
                           'started_following_repo',
                           RepoTemp(follow_repo_id))
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def toggle_following_user(self, follow_user_id, user_id):
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user_id == follow_user_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 self.sa.delete(f)
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_user_id = follow_user_id
             self.sa.add(f)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def is_following_repo(self, repo_name, user_id, cache=False):
         r = self.sa.query(Repository)\
             .filter(Repository.repo_name == repo_name).scalar()
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repository == r)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def is_following_user(self, username, user_id, cache=False):
         u = User.get_by_username(username)
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user == u)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def get_followers(self, repo):
         repo = self._get_repo(repo)
 
         return self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repository == repo).count()
 
     def get_forks(self, repo):
         repo = self._get_repo(repo)
         return self.sa.query(Repository)\
             .filter(Repository.fork == repo).count()
 
     def get_pull_requests(self, repo):
         repo = self._get_repo(repo)
         return self.sa.query(PullRequest)\
             .filter(PullRequest.target_repo == repo)\
             .filter(PullRequest.status != PullRequest.STATUS_CLOSED).count()
 
     def mark_as_fork(self, repo, fork, user):
         repo = self._get_repo(repo)
         fork = self._get_repo(fork)
         if fork and repo.repo_id == fork.repo_id:
             raise Exception("Cannot set repository as fork of itself")
 
         if fork and repo.repo_type != fork.repo_type:
             raise RepositoryError(
                 "Cannot set repository as fork of repository with other type")
 
         repo.fork = fork
         self.sa.add(repo)
         return repo
 
     def pull_changes(self, repo, username):
         dbrepo = self._get_repo(repo)
         clone_uri = dbrepo.clone_uri
         if not clone_uri:
             raise Exception("This repository doesn't have a clone uri")
 
         repo = dbrepo.scm_instance(cache=False)
         # TODO: marcink fix this an re-enable since we need common logic
         # for hg/git remove hooks so we don't trigger them on fetching
         # commits from remote
         repo.config.clear_section('hooks')
 
         repo_name = dbrepo.repo_name
         try:
             # TODO: we need to make sure those operations call proper hooks !
             repo.pull(clone_uri)
 
             self.mark_for_invalidation(repo_name)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def commit_change(self, repo, repo_name, commit, user, author, message,
                       content, f_path):
         """
         Commits changes
 
         :param repo: SCM instance
 
         """
         user = self._get_user(user)
 
         # decoding here will force that we have proper encoded values
         # in any other case this will throw exceptions and deny commit
         content = safe_str(content)
         path = safe_str(f_path)
         # message and author needs to be unicode
         # proper backend should then translate that into required type
         message = safe_unicode(message)
         author = safe_unicode(author)
         imc = repo.in_memory_commit
         imc.change(FileNode(path, content, mode=commit.get_file_mode(f_path)))
         try:
             # TODO: handle pre-push action !
             tip = imc.commit(
                 message=message, author=author, parents=[commit],
                 branch=commit.branch)
         except Exception as e:
             log.error(traceback.format_exc())
             raise IMCCommitError(str(e))
         finally:
             # always clear caches, if commit fails we want fresh object also
             self.mark_for_invalidation(repo_name)
 
         # We trigger the post-push action
         hooks_utils.trigger_post_push_hook(
             username=user.username, action='push_local', repo_name=repo_name,
             repo_alias=repo.alias, commit_ids=[tip.raw_id])
         return tip
 
     def _sanitize_path(self, f_path):
-        if f_path.startswith('/') or f_path.startswith('.') or '../' in f_path:
+        if f_path.startswith('/') or f_path.startswith('./') or '../' in f_path:
             raise NonRelativePathError('%s is not an relative path' % f_path)
         if f_path:
             f_path = os.path.normpath(f_path)
         return f_path
 
     def get_dirnode_metadata(self, commit, dir_node):
         if not dir_node.is_dir():
             return []
 
         data = []
         for node in dir_node:
             if not node.is_file():
                 # we skip file-nodes
                 continue
 
             last_commit = node.last_commit
             last_commit_date = last_commit.date
             data.append({
                 'name': node.name,
                 'size': h.format_byte_size_binary(node.size),
                 'modified_at': h.format_date(last_commit_date),
                 'modified_ts': last_commit_date.isoformat(),
                 'revision': last_commit.revision,
                 'short_id': last_commit.short_id,
                 'message': h.escape(last_commit.message),
                 'author': h.escape(last_commit.author),
                 'user_profile': h.gravatar_with_user(last_commit.author),
             })
 
         return data
 
     def get_nodes(self, repo_name, commit_id, root_path='/', flat=True,
                   extended_info=False, content=False):
         """
         recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
 
         :param repo_name: name of repository
         :param commit_id: commit id for which to list nodes
         :param root_path: root path to list
         :param flat: return as a list, if False returns a dict with decription
 
         """
         _files = list()
         _dirs = list()
         try:
             _repo = self._get_repo(repo_name)
             commit = _repo.scm_instance().get_commit(commit_id=commit_id)
             root_path = root_path.lstrip('/')
             for __, dirs, files in commit.walk(root_path):
                 for f in files:
                     _content = None
                     _data = f.unicode_path
                     if not flat:
                         _data = {
                             "name": f.unicode_path,
                             "type": "file",
                             }
                         if extended_info:
                             _content = safe_str(f.content)
                             _data.update({
                                 "md5": md5(_content),
                                 "binary": f.is_binary,
                                 "size": f.size,
                                 "extension": f.extension,
 
                                 "mimetype": f.mimetype,
                                 "lines": f.lines()[0]
                             })
                         if content:
                             full_content = None
                             if not f.is_binary:
                                 # in case we loaded the _content already
                                 # re-use it, or load from f[ile]
                                 full_content = _content or safe_str(f.content)
 
                             _data.update({
                                 "content": full_content
                             })
                     _files.append(_data)
                 for d in dirs:
                     _data = d.unicode_path
                     if not flat:
                         _data = {
                             "name": d.unicode_path,
                             "type": "dir",
                             }
                     if extended_info:
                         _data.update({
                             "md5": None,
                             "binary": None,
                             "size": None,
                             "extension": None,
                         })
                     if content:
                         _data.update({
                             "content": None
                         })
                     _dirs.append(_data)
         except RepositoryError:
             log.debug("Exception in get_nodes", exc_info=True)
             raise
 
         return _dirs, _files
 
     def create_nodes(self, user, repo, message, nodes, parent_commit=None,
                      author=None, trigger_push_hook=True):
         """
         Commits given multiple nodes into repo
 
         :param user: RhodeCode User object or user_id, the commiter
         :param repo: RhodeCode Repository object
         :param message: commit message
         :param nodes: mapping {filename:{'content':content},...}
         :param parent_commit: parent commit, can be empty than it's
            initial commit
         :param author: author of commit, cna be different that commiter
            only for git
         :param trigger_push_hook: trigger push hooks
 
         :returns: new commited commit
         """
 
         user = self._get_user(user)
         scm_instance = repo.scm_instance(cache=False)
 
         processed_nodes = []
         for f_path in nodes:
             f_path = self._sanitize_path(f_path)
             content = nodes[f_path]['content']
             f_path = safe_str(f_path)
             # decoding here will force that we have proper encoded values
             # in any other case this will throw exceptions and deny commit
             if isinstance(content, (basestring,)):
                 content = safe_str(content)
             elif isinstance(content, (file, cStringIO.OutputType,)):
                 content = content.read()
             else:
                 raise Exception('Content is of unrecognized type %s' % (
                     type(content)
                 ))
             processed_nodes.append((f_path, content))
 
         message = safe_unicode(message)
         commiter = user.full_contact
         author = safe_unicode(author) if author else commiter
 
         imc = scm_instance.in_memory_commit
 
         if not parent_commit:
             parent_commit = EmptyCommit(alias=scm_instance.alias)
 
         if isinstance(parent_commit, EmptyCommit):
             # EmptyCommit means we we're editing empty repository
             parents = None
         else:
             parents = [parent_commit]
         # add multiple nodes
         for path, content in processed_nodes:
             imc.add(FileNode(path, content=content))
         # TODO: handle pre push scenario
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_commit.branch)
 
         self.mark_for_invalidation(repo.repo_name)
         if trigger_push_hook:
             hooks_utils.trigger_post_push_hook(
                 username=user.username, action='push_local',
                 repo_name=repo.repo_name, repo_alias=scm_instance.alias,
                 commit_ids=[tip.raw_id])
         return tip
 
     def update_nodes(self, user, repo, message, nodes, parent_commit=None,
                      author=None, trigger_push_hook=True):
         user = self._get_user(user)
         scm_instance = repo.scm_instance(cache=False)
 
         message = safe_unicode(message)
         commiter = user.full_contact
         author = safe_unicode(author) if author else commiter
 
         imc = scm_instance.in_memory_commit
 
         if not parent_commit:
             parent_commit = EmptyCommit(alias=scm_instance.alias)
 
         if isinstance(parent_commit, EmptyCommit):
             # EmptyCommit means we we're editing empty repository
             parents = None
         else:
             parents = [parent_commit]
 
         # add multiple nodes
         for _filename, data in nodes.items():
             # new filename, can be renamed from the old one, also sanitaze
             # the path for any hack around relative paths like ../../ etc.
             filename = self._sanitize_path(data['filename'])
             old_filename = self._sanitize_path(_filename)
             content = data['content']
 
             filenode = FileNode(old_filename, content=content)
             op = data['op']
             if op == 'add':
                 imc.add(filenode)
             elif op == 'del':
                 imc.remove(filenode)
             elif op == 'mod':
                 if filename != old_filename:
                     # TODO: handle renames more efficient, needs vcs lib
                     # changes
                     imc.remove(filenode)
                     imc.add(FileNode(filename, content=content))
                 else:
                     imc.change(filenode)
 
         try:
             # TODO: handle pre push scenario
             # commit changes
             tip = imc.commit(message=message,
                              author=author,
                              parents=parents,
                              branch=parent_commit.branch)
         except NodeNotChangedError:
             raise
         except Exception as e:
             log.exception("Unexpected exception during call to imc.commit")
             raise IMCCommitError(str(e))
         finally:
             # always clear caches, if commit fails we want fresh object also
             self.mark_for_invalidation(repo.repo_name)
 
         if trigger_push_hook:
             hooks_utils.trigger_post_push_hook(
                 username=user.username, action='push_local',
                 repo_name=repo.repo_name, repo_alias=scm_instance.alias,
                 commit_ids=[tip.raw_id])
 
     def delete_nodes(self, user, repo, message, nodes, parent_commit=None,
                      author=None, trigger_push_hook=True):
         """
         Deletes given multiple nodes into `repo`
 
         :param user: RhodeCode User object or user_id, the committer
         :param repo: RhodeCode Repository object
         :param message: commit message
         :param nodes: mapping {filename:{'content':content},...}
         :param parent_commit: parent commit, can be empty than it's initial
            commit
         :param author: author of commit, cna be different that commiter only
            for git
         :param trigger_push_hook: trigger push hooks
 
         :returns: new commit after deletion
         """
 
         user = self._get_user(user)
         scm_instance = repo.scm_instance(cache=False)
 
         processed_nodes = []
         for f_path in nodes:
             f_path = self._sanitize_path(f_path)
             # content can be empty but for compatabilty it allows same dicts
             # structure as add_nodes
             content = nodes[f_path].get('content')
             processed_nodes.append((f_path, content))
 
         message = safe_unicode(message)
         commiter = user.full_contact
         author = safe_unicode(author) if author else commiter
 
         imc = scm_instance.in_memory_commit
 
         if not parent_commit:
             parent_commit = EmptyCommit(alias=scm_instance.alias)
 
         if isinstance(parent_commit, EmptyCommit):
             # EmptyCommit means we we're editing empty repository
             parents = None
         else:
             parents = [parent_commit]
         # add multiple nodes
         for path, content in processed_nodes:
             imc.remove(FileNode(path, content=content))
 
         # TODO: handle pre push scenario
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_commit.branch)
 
         self.mark_for_invalidation(repo.repo_name)
         if trigger_push_hook:
             hooks_utils.trigger_post_push_hook(
                 username=user.username, action='push_local',
                 repo_name=repo.repo_name, repo_alias=scm_instance.alias,
                 commit_ids=[tip.raw_id])
         return tip
 
     def strip(self, repo, commit_id, branch):
         scm_instance = repo.scm_instance(cache=False)
         scm_instance.config.clear_section('hooks')
         scm_instance.strip(commit_id, branch)
         self.mark_for_invalidation(repo.repo_name)
 
     def get_unread_journal(self):
         return self.sa.query(UserLog).count()
 
     def get_repo_landing_revs(self, repo=None):
         """
         Generates select option with tags branches and bookmarks (for hg only)
         grouped by type
 
         :param repo:
         """
 
         hist_l = []
         choices = []
         repo = self._get_repo(repo)
         hist_l.append(['rev:tip', _('latest tip')])
         choices.append('rev:tip')
         if not repo:
             return choices, hist_l
 
         repo = repo.scm_instance()
 
         branches_group = (
             [(u'branch:%s' % safe_unicode(b), safe_unicode(b))
                 for b in repo.branches],
             _("Branches"))
         hist_l.append(branches_group)
         choices.extend([x[0] for x in branches_group[0]])
 
         if repo.alias == 'hg':
             bookmarks_group = (
                 [(u'book:%s' % safe_unicode(b), safe_unicode(b))
                     for b in repo.bookmarks],
                 _("Bookmarks"))
             hist_l.append(bookmarks_group)
             choices.extend([x[0] for x in bookmarks_group[0]])
 
         tags_group = (
             [(u'tag:%s' % safe_unicode(t), safe_unicode(t))
                 for t in repo.tags],
             _("Tags"))
         hist_l.append(tags_group)
         choices.extend([x[0] for x in tags_group[0]])
 
         return choices, hist_l
 
     def install_git_hook(self, repo, force_create=False):
         """
         Creates a rhodecode hook inside a git repository
 
         :param repo: Instance of VCS repo
         :param force_create: Create even if same name hook exists
         """
 
         loc = os.path.join(repo.path, 'hooks')
         if not repo.bare:
             loc = os.path.join(repo.path, '.git', 'hooks')
         if not os.path.isdir(loc):
             os.makedirs(loc, mode=0777)
 
         tmpl_post = pkg_resources.resource_string(
             'rhodecode', '/'.join(
                 ('config', 'hook_templates', 'git_post_receive.py.tmpl')))
         tmpl_pre = pkg_resources.resource_string(
             'rhodecode', '/'.join(
                 ('config', 'hook_templates', 'git_pre_receive.py.tmpl')))
 
         for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:
             _hook_file = os.path.join(loc, '%s-receive' % h_type)
             log.debug('Installing git hook in repo %s', repo)
             _rhodecode_hook = _check_rhodecode_hook(_hook_file)
 
             if _rhodecode_hook or force_create:
                 log.debug('writing %s hook file !', h_type)
                 try:
                     with open(_hook_file, 'wb') as f:
                         tmpl = tmpl.replace('_TMPL_', rhodecode.__version__)
                         tmpl = tmpl.replace('_ENV_', sys.executable)
                         f.write(tmpl)
                     os.chmod(_hook_file, 0755)
                 except IOError:
                     log.exception('error writing hook file %s', _hook_file)
             else:
                 log.debug('skipping writing hook file')
 
     def install_svn_hooks(self, repo, force_create=False):
         """
         Creates rhodecode hooks inside a svn repository
 
         :param repo: Instance of VCS repo
         :param force_create: Create even if same name hook exists
         """
         hooks_path = os.path.join(repo.path, 'hooks')
         if not os.path.isdir(hooks_path):
             os.makedirs(hooks_path)
         post_commit_tmpl = pkg_resources.resource_string(
             'rhodecode', '/'.join(
                 ('config', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
         pre_commit_template = pkg_resources.resource_string(
             'rhodecode', '/'.join(
                 ('config', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
         templates = {
             'post-commit': post_commit_tmpl,
             'pre-commit': pre_commit_template
         }
         for filename in templates:
             _hook_file = os.path.join(hooks_path, filename)
             _rhodecode_hook = _check_rhodecode_hook(_hook_file)
             if _rhodecode_hook or force_create:
                 log.debug('writing %s hook file !', filename)
                 template = templates[filename]
                 try:
                     with open(_hook_file, 'wb') as f:
                         template = template.replace(
                             '_TMPL_', rhodecode.__version__)
                         template = template.replace('_ENV_', sys.executable)
                         f.write(template)
                     os.chmod(_hook_file, 0755)
                 except IOError:
                     log.exception('error writing hook file %s', filename)
             else:
                 log.debug('skipping writing hook file')
 
     def install_hooks(self, repo, repo_type):
         if repo_type == 'git':
             self.install_git_hook(repo)
         elif repo_type == 'svn':
             self.install_svn_hooks(repo)
 
     def get_server_info(self, environ=None):
         import platform
         import rhodecode
         import pkg_resources
         from rhodecode.model.meta import Base as sql_base, Session
         from sqlalchemy.engine import url
         from rhodecode.lib.base import get_server_ip_addr, get_server_port
         from rhodecode.lib.vcs.backends.git import discover_git_version
         from rhodecode.model.gist import GIST_STORE_LOC
 
         try:
             # cygwin cannot have yet psutil support.
             import psutil
         except ImportError:
             psutil = None
 
         environ = environ or {}
         _NA = 'NOT AVAILABLE'
         _memory = _NA
         _uptime = _NA
         _boot_time = _NA
         _cpu = _NA
         _disk = dict(percent=0, used=0, total=0, error='')
         _load = {'1_min': _NA, '5_min': _NA, '15_min': _NA}
 
         model = VcsSettingsModel()
         storage_path = model.get_repos_location()
         gist_storage_path = os.path.join(storage_path, GIST_STORE_LOC)
         archive_storage_path = rhodecode.CONFIG.get('archive_cache_dir', '')
         search_index_storage_path = rhodecode.CONFIG.get('search.location', '')
 
         if psutil:
             # disk storage
             try:
                 _disk = dict(psutil.disk_usage(storage_path)._asdict())
             except Exception as e:
                 log.exception('Failed to fetch disk info')
                 _disk = {'percent': 0, 'used': 0, 'total': 0, 'error': str(e)}
 
             # memory
             _memory = dict(psutil.virtual_memory()._asdict())
             _memory['percent2'] = psutil._common.usage_percent(
                 (_memory['total'] - _memory['free']),
                 _memory['total'], 1)
 
             # load averages
             if hasattr(psutil.os, 'getloadavg'):
                 _load = dict(zip(
                     ['1_min', '5_min', '15_min'], psutil.os.getloadavg()))
             _uptime = time.time() - psutil.boot_time()
             _boot_time = psutil.boot_time()
             _cpu = psutil.cpu_percent(0.5)
 
         mods = dict([(p.project_name, p.version)
                      for p in pkg_resources.working_set])
 
         def get_storage_size(storage_path):
             sizes = []
             for file_ in os.listdir(storage_path):
                 storage_file = os.path.join(storage_path, file_)
                 if os.path.isfile(storage_file):
                     try:
                         sizes.append(os.path.getsize(storage_file))
                     except OSError:
                         log.exception('Failed to get size of storage file %s',
                                       storage_file)
                         pass
 
             return sum(sizes)
 
         # archive cache storage
         _disk_archive = {'percent': 0, 'used': 0, 'total': 0}
         try:
             archive_storage_path_exists = os.path.isdir(
                 archive_storage_path)
             if archive_storage_path and archive_storage_path_exists:
                 used = get_storage_size(archive_storage_path)
                 _disk_archive.update({
                     'used': used,
                     'total': used,
                 })
         except Exception as e:
             log.exception('failed to fetch archive cache storage')
             _disk_archive['error'] = str(e)
 
         # search index storage
         _disk_index = {'percent': 0, 'used': 0, 'total': 0}
         try:
             search_index_storage_path_exists = os.path.isdir(
                 search_index_storage_path)
             if search_index_storage_path_exists:
                 used = get_storage_size(search_index_storage_path)
                 _disk_index.update({
                     'percent': 100,
                     'used': used,
                     'total': used,
                 })
         except Exception as e:
             log.exception('failed to fetch search index storage')
             _disk_index['error'] = str(e)
 
         # gist storage
         _disk_gist = {'percent': 0, 'used': 0, 'total': 0, 'items': 0}
         try:
             items_count = 0
             used = 0
             for root, dirs, files in os.walk(safe_str(gist_storage_path)):
                 if root == gist_storage_path:
                     items_count = len(dirs)
 
                 for f in files:
                     try:
                         used += os.path.getsize(os.path.join(root, f))
                     except OSError:
                         pass
             _disk_gist.update({
                 'percent': 100,
                 'used': used,
                 'total': used,
                 'items': items_count
             })
         except Exception as e:
             log.exception('failed to fetch gist storage items')
             _disk_gist['error'] = str(e)
 
         # GIT info
         git_ver = discover_git_version()
 
         # SVN info
         # TODO: johbo: Add discover_svn_version to replace this code.
         try:
             import svn.core
             svn_ver = svn.core.SVN_VERSION
         except ImportError:
             svn_ver = None
 
         # DB stuff
         db_info = url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
         db_type = db_info.__to_string__()
         try:
             engine = sql_base.metadata.bind
             db_server_info = engine.dialect._get_server_version_info(
                 Session.connection(bind=engine))
             db_version = '%s %s' % (db_info.drivername,
                                     '.'.join(map(str, db_server_info)))
         except Exception:
             log.exception('failed to fetch db version')
             db_version = '%s %s' % (db_info.drivername, '?')
 
         db_migrate = DbMigrateVersion.query().filter(
             DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
         db_migrate_version = db_migrate.version
 
         info = {
             'py_version': ' '.join(platform._sys_version()),
             'py_path': sys.executable,
             'py_modules': sorted(mods.items(), key=lambda k: k[0].lower()),
 
             'platform': safe_unicode(platform.platform()),
             'storage': storage_path,
             'archive_storage': archive_storage_path,
             'index_storage': search_index_storage_path,
             'gist_storage': gist_storage_path,
 
 
             'db_type': db_type,
             'db_version': db_version,
             'db_migrate_version': db_migrate_version,
 
             'rhodecode_version': rhodecode.__version__,
             'rhodecode_config_ini': rhodecode.CONFIG.get('__file__'),
             'server_ip': '%s:%s' % (
                 get_server_ip_addr(environ, log_errors=False),
                 get_server_port(environ)
             ),
             'server_id': rhodecode.CONFIG.get('instance_id'),
 
             'git_version': safe_unicode(git_ver),
             'hg_version': mods.get('mercurial'),
             'svn_version': svn_ver,
 
             'uptime': _uptime,
             'boot_time': _boot_time,
             'load': _load,
             'cpu': _cpu,
             'memory': _memory,
             'disk': _disk,
             'disk_archive': _disk_archive,
             'disk_gist': _disk_gist,
             'disk_index': _disk_index,
         }
         return info
 
 
 def _check_rhodecode_hook(hook_path):
     """
     Check if the hook was created by RhodeCode
     """
     if not os.path.exists(hook_path):
         return True
 
     log.debug('hook exists, checking if it is from rhodecode')
     hook_content = _read_hook(hook_path)
     matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content)
     if matches:
         try:
             version = matches.groups()[0]
             log.debug('got %s, it is rhodecode', version)
             return True
         except Exception:
             log.exception("Exception while reading the hook version.")
 
     return False
 
 
 def _read_hook(hook_path):
     with open(hook_path, 'rb') as f:
         content = f.read()
     return content