fixed python2.5 compat and repo mapper issue

rhodecode/lib/utils.py

@@ -1,697 +1,697 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.utils
     ~~~~~~~~~~~~~~~~~~~
 
     Utilities library for RhodeCode
 
     :created_on: Apr 18, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import re
 import logging
 import datetime
 import traceback
 import paste
 import beaker
 import tarfile
 import shutil
 from os.path import abspath
 from os.path import dirname as dn, join as jn
 
 from paste.script.command import Command, BadCommand
 
 from mercurial import ui, config
 
 from webhelpers.text import collapse, remove_formatting, strip_tags
 
 from rhodecode.lib.vcs import get_backend
 from rhodecode.lib.vcs.backends.base import BaseChangeset
 from rhodecode.lib.vcs.utils.lazy import LazyProperty
 from rhodecode.lib.vcs.utils.helpers import get_scm
 from rhodecode.lib.vcs.exceptions import VCSError
 
 from rhodecode.lib.caching_query import FromCache
 
 from rhodecode.model import meta
 from rhodecode.model.db import Repository, User, RhodeCodeUi, \
     UserLog, RepoGroup, RhodeCodeSetting, CacheInvalidation
 from rhodecode.model.meta import Session
 from rhodecode.model.repos_group import ReposGroupModel
 from rhodecode.lib.utils2 import safe_str, safe_unicode
 from rhodecode.lib.vcs.utils.fakemod import create_module
 
 log = logging.getLogger(__name__)
 
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
 
 
 def recursive_replace(str_, replace=' '):
     """
     Recursive replace of given sign to just one instance
 
     :param str_: given string
     :param replace: char to find and replace multiple instances
 
     Examples::
     >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
     'Mighty-Mighty-Bo-sstones'
     """
 
     if str_.find(replace * 2) == -1:
         return str_
     else:
         str_ = str_.replace(replace * 2, replace)
         return recursive_replace(str_, replace)
 
 
 def repo_name_slug(value):
     """
     Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
 
     slug = remove_formatting(value)
     slug = strip_tags(slug)
 
     for c in """=[]\;'"<>,/~!@#$%^&*()+{}|: """:
         slug = slug.replace(c, '-')
     slug = recursive_replace(slug, '-')
     slug = collapse(slug, '-')
     return slug
 
 
 def get_repo_slug(request):
     _repo = request.environ['pylons.routes_dict'].get('repo_name')
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 
 
 def get_repos_group_slug(request):
     _group = request.environ['pylons.routes_dict'].get('group_name')
     if _group:
         _group = _group.rstrip('/')
     return _group
 
 
 def action_logger(user, action, repo, ipaddr='', sa=None, commit=False):
     """
     Action logger for various actions made by users
 
     :param user: user that made this action, can be a unique username string or
         object containing user_id attribute
     :param action: action to log, should be on of predefined unique actions for
         easy translations
     :param repo: string name of repository or object containing repo_id,
         that action was made on
     :param ipaddr: optional ip address from what the action was made
     :param sa: optional sqlalchemy session
 
     """
 
     if not sa:
         sa = meta.Session()
 
     try:
         if hasattr(user, 'user_id'):
             user_obj = user
         elif isinstance(user, basestring):
             user_obj = User.get_by_username(user)
         else:
             raise Exception('You have to provide user object or username')
 
         if hasattr(repo, 'repo_id'):
             repo_obj = Repository.get(repo.repo_id)
             repo_name = repo_obj.repo_name
         elif  isinstance(repo, basestring):
             repo_name = repo.lstrip('/')
             repo_obj = Repository.get_by_repo_name(repo_name)
         else:
             repo_obj = None
             repo_name = ''
 
         user_log = UserLog()
         user_log.user_id = user_obj.user_id
         user_log.action = safe_unicode(action)
 
         user_log.repository = repo_obj
         user_log.repository_name = repo_name
 
         user_log.action_date = datetime.datetime.now()
         user_log.user_ip = ipaddr
         sa.add(user_log)
 
         log.info(
             'Adding user %s, action %s on %s' % (user_obj, action,
                                                  safe_unicode(repo))
         )
         if commit:
             sa.commit()
     except:
         log.error(traceback.format_exc())
         raise
 
 
 def get_repos(path, recursive=False):
     """
     Scans given path for repos and return (name,(type,path)) tuple
 
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
 
     # remove ending slash for better results
     path = path.rstrip(os.sep)
 
     def _get_repos(p):
         if not os.access(p, os.W_OK):
             return
         for dirpath in os.listdir(p):
             if os.path.isfile(os.path.join(p, dirpath)):
                 continue
             cur_path = os.path.join(p, dirpath)
             try:
                 scm_info = get_scm(cur_path)
                 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
             except VCSError:
                 if not recursive:
                     continue
                 #check if this dir containts other repos for recursive scan
                 rec_path = os.path.join(p, dirpath)
                 if os.path.isdir(rec_path):
                     for inner_scm in _get_repos(rec_path):
                         yield inner_scm
 
     return _get_repos(path)
 
 
 def is_valid_repo(repo_name, base_path):
     """
     Returns True if given path is a valid repository False otherwise
 
     :param repo_name:
     :param base_path:
 
     :return True: if given path is a valid repository
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
 
     try:
         get_scm(full_path)
         return True
     except VCSError:
         return False
 
 
 def is_valid_repos_group(repos_group_name, base_path):
     """
     Returns True if given path is a repos group False otherwise
 
     :param repo_name:
     :param base_path:
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repos_group_name))
 
     # check if it's not a repo
     if is_valid_repo(repos_group_name, base_path):
         return False
 
     try:
         # we need to check bare git repos at higher level
         # since we might match branches/hooks/info/objects or possible
         # other things inside bare git repo
         get_scm(os.path.dirname(full_path))
         return False
     except VCSError:
         pass
 
     # check if it's a valid path
     if os.path.isdir(full_path):
         return True
 
     return False
 
 
 def ask_ok(prompt, retries=4, complaint='Yes or no, please!'):
     while True:
         ok = raw_input(prompt)
         if ok in ('y', 'ye', 'yes'):
             return True
         if ok in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise IOError
         print complaint
 
 #propagated from mercurial documentation
 ui_sections = ['alias', 'auth',
                 'decode/encode', 'defaults',
                 'diff', 'email',
                 'extensions', 'format',
                 'merge-patterns', 'merge-tools',
                 'hooks', 'http_proxy',
                 'smtp', 'patch',
                 'paths', 'profiling',
                 'server', 'trusted',
                 'ui', 'web', ]
 
 
 def make_ui(read_from='file', path=None, checkpaths=True):
     """
     A function that will read python rc files or database
     and make an mercurial ui object from read options
 
     :param path: path to mercurial config file
     :param checkpaths: check the path
     :param read_from: read from 'file' or 'db'
     """
 
     baseui = ui.ui()
 
     # clean the baseui object
     baseui._ocfg = config.config()
     baseui._ucfg = config.config()
     baseui._tcfg = config.config()
 
     if read_from == 'file':
         if not os.path.isfile(path):
             log.debug('hgrc file is not present at %s skipping...' % path)
             return False
         log.debug('reading hgrc from %s' % path)
         cfg = config.config()
         cfg.read(path)
         for section in ui_sections:
             for k, v in cfg.items(section):
                 log.debug('settings ui from file[%s]%s:%s' % (section, k, v))
                 baseui.setconfig(section, k, v)
 
     elif read_from == 'db':
         sa = meta.Session()
         ret = sa.query(RhodeCodeUi)\
             .options(FromCache("sql_cache_short", "get_hg_ui_settings"))\
             .all()
 
         hg_ui = ret
         for ui_ in hg_ui:
             if ui_.ui_active:
                 log.debug('settings ui from db[%s]%s:%s', ui_.ui_section,
                           ui_.ui_key, ui_.ui_value)
                 baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)
 
         meta.Session.remove()
     return baseui
 
 
 def set_rhodecode_config(config):
     """
     Updates pylons config with new settings from database
 
     :param config:
     """
     hgsettings = RhodeCodeSetting.get_app_settings()
 
     for k, v in hgsettings.items():
         config[k] = v
 
 
 def invalidate_cache(cache_key, *args):
     """
     Puts cache invalidation task into db for
     further global cache invalidation
     """
 
     from rhodecode.model.scm import ScmModel
 
     if cache_key.startswith('get_repo_cached_'):
         name = cache_key.split('get_repo_cached_')[-1]
         ScmModel().mark_for_invalidation(name)
 
 
 class EmptyChangeset(BaseChangeset):
     """
     An dummy empty changeset. It's possible to pass hash when creating
     an EmptyChangeset
     """
 
     def __init__(self, cs='0' * 40, repo=None, requested_revision=None,
                  alias=None):
         self._empty_cs = cs
         self.revision = -1
         self.message = ''
         self.author = ''
         self.date = ''
         self.repository = repo
         self.requested_revision = requested_revision
         self.alias = alias
 
     @LazyProperty
     def raw_id(self):
         """
         Returns raw string identifying this changeset, useful for web
         representation.
         """
 
         return self._empty_cs
 
     @LazyProperty
     def branch(self):
         return get_backend(self.alias).DEFAULT_BRANCH_NAME
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     def get_file_changeset(self, path):
         return self
 
     def get_file_content(self, path):
         return u''
 
     def get_file_size(self, path):
         return 0
 
 
 def map_groups(path):
     """
     Given a full path to a repository, create all nested groups that this
     repo is inside. This function creates parent-child relationships between
     groups and creates default perms for all new groups.
 
     :param paths: full path to repository
     """
     sa = meta.Session()
     groups = path.split(Repository.url_sep())
     parent = None
     group = None
 
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = ReposGroupModel(sa)
     for lvl, group_name in enumerate(groups):
         group_name = '/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
 
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
 
         if group is None:
             log.debug('creating group level: %s group_name: %s' % (lvl,
                                                                    group_name))
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             sa.add(group)
             rgm._create_default_perms(group)
             sa.flush()
         parent = group
     return group
 
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False,
                    install_git_hook=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also check for db entries
     that are not in initial_repo_list and removes them.
 
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     :param install_git_hook: if this is True, also check and install githook
         for a repo if missing
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.scm import ScmModel
     sa = meta.Session()
     rm = RepoModel()
     user = sa.query(User).filter(User.admin == True).first()
     if user is None:
         raise Exception('Missing administrative account !')
     added = []
 
     for name, repo in initial_repo_list.items():
         group = map_groups(name)
-        repo = rm.get_by_repo_name(name)
-        if not repo:
+        db_repo = rm.get_by_repo_name(name)
+        if not db_repo:
             log.info('repository %s not found creating now' % name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
             rm.create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repos_group=getattr(group, 'group_id', None),
                 owner=user,
                 just_db=True
             )
         elif install_git_hook:
-            if repo.repo_type == 'git':
-                ScmModel().install_git_hook(repo.scm_instance)
+            if db_repo.repo_type == 'git':
+                ScmModel().install_git_hook(db_repo.scm_instance)
     sa.commit()
     removed = []
     if remove_obsolete:
         # remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in initial_repo_list.keys():
                 log.debug("Removing non existing repository found in db %s" %
                           repo.repo_name)
                 try:
                     sa.delete(repo)
                     sa.commit()
                     removed.append(repo.repo_name)
                 except:
                     #don't hold further removals on error
                     log.error(traceback.format_exc())
 
     # clear cache keys
     log.debug("Clearing cache keys now...")
     CacheInvalidation.clear_cache()
     sa.commit()
     return added, removed
 
 
 # set cache regions for beaker so celery can utilise it
 def add_cache(settings):
     cache_settings = {'regions': None}
     for key in settings.keys():
         for prefix in ['beaker.cache.', 'cache.']:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 cache_settings[name] = settings[key].strip()
     if cache_settings['regions']:
         for region in cache_settings['regions'].split(','):
             region = region.strip()
             region_settings = {}
             for key, value in cache_settings.items():
                 if key.startswith(region):
                     region_settings[key.split('.')[1]] = value
             region_settings['expire'] = int(region_settings.get('expire',
                                                                 60))
             region_settings.setdefault('lock_dir',
                                        cache_settings.get('lock_dir'))
             region_settings.setdefault('data_dir',
                                        cache_settings.get('data_dir'))
 
             if 'type' not in region_settings:
                 region_settings['type'] = cache_settings.get('type',
                                                              'memory')
             beaker.cache.cache_regions[region] = region_settings
 
 
 def load_rcextensions(root_path):
     import rhodecode
     from rhodecode.config import conf
 
     path = os.path.join(root_path, 'rcextensions', '__init__.py')
     if os.path.isfile(path):
         rcext = create_module('rc', path)
         EXT = rhodecode.EXTENSIONS = rcext
         log.debug('Found rcextensions now loading %s...' % rcext)
 
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
 
         #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)
 
         if getattr(EXT, 'INDEX_EXTENSIONS', []) != []:
             log.debug('settings custom INDEX_EXTENSIONS')
             conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])
 
         #ADDITIONAL MAPPINGS
         log.debug('adding extra into INDEX_EXTENSIONS')
         conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))
 
 
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config, full_index):
     """
     Makes default test index
 
     :param config: test config
     :param full_index:
     """
 
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     from rhodecode.lib.pidlock import DaemonLock, LockHeld
 
     repo_location = repo_location
 
     index_location = os.path.join(config['app_conf']['index_dir'])
     if not os.path.exists(index_location):
         os.makedirs(index_location)
 
     try:
         l = DaemonLock(file_=jn(dn(index_location), 'make_index.lock'))
         WhooshIndexingDaemon(index_location=index_location,
                              repo_location=repo_location)\
             .run(full_index=full_index)
         l.release()
     except LockHeld:
         pass
 
 
 def create_test_env(repos_test_path, config):
     """
     Makes a fresh database and
     install test repository into tmp dir
     """
     from rhodecode.lib.db_manage import DbManage
     from rhodecode.tests import HG_REPO, GIT_REPO, TESTS_TMP_PATH
 
     # PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     log.debug('making test db %s' % dbconf)
 
     # create test dir if it doesn't exist
     if not os.path.isdir(repos_test_path):
         log.debug('Creating testdir %s' % repos_test_path)
         os.makedirs(repos_test_path)
 
     dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],
                         tests=True)
     dbmanage.create_tables(override=True)
     dbmanage.create_settings(dbmanage.config_prompt(repos_test_path))
     dbmanage.create_default_user()
     dbmanage.admin_prompt()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
     # PART TWO make test repo
     log.debug('making test vcs repositories')
 
     idx_path = config['app_conf']['index_dir']
     data_path = config['app_conf']['cache_dir']
 
     #clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s' % idx_path)
         shutil.rmtree(idx_path)
 
     if data_path and os.path.exists(data_path):
         log.debug('remove %s' % data_path)
         shutil.rmtree(data_path)
 
     #CREATE DEFAULT TEST REPOS
     cur_dir = dn(dn(abspath(__file__)))
     tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test_hg.tar.gz"))
     tar.extractall(jn(TESTS_TMP_PATH, HG_REPO))
     tar.close()
 
     cur_dir = dn(dn(abspath(__file__)))
     tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test_git.tar.gz"))
     tar.extractall(jn(TESTS_TMP_PATH, GIT_REPO))
     tar.close()
 
     #LOAD VCS test stuff
     from rhodecode.tests.vcs import setup_package
     setup_package()
 
 
 #==============================================================================
 # PASTER COMMANDS
 #==============================================================================
 class BasePasterCommand(Command):
     """
     Abstract Base Class for paster commands.
 
     The celery commands are somewhat aggressive about loading
     celery.conf, and since our module sets the `CELERY_LOADER`
     environment variable to our loader, we have to bootstrap a bit and
     make sure we've had a chance to load the pylons config off of the
     command line, otherwise everything fails.
     """
     min_args = 1
     min_args_error = "Please provide a paster config file as an argument."
     takes_config_file = 1
     requires_config_file = True
 
     def notify_msg(self, msg, log=False):
         """Make a notification to user, additionally if logger is passed
         it logs this action using given logger
 
         :param msg: message that will be printed to user
         :param log: logging instance, to use to additionally log this message
 
         """
         if log and isinstance(log, logging):
             log(msg)
 
     def run(self, args):
         """
         Overrides Command.run
 
         Checks for a config file argument and loads it.
         """
         if len(args) < self.min_args:
             raise BadCommand(
                 self.min_args_error % {'min_args': self.min_args,
                                        'actual_args': len(args)})
 
         # Decrement because we're going to lob off the first argument.
         # @@ This is hacky
         self.min_args -= 1
         self.bootstrap_config(args[0])
         self.update_parser()
         return super(BasePasterCommand, self).run(args[1:])
 
     def update_parser(self):
         """
         Abstract method.  Allows for the class's parser to be updated
         before the superclass's `run` method is called.  Necessary to
         allow options/arguments to be passed through to the underlying
         celery command.
         """
         raise NotImplementedError("Abstract Method.")
 
     def bootstrap_config(self, conf):
         """
         Loads the pylons configuration.
         """
         from pylons import config as pylonsconfig
 
         self.path_to_ini_file = os.path.realpath(conf)
         conf = paste.deploy.appconfig('config:' + self.path_to_ini_file)
         pylonsconfig.init_app(conf.global_conf, conf.local_conf)

rhodecode/model/scm.py

@@ -1,597 +1,598 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.model.scm
     ~~~~~~~~~~~~~~~~~~~
 
     Scm model for RhodeCode
 
     :created_on: Apr 9, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import with_statement
 import os
 import re
 import time
 import traceback
 import logging
 import cStringIO
 import pkg_resources
 from os.path import dirname as dn, join as jn
 
 from sqlalchemy import func
 from pylons.i18n.translation import _
 
 import rhodecode
 from rhodecode.lib.vcs import get_backend
 from rhodecode.lib.vcs.exceptions import RepositoryError
 from rhodecode.lib.vcs.utils.lazy import LazyProperty
 from rhodecode.lib.vcs.nodes import FileNode
 
 from rhodecode import BACKENDS
 from rhodecode.lib import helpers as h
 from rhodecode.lib.utils2 import safe_str, safe_unicode
 from rhodecode.lib.auth import HasRepoPermissionAny, HasReposGroupPermissionAny
 from rhodecode.lib.utils import get_repos as get_filesystem_repos, make_ui, \
     action_logger, EmptyChangeset, REMOVED_REPO_PAT
 from rhodecode.model import BaseModel
 from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \
     UserFollowing, UserLog, User, RepoGroup, PullRequest
 
 log = logging.getLogger(__name__)
 
 
 class UserTemp(object):
     def __init__(self, user_id):
         self.user_id = user_id
 
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
 
 
 class RepoTemp(object):
     def __init__(self, repo_id):
         self.repo_id = repo_id
 
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
 
 
 class CachedRepoList(object):
     """
     Cached repo list, uses in-memory cache after initialization, that is
     super fast
     """
 
     def __init__(self, db_repo_list, repos_path, order_by=None):
         self.db_repo_list = db_repo_list
         self.repos_path = repos_path
         self.order_by = order_by
         self.reversed = (order_by or '').startswith('-')
 
     def __len__(self):
         return len(self.db_repo_list)
 
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
 
     def __iter__(self):
         # pre-propagated cache_map to save executing select statements
         # for each repo
         cache_map = CacheInvalidation.get_cache_map()
 
         for dbr in self.db_repo_list:
             scmr = dbr.scm_instance_cached(cache_map)
             # check permission at this level
             if not HasRepoPermissionAny(
                 'repository.read', 'repository.write', 'repository.admin'
             )(dbr.repo_name, 'get repo check'):
                 continue
 
             if scmr is None:
                 log.error(
                     '%s this repository is present in database but it '
                     'cannot be created as an scm instance' % dbr.repo_name
                 )
                 continue
 
             last_change = scmr.last_change
             tip = h.get_changeset_safe(scmr, 'tip')
 
             tmp_d = {}
             tmp_d['name'] = dbr.repo_name
             tmp_d['name_sort'] = tmp_d['name'].lower()
             tmp_d['description'] = dbr.description
             tmp_d['description_sort'] = tmp_d['description'].lower()
             tmp_d['last_change'] = last_change
             tmp_d['last_change_sort'] = time.mktime(last_change.timetuple())
             tmp_d['tip'] = tip.raw_id
             tmp_d['tip_sort'] = tip.revision
             tmp_d['rev'] = tip.revision
             tmp_d['contact'] = dbr.user.full_contact
             tmp_d['contact_sort'] = tmp_d['contact']
             tmp_d['owner_sort'] = tmp_d['contact']
             tmp_d['repo_archives'] = list(scmr._get_archives())
             tmp_d['last_msg'] = tip.message
             tmp_d['author'] = tip.author
             tmp_d['dbrepo'] = dbr.get_dict()
             tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork else {}
             yield tmp_d
 
 
 class SimpleCachedRepoList(CachedRepoList):
     """
     Lighter version of CachedRepoList without the scm initialisation
     """
 
     def __iter__(self):
         for dbr in self.db_repo_list:
             # check permission at this level
             if not HasRepoPermissionAny(
                 'repository.read', 'repository.write', 'repository.admin'
             )(dbr.repo_name, 'get repo check'):
                 continue
 
             tmp_d = {}
             tmp_d['name'] = dbr.repo_name
             tmp_d['name_sort'] = tmp_d['name'].lower()
             tmp_d['description'] = dbr.description
             tmp_d['description_sort'] = tmp_d['description'].lower()
             tmp_d['dbrepo'] = dbr.get_dict()
             tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork else {}
             yield tmp_d
 
 
 class GroupList(object):
 
     def __init__(self, db_repo_group_list):
         self.db_repo_group_list = db_repo_group_list
 
     def __len__(self):
         return len(self.db_repo_group_list)
 
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
 
     def __iter__(self):
         for dbgr in self.db_repo_group_list:
             # check permission at this level
             if not HasReposGroupPermissionAny(
                 'group.read', 'group.write', 'group.admin'
             )(dbgr.group_name, 'get group repo check'):
                 continue
 
             yield dbgr
 
 
 class ScmModel(BaseModel):
     """
     Generic Scm Model
     """
 
     def __get_repo(self, instance):
         cls = Repository
         if isinstance(instance, cls):
             return instance
         elif isinstance(instance, int) or str(instance).isdigit():
             return cls.get(instance)
         elif isinstance(instance, basestring):
             return cls.get_by_repo_name(instance)
         elif instance:
             raise Exception('given object must be int, basestr or Instance'
                             ' of %s got %s' % (type(cls), type(instance)))
 
     @LazyProperty
     def repos_path(self):
         """
         Get's the repositories root path from database
         """
 
         q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
 
         return q.ui_value
 
     def repo_scan(self, repos_path=None):
         """
         Listing of repositories in given path. This path should not be a
         repository itself. Return a dictionary of repository objects
 
         :param repos_path: path to directory containing repositories
         """
 
         if repos_path is None:
             repos_path = self.repos_path
 
         log.info('scanning for repositories in %s' % repos_path)
 
         baseui = make_ui('db')
         repos = {}
 
         for name, path in get_filesystem_repos(repos_path, recursive=True):
             # skip removed repos
             if REMOVED_REPO_PAT.match(name):
                 continue
 
             # name need to be decomposed and put back together using the /
             # since this is internal storage separator for rhodecode
             name = Repository.url_sep().join(name.split(os.sep))
 
             try:
                 if name in repos:
                     raise RepositoryError('Duplicate repository name %s '
                                           'found in %s' % (name, path))
                 else:
 
                     klass = get_backend(path[0])
 
                     if path[0] == 'hg' and path[0] in BACKENDS.keys():
                         repos[name] = klass(safe_str(path[1]), baseui=baseui)
 
                     if path[0] == 'git' and path[0] in BACKENDS.keys():
                         repos[name] = klass(path[1])
             except OSError:
                 continue
 
         return repos
 
     def get_repos(self, all_repos=None, sort_key=None, simple=False):
         """
         Get all repos from db and for each repo create it's
         backend instance and fill that backed with information from database
 
         :param all_repos: list of repository names as strings
             give specific repositories list, good for filtering
 
         :param sort_key: initial sorting of repos
         :param simple: use SimpleCachedList - one without the SCM info
         """
         if all_repos is None:
             all_repos = self.sa.query(Repository)\
                         .filter(Repository.group_id == None)\
                         .order_by(func.lower(Repository.repo_name)).all()
         if simple:
             repo_iter = SimpleCachedRepoList(all_repos,
                                              repos_path=self.repos_path,
                                              order_by=sort_key)
         else:
             repo_iter = CachedRepoList(all_repos,
                                        repos_path=self.repos_path,
                                        order_by=sort_key)
 
         return repo_iter
 
     def get_repos_groups(self, all_groups=None):
         if all_groups is None:
             all_groups = RepoGroup.query()\
                 .filter(RepoGroup.group_parent_id == None).all()
         group_iter = GroupList(all_groups)
 
         return group_iter
 
     def mark_for_invalidation(self, repo_name):
         """
         Puts cache invalidation task into db for
         further global cache invalidation
 
         :param repo_name: this repo that should invalidation take place
         """
         CacheInvalidation.set_invalidate(repo_name)
 
     def toggle_following_repo(self, follow_repo_id, user_id):
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repo_id == follow_repo_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 self.sa.delete(f)
                 action_logger(UserTemp(user_id),
                               'stopped_following_repo',
                               RepoTemp(follow_repo_id))
                 return
             except:
                 log.error(traceback.format_exc())
                 raise
 
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_repo_id = follow_repo_id
             self.sa.add(f)
 
             action_logger(UserTemp(user_id),
                           'started_following_repo',
                           RepoTemp(follow_repo_id))
         except:
             log.error(traceback.format_exc())
             raise
 
     def toggle_following_user(self, follow_user_id, user_id):
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user_id == follow_user_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 self.sa.delete(f)
                 return
             except:
                 log.error(traceback.format_exc())
                 raise
 
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_user_id = follow_user_id
             self.sa.add(f)
         except:
             log.error(traceback.format_exc())
             raise
 
     def is_following_repo(self, repo_name, user_id, cache=False):
         r = self.sa.query(Repository)\
             .filter(Repository.repo_name == repo_name).scalar()
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repository == r)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def is_following_user(self, username, user_id, cache=False):
         u = User.get_by_username(username)
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user == u)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def get_followers(self, repo):
         repo = self._get_repo(repo)
 
         return self.sa.query(UserFollowing)\
                 .filter(UserFollowing.follows_repository == repo).count()
 
     def get_forks(self, repo):
         repo = self._get_repo(repo)
         return self.sa.query(Repository)\
                 .filter(Repository.fork == repo).count()
 
     def get_pull_requests(self, repo):
         repo = self._get_repo(repo)
         return self.sa.query(PullRequest)\
                 .filter(PullRequest.other_repo == repo).count()
 
     def mark_as_fork(self, repo, fork, user):
         repo = self.__get_repo(repo)
         fork = self.__get_repo(fork)
         repo.fork = fork
         self.sa.add(repo)
         return repo
 
     def pull_changes(self, repo, username):
         dbrepo = self.__get_repo(repo)
         clone_uri = dbrepo.clone_uri
         if not clone_uri:
             raise Exception("This repository doesn't have a clone uri")
 
         repo = dbrepo.scm_instance
         try:
             extras = {
                 'ip': '',
                 'username': username,
                 'action': 'push_remote',
                 'repository': dbrepo.repo_name,
                 'scm': repo.alias,
             }
             Repository.inject_ui(repo, extras=extras)
 
             if repo.alias == 'git':
                 repo.fetch(clone_uri)
             else:
                 repo.pull(clone_uri)
             self.mark_for_invalidation(dbrepo.repo_name)
         except:
             log.error(traceback.format_exc())
             raise
 
     def commit_change(self, repo, repo_name, cs, user, author, message,
                       content, f_path):
 
         if repo.alias == 'hg':
             from rhodecode.lib.vcs.backends.hg import \
                 MercurialInMemoryChangeset as IMC
         elif repo.alias == 'git':
             from rhodecode.lib.vcs.backends.git import \
                 GitInMemoryChangeset as IMC
 
         # decoding here will force that we have proper encoded values
         # in any other case this will throw exceptions and deny commit
         content = safe_str(content)
         path = safe_str(f_path)
         # message and author needs to be unicode
         # proper backend should then translate that into required type
         message = safe_unicode(message)
         author = safe_unicode(author)
         m = IMC(repo)
         m.change(FileNode(path, content))
         tip = m.commit(message=message,
                        author=author,
                        parents=[cs], branch=cs.branch)
 
         new_cs = tip.short_id
         action = 'push_local:%s' % new_cs
 
         action_logger(user, action, repo_name)
 
         self.mark_for_invalidation(repo_name)
 
     def create_node(self, repo, repo_name, cs, user, author, message, content,
                       f_path):
         if repo.alias == 'hg':
             from rhodecode.lib.vcs.backends.hg import MercurialInMemoryChangeset as IMC
         elif repo.alias == 'git':
             from rhodecode.lib.vcs.backends.git import GitInMemoryChangeset as IMC
         # decoding here will force that we have proper encoded values
         # in any other case this will throw exceptions and deny commit
 
         if isinstance(content, (basestring,)):
             content = safe_str(content)
         elif isinstance(content, (file, cStringIO.OutputType,)):
             content = content.read()
         else:
             raise Exception('Content is of unrecognized type %s' % (
                 type(content)
             ))
 
         message = safe_unicode(message)
         author = safe_unicode(author)
         path = safe_str(f_path)
         m = IMC(repo)
 
         if isinstance(cs, EmptyChangeset):
             # EmptyChangeset means we we're editing empty repository
             parents = None
         else:
             parents = [cs]
 
         m.add(FileNode(path, content=content))
         tip = m.commit(message=message,
                        author=author,
                        parents=parents, branch=cs.branch)
         new_cs = tip.short_id
         action = 'push_local:%s' % new_cs
 
         action_logger(user, action, repo_name)
 
         self.mark_for_invalidation(repo_name)
 
     def get_nodes(self, repo_name, revision, root_path='/', flat=True):
         """
         recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
 
         :param repo_name: name of repository
         :param revision: revision for which to list nodes
         :param root_path: root path to list
         :param flat: return as a list, if False returns a dict with decription
 
         """
         _files = list()
         _dirs = list()
         try:
             _repo = self.__get_repo(repo_name)
             changeset = _repo.scm_instance.get_changeset(revision)
             root_path = root_path.lstrip('/')
             for topnode, dirs, files in changeset.walk(root_path):
                 for f in files:
                     _files.append(f.path if flat else {"name": f.path,
                                                        "type": "file"})
                 for d in dirs:
                     _dirs.append(d.path if flat else {"name": d.path,
                                                       "type": "dir"})
         except RepositoryError:
             log.debug(traceback.format_exc())
             raise
 
         return _dirs, _files
 
     def get_unread_journal(self):
         return self.sa.query(UserLog).count()
 
     def get_repo_landing_revs(self, repo=None):
         """
         Generates select option with tags branches and bookmarks (for hg only)
         grouped by type
 
         :param repo:
         :type repo:
         """
 
         hist_l = []
         choices = []
         repo = self.__get_repo(repo)
         hist_l.append(['tip', _('latest tip')])
         choices.append('tip')
         if not repo:
             return choices, hist_l
 
         repo = repo.scm_instance
 
         branches_group = ([(k, k) for k, v in
                            repo.branches.iteritems()], _("Branches"))
         hist_l.append(branches_group)
         choices.extend([x[0] for x in branches_group[0]])
 
         if repo.alias == 'hg':
             bookmarks_group = ([(k, k) for k, v in
                                 repo.bookmarks.iteritems()], _("Bookmarks"))
             hist_l.append(bookmarks_group)
             choices.extend([x[0] for x in bookmarks_group[0]])
 
         tags_group = ([(k, k) for k, v in
                        repo.tags.iteritems()], _("Tags"))
         hist_l.append(tags_group)
         choices.extend([x[0] for x in tags_group[0]])
 
         return choices, hist_l
 
     def install_git_hook(self, repo, force_create=False):
         """
         Creates a rhodecode hook inside a git repository
 
         :param repo: Instance of VCS repo
         :param force_create: Create even if same name hook exists
         """
 
         loc = jn(repo.path, 'hooks')
         if not repo.bare:
             loc = jn(repo.path, '.git', 'hooks')
         if not os.path.isdir(loc):
             os.makedirs(loc)
 
         tmpl = pkg_resources.resource_string(
             'rhodecode', jn('config', 'post_receive_tmpl.py')
         )
 
         _hook_file = jn(loc, 'post-receive')
         _rhodecode_hook = False
         log.debug('Installing git hook in repo %s' % repo)
         if os.path.exists(_hook_file):
             # let's take a look at this hook, maybe it's rhodecode ?
             log.debug('hook exists, checking if it is from rhodecode')
             _HOOK_VER_PAT = re.compile(r'^RC_HOOK_VER')
             with open(_hook_file, 'rb') as f:
                 data = f.read()
                 matches = re.compile(r'(?:%s)\s*=\s*(.*)'
                                      % 'RC_HOOK_VER').search(data)
                 if matches:
                     try:
                         ver = matches.groups()[0]
                         log.debug('got %s it is rhodecode' % (ver))
                         _rhodecode_hook = True
                     except:
                         log.error(traceback.format_exc())
 
         if _rhodecode_hook or force_create:
             log.debug('writing hook file !')
             with open(_hook_file, 'wb') as f:
                 tmpl = tmpl.replace('_TMPL_', rhodecode.__version__)
                 f.write(tmpl)
             os.chmod(_hook_file, 0755)
         else:
             log.debug('skipping writing hook file')
  No newline at end of file