cleanup: get rid of dn as shortcut for os.path.dirname

We keep 'dirname' as shortcut despite having removed the 'join' shortcut:
* the name is less ambiguous than 'join'.
* dirname is often applied multiple times - spelling it out would be too
  verbose.

kallithea/lib/db_manage.py

@@ -1,572 +1,572 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.db_manage
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 Database creation, and setup module for Kallithea. Used for creation
 of database as well as for migration operations
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 10, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import os
 import sys
 import time
 import uuid
 import logging
-from os.path import dirname as dn
+from os.path import dirname
 
 from kallithea import __dbversion__, __py_version__, EXTERN_TYPE_INTERNAL, DB_MIGRATIONS
 from kallithea.model.user import UserModel
 from kallithea.lib.utils import ask_ok
 from kallithea.model import init_model
 from kallithea.model.db import User, Permission, Ui, \
     Setting, UserToPerm, DbMigrateVersion, RepoGroup, \
     UserRepoGroupToPerm, CacheInvalidation, Repository
 
 from sqlalchemy.engine import create_engine
 from kallithea.model.repo_group import RepoGroupModel
 #from kallithea.model import meta
 from kallithea.model.meta import Session, Base
 from kallithea.model.repo import RepoModel
 from kallithea.model.permission import PermissionModel
 
 
 log = logging.getLogger(__name__)
 
 
 def notify(msg):
     """
     Notification for migrations messages
     """
     ml = len(msg) + (4 * 2)
     print('\n%s\n*** %s ***\n%s' % ('*' * ml, msg, '*' * ml)).upper()
 
 
 class DbManage(object):
     def __init__(self, log_sql, dbconf, root, tests=False, SESSION=None, cli_args=None):
         self.dbname = dbconf.split('/')[-1]
         self.tests = tests
         self.root = root
         self.dburi = dbconf
         self.log_sql = log_sql
         self.db_exists = False
         self.cli_args = cli_args or {}
         self.init_db(SESSION=SESSION)
 
         force_ask = self.cli_args.get('force_ask')
         if force_ask is not None:
             global ask_ok
             ask_ok = lambda *args, **kwargs: force_ask
 
     def init_db(self, SESSION=None):
         if SESSION:
             self.sa = SESSION
         else:
             #init new sessions
             engine = create_engine(self.dburi, echo=self.log_sql)
             init_model(engine)
             self.sa = Session()
 
     def create_tables(self, override=False):
         """
         Create a auth database
         """
 
         log.info("Any existing database is going to be destroyed")
         if self.tests:
             destroy = True
         else:
             destroy = ask_ok('Are you sure to destroy old database ? [y/n]')
         if not destroy:
             print 'Nothing done.'
             sys.exit(0)
         if destroy:
             Base.metadata.drop_all()
 
         checkfirst = not override
         Base.metadata.create_all(checkfirst=checkfirst)
         log.info('Created tables for %s', self.dbname)
 
     def set_db_version(self):
         ver = DbMigrateVersion()
         ver.version = __dbversion__
         ver.repository_id = DB_MIGRATIONS
         ver.repository_path = 'versions'
         self.sa.add(ver)
         log.info('db version set to: %s', __dbversion__)
 
     def upgrade(self):
         """
         Upgrades given database schema to given revision following
         all needed steps, to perform the upgrade
 
         """
 
         from kallithea.lib.dbmigrate.migrate.versioning import api
         from kallithea.lib.dbmigrate.migrate.exceptions import \
             DatabaseNotControlledError
 
         if 'sqlite' in self.dburi:
             print (
                '********************** WARNING **********************\n'
                'Make sure your version of sqlite is at least 3.7.X.  \n'
                'Earlier versions are known to fail on some migrations\n'
                '*****************************************************\n')
 
         upgrade = ask_ok('You are about to perform database upgrade, make '
                          'sure You backed up your database before. '
                          'Continue ? [y/n]')
         if not upgrade:
             print 'No upgrade performed'
             sys.exit(0)
 
-        repository_path = os.path.join(dn(dn(dn(os.path.realpath(__file__)))),
+        repository_path = os.path.join(dirname(dirname(dirname(os.path.realpath(__file__)))),
                                        'kallithea', 'lib', 'dbmigrate')
         db_uri = self.dburi
 
         try:
             curr_version = api.db_version(db_uri, repository_path)
             msg = ('Found current database under version '
                    'control with version %s' % curr_version)
 
         except (RuntimeError, DatabaseNotControlledError):
             curr_version = 1
             msg = ('Current database is not under version control. Setting '
                    'as version %s' % curr_version)
             api.version_control(db_uri, repository_path, curr_version)
 
         notify(msg)
         if curr_version == __dbversion__:
             print 'This database is already at the newest version'
             sys.exit(0)
 
         # clear cache keys
         log.info("Clearing cache keys now...")
         CacheInvalidation.clear_cache()
 
         upgrade_steps = range(curr_version + 1, __dbversion__ + 1)
         notify('attempting to do database upgrade from '
                'version %s to version %s' % (curr_version, __dbversion__))
 
         # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
         _step = None
         for step in upgrade_steps:
             notify('performing upgrade step %s' % step)
             time.sleep(0.5)
 
             api.upgrade(db_uri, repository_path, step)
             notify('schema upgrade for step %s completed' % (step,))
 
             _step = step
 
         notify('upgrade to version %s successful' % _step)
 
     def fix_repo_paths(self):
         """
         Fixes a old kallithea version path into new one without a '*'
         """
 
         paths = self.sa.query(Ui) \
                 .filter(Ui.ui_key == '/') \
                 .scalar()
 
         paths.ui_value = paths.ui_value.replace('*', '')
 
         self.sa.add(paths)
         self.sa.commit()
 
     def fix_default_user(self):
         """
         Fixes a old default user with some 'nicer' default values,
         used mostly for anonymous access
         """
         def_user = self.sa.query(User) \
                 .filter(User.username == User.DEFAULT_USER) \
                 .one()
 
         def_user.name = 'Anonymous'
         def_user.lastname = 'User'
         def_user.email = '[e-mail]'
 
         self.sa.add(def_user)
         self.sa.commit()
 
     def fix_settings(self):
         """
         Fixes kallithea settings adds ga_code key for google analytics
         """
 
         hgsettings3 = Setting('ga_code', '')
 
         self.sa.add(hgsettings3)
         self.sa.commit()
 
     def admin_prompt(self, second=False):
         if not self.tests:
             import getpass
 
             # defaults
             defaults = self.cli_args
             username = defaults.get('username')
             password = defaults.get('password')
             email = defaults.get('email')
 
             def get_password():
                 password = getpass.getpass('Specify admin password '
                                            '(min 6 chars):')
                 confirm = getpass.getpass('Confirm password:')
 
                 if password != confirm:
                     log.error('passwords mismatch')
                     return False
                 if len(password) < 6:
                     log.error('password is to short use at least 6 characters')
                     return False
 
                 return password
             if username is None:
                 username = raw_input('Specify admin username:')
             if password is None:
                 password = get_password()
                 if not password:
                     #second try
                     password = get_password()
                     if not password:
                         sys.exit()
             if email is None:
                 email = raw_input('Specify admin email:')
             self.create_user(username, password, email, True)
         else:
             log.info('creating admin and regular test users')
             from kallithea.tests import TEST_USER_ADMIN_LOGIN, \
             TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, \
             TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, \
             TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR2_LOGIN, \
             TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL
 
             self.create_user(TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
                              TEST_USER_ADMIN_EMAIL, True)
 
             self.create_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
                              TEST_USER_REGULAR_EMAIL, False)
 
             self.create_user(TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS,
                              TEST_USER_REGULAR2_EMAIL, False)
 
     def create_ui_settings(self, repo_store_path):
         """
         Creates ui settings, fills out hooks
         """
 
         #HOOKS
         hooks1_key = Ui.HOOK_UPDATE
         hooks1_ = self.sa.query(Ui) \
             .filter(Ui.ui_key == hooks1_key).scalar()
 
         hooks1 = Ui() if hooks1_ is None else hooks1_
         hooks1.ui_section = 'hooks'
         hooks1.ui_key = hooks1_key
         hooks1.ui_value = 'hg update >&2'
         hooks1.ui_active = False
         self.sa.add(hooks1)
 
         hooks2_key = Ui.HOOK_REPO_SIZE
         hooks2_ = self.sa.query(Ui) \
             .filter(Ui.ui_key == hooks2_key).scalar()
         hooks2 = Ui() if hooks2_ is None else hooks2_
         hooks2.ui_section = 'hooks'
         hooks2.ui_key = hooks2_key
         hooks2.ui_value = 'python:kallithea.lib.hooks.repo_size'
         self.sa.add(hooks2)
 
         hooks3 = Ui()
         hooks3.ui_section = 'hooks'
         hooks3.ui_key = Ui.HOOK_PUSH
         hooks3.ui_value = 'python:kallithea.lib.hooks.log_push_action'
         self.sa.add(hooks3)
 
         hooks4 = Ui()
         hooks4.ui_section = 'hooks'
         hooks4.ui_key = Ui.HOOK_PRE_PUSH
         hooks4.ui_value = 'python:kallithea.lib.hooks.pre_push'
         self.sa.add(hooks4)
 
         hooks5 = Ui()
         hooks5.ui_section = 'hooks'
         hooks5.ui_key = Ui.HOOK_PULL
         hooks5.ui_value = 'python:kallithea.lib.hooks.log_pull_action'
         self.sa.add(hooks5)
 
         hooks6 = Ui()
         hooks6.ui_section = 'hooks'
         hooks6.ui_key = Ui.HOOK_PRE_PULL
         hooks6.ui_value = 'python:kallithea.lib.hooks.pre_pull'
         self.sa.add(hooks6)
 
         # enable largefiles
         largefiles = Ui()
         largefiles.ui_section = 'extensions'
         largefiles.ui_key = 'largefiles'
         largefiles.ui_value = ''
         self.sa.add(largefiles)
 
         # set default largefiles cache dir, defaults to
         # /repo location/.cache/largefiles
         largefiles = Ui()
         largefiles.ui_section = 'largefiles'
         largefiles.ui_key = 'usercache'
         largefiles.ui_value = os.path.join(repo_store_path, '.cache',
                                            'largefiles')
         self.sa.add(largefiles)
 
         # enable hgsubversion disabled by default
         hgsubversion = Ui()
         hgsubversion.ui_section = 'extensions'
         hgsubversion.ui_key = 'hgsubversion'
         hgsubversion.ui_value = ''
         hgsubversion.ui_active = False
         self.sa.add(hgsubversion)
 
         # enable hggit disabled by default
         hggit = Ui()
         hggit.ui_section = 'extensions'
         hggit.ui_key = 'hggit'
         hggit.ui_value = ''
         hggit.ui_active = False
         self.sa.add(hggit)
 
     def create_auth_plugin_options(self, skip_existing=False):
         """
         Create default auth plugin settings, and make it active
 
         :param skip_existing:
         """
 
         for k, v, t in [('auth_plugins', 'kallithea.lib.auth_modules.auth_internal', 'list'),
                      ('auth_internal_enabled', 'True', 'bool')]:
             if skip_existing and Setting.get_by_name(k) != None:
                 log.debug('Skipping option %s', k)
                 continue
             setting = Setting(k, v, t)
             self.sa.add(setting)
 
     def create_default_options(self, skip_existing=False):
         """Creates default settings"""
 
         for k, v, t in [
             ('default_repo_enable_locking',  False, 'bool'),
             ('default_repo_enable_downloads', False, 'bool'),
             ('default_repo_enable_statistics', False, 'bool'),
             ('default_repo_private', False, 'bool'),
             ('default_repo_type', 'hg', 'unicode')]:
 
             if skip_existing and Setting.get_by_name(k) is not None:
                 log.debug('Skipping option %s', k)
                 continue
             setting = Setting(k, v, t)
             self.sa.add(setting)
 
     def fixup_groups(self):
         def_usr = User.get_default_user()
         for g in RepoGroup.query().all():
             g.group_name = g.get_new_name(g.name)
             self.sa.add(g)
             # get default perm
             default = UserRepoGroupToPerm.query() \
                 .filter(UserRepoGroupToPerm.group == g) \
                 .filter(UserRepoGroupToPerm.user == def_usr) \
                 .scalar()
 
             if default is None:
                 log.debug('missing default permission for group %s adding', g)
                 perm_obj = RepoGroupModel()._create_default_perms(g)
                 self.sa.add(perm_obj)
 
     def reset_permissions(self, username):
         """
         Resets permissions to default state, useful when old systems had
         bad permissions, we must clean them up
 
         :param username:
         """
         default_user = User.get_by_username(username)
         if not default_user:
             return
 
         u2p = UserToPerm.query() \
             .filter(UserToPerm.user == default_user).all()
         fixed = False
         if len(u2p) != len(Permission.DEFAULT_USER_PERMISSIONS):
             for p in u2p:
                 Session().delete(p)
             fixed = True
             self.populate_default_permissions()
         return fixed
 
     def update_repo_info(self):
         RepoModel.update_repoinfo()
 
     def config_prompt(self, test_repo_path='', retries=3):
         defaults = self.cli_args
         _path = defaults.get('repos_location')
         if retries == 3:
             log.info('Setting up repositories config')
 
         if _path is not None:
             path = _path
         elif not self.tests and not test_repo_path:
             path = raw_input(
                  'Enter a valid absolute path to store repositories. '
                  'All repositories in that path will be added automatically:'
             )
         else:
             path = test_repo_path
         path_ok = True
 
         # check proper dir
         if not os.path.isdir(path):
             path_ok = False
             log.error('Given path %s is not a valid directory', path)
 
         elif not os.path.isabs(path):
             path_ok = False
             log.error('Given path %s is not an absolute path', path)
 
         # check if path is at least readable.
         if not os.access(path, os.R_OK):
             path_ok = False
             log.error('Given path %s is not readable', path)
 
         # check write access, warn user about non writeable paths
         elif not os.access(path, os.W_OK) and path_ok:
             log.warning('No write permission to given path %s', path)
             if not ask_ok('Given path %s is not writeable, do you want to '
                           'continue with read only mode ? [y/n]' % (path,)):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         if retries == 0:
             sys.exit('max retries reached')
         if not path_ok:
             if _path is not None:
                 sys.exit('Invalid repo path: %s' % _path)
             retries -= 1
             return self.config_prompt(test_repo_path, retries) # recursing!!!
 
         real_path = os.path.normpath(os.path.realpath(path))
 
         if real_path != os.path.normpath(path):
             log.warning('Using normalized path %s instead of %s', real_path, path)
 
         return real_path
 
     def create_settings(self, path):
 
         self.create_ui_settings(path)
 
         ui_config = [
             ('web', 'push_ssl', 'false'),
             ('web', 'allow_archive', 'gz zip bz2'),
             ('web', 'allow_push', '*'),
             ('web', 'baseurl', '/'),
             ('paths', '/', path),
             #('phases', 'publish', 'false')
         ]
         for section, key, value in ui_config:
             ui_conf = Ui()
             setattr(ui_conf, 'ui_section', section)
             setattr(ui_conf, 'ui_key', key)
             setattr(ui_conf, 'ui_value', value)
             self.sa.add(ui_conf)
 
         settings = [
             ('realm', 'Kallithea', 'unicode'),
             ('title', '', 'unicode'),
             ('ga_code', '', 'unicode'),
             ('show_public_icon', True, 'bool'),
             ('show_private_icon', True, 'bool'),
             ('stylify_metatags', False, 'bool'),
             ('dashboard_items', 100, 'int'),
             ('admin_grid_items', 25, 'int'),
             ('show_version', True, 'bool'),
             ('use_gravatar', True, 'bool'),
             ('gravatar_url', User.DEFAULT_GRAVATAR_URL, 'unicode'),
             ('clone_uri_tmpl', Repository.DEFAULT_CLONE_URI, 'unicode'),
             ('update_url', Setting.DEFAULT_UPDATE_URL, 'unicode'),
         ]
         for key, val, type_ in settings:
             sett = Setting(key, val, type_)
             self.sa.add(sett)
 
         self.create_auth_plugin_options()
         self.create_default_options()
 
         log.info('created ui config')
 
     def create_user(self, username, password, email='', admin=False):
         log.info('creating user %s', username)
         UserModel().create_or_update(username, password, email,
                                      firstname=u'Kallithea', lastname=u'Admin',
                                      active=True, admin=admin,
                                      extern_type=EXTERN_TYPE_INTERNAL)
 
     def create_default_user(self):
         log.info('creating default user')
         # create default user for handling default permissions.
         user = UserModel().create_or_update(username=User.DEFAULT_USER,
                                             password=str(uuid.uuid1())[:20],
                                             email='[e-mail]',
                                             firstname=u'Anonymous',
                                             lastname=u'User')
         # based on configuration options activate/deactivate this user which
         # controls anonymous access
         if self.cli_args.get('public_access') is False:
             log.info('Public access disabled')
             user.active = False
             Session().add(user)
             Session().commit()
 
     def create_permissions(self):
         """
         Creates all permissions defined in the system
         """
         # module.(access|create|change|delete)_[name]
         # module.(none|read|write|admin)
         log.info('creating permissions')
         PermissionModel(self.sa).create_permissions()
 
     def populate_default_permissions(self):
         """
         Populate default permissions. It will create only the default
         permissions that are missing, and not alter already defined ones
         """
         log.info('creating default user permissions')
         PermissionModel(self.sa).create_default_permissions(user=User.DEFAULT_USER)
 
     @staticmethod
     def check_waitress():
         """
         Function executed at the end of setup
         """
         if not __py_version__ >= (2, 6):
             notify('Python2.5 detected, please switch '
                    'egg:waitress#main -> egg:Paste#http '
                    'in your .ini file')

kallithea/lib/indexers/__init__.py

@@ -1,197 +1,197 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.indexers
 ~~~~~~~~~~~~~~~~~~~~~~
 
 Whoosh indexing module for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Aug 17, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import os
 import sys
 import logging
-from os.path import dirname as dn
+from os.path import dirname
 
 # Add location of top level folder to sys.path
-sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
+sys.path.append(dirname(dirname(dirname(os.path.realpath(__file__)))))
 
 from whoosh.analysis import RegexTokenizer, LowercaseFilter
 from whoosh.fields import TEXT, ID, STORED, NUMERIC, BOOLEAN, Schema, FieldType, DATETIME
 from whoosh.formats import Characters
 from whoosh.highlight import highlight as whoosh_highlight, HtmlFormatter, ContextFragmenter
 from kallithea.lib.utils2 import LazyProperty
 
 log = logging.getLogger(__name__)
 
 # CUSTOM ANALYZER wordsplit + lowercase filter
 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
 
 #INDEX SCHEMA DEFINITION
 SCHEMA = Schema(
     fileid=ID(unique=True),
     owner=TEXT(),
     repository=TEXT(stored=True),
     path=TEXT(stored=True),
     content=FieldType(format=Characters(), analyzer=ANALYZER,
                       scorable=True, stored=True),
     modtime=STORED(),
     extension=TEXT(stored=True)
 )
 
 IDX_NAME = 'HG_INDEX'
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = ContextFragmenter(200)
 
 CHGSETS_SCHEMA = Schema(
     raw_id=ID(unique=True, stored=True),
     date=NUMERIC(stored=True),
     last=BOOLEAN(),
     owner=TEXT(),
     repository=ID(unique=True, stored=True),
     author=TEXT(stored=True),
     message=FieldType(format=Characters(), analyzer=ANALYZER,
                       scorable=True, stored=True),
     parents=TEXT(),
     added=TEXT(),
     removed=TEXT(),
     changed=TEXT(),
 )
 
 CHGSET_IDX_NAME = 'CHGSET_INDEX'
 
 # used only to generate queries in journal
 JOURNAL_SCHEMA = Schema(
     username=TEXT(),
     date=DATETIME(),
     action=TEXT(),
     repository=TEXT(),
     ip=TEXT(),
 )
 
 
 class WhooshResultWrapper(object):
     def __init__(self, search_type, searcher, matcher, highlight_items,
                  repo_location):
         self.search_type = search_type
         self.searcher = searcher
         self.matcher = matcher
         self.highlight_items = highlight_items
         self.fragment_size = 200
         self.repo_location = repo_location
 
     @LazyProperty
     def doc_ids(self):
         docs_id = []
         while self.matcher.is_active():
             docnum = self.matcher.id()
             chunks = [offsets for offsets in self.get_chunks()]
             docs_id.append([docnum, chunks])
             self.matcher.next()
         return docs_id
 
     def __str__(self):
         return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids))
 
     def __repr__(self):
         return self.__str__()
 
     def __len__(self):
         return len(self.doc_ids)
 
     def __iter__(self):
         """
         Allows Iteration over results,and lazy generate content
 
         *Requires* implementation of ``__getitem__`` method.
         """
         for docid in self.doc_ids:
             yield self.get_full_content(docid)
 
     def __getitem__(self, key):
         """
         Slicing of resultWrapper
         """
         i, j = key.start, key.stop
 
         slices = []
         for docid in self.doc_ids[i:j]:
             slices.append(self.get_full_content(docid))
         return slices
 
     def get_full_content(self, docid):
         res = self.searcher.stored_fields(docid[0])
         log.debug('result: %s', res)
         if self.search_type == 'content':
             full_repo_path = os.path.join(self.repo_location, res['repository'])
             f_path = res['path'].split(full_repo_path)[-1]
             f_path = f_path.lstrip(os.sep)
             content_short = self.get_short_content(res, docid[1])
             res.update({'content_short': content_short,
                         'content_short_hl': self.highlight(content_short),
                         'f_path': f_path
             })
         elif self.search_type == 'path':
             full_repo_path = os.path.join(self.repo_location, res['repository'])
             f_path = res['path'].split(full_repo_path)[-1]
             f_path = f_path.lstrip(os.sep)
             res.update({'f_path': f_path})
         elif self.search_type == 'message':
             res.update({'message_hl': self.highlight(res['message'])})
 
         log.debug('result: %s', res)
 
         return res
 
     def get_short_content(self, res, chunks):
 
         return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks])
 
     def get_chunks(self):
         """
         Smart function that implements chunking the content
         but not overlap chunks so it doesn't highlight the same
         close occurrences twice.
         """
         memory = [(0, 0)]
         if self.matcher.supports('positions'):
             for span in self.matcher.spans():
                 start = span.startchar or 0
                 end = span.endchar or 0
                 start_offseted = max(0, start - self.fragment_size)
                 end_offseted = end + self.fragment_size
 
                 if start_offseted < memory[-1][1]:
                     start_offseted = memory[-1][1]
                 memory.append((start_offseted, end_offseted,))
                 yield (start_offseted, end_offseted,)
 
     def highlight(self, content, top=5):
         if self.search_type not in ['content', 'message']:
             return ''
         hl = whoosh_highlight(
             text=content,
             terms=self.highlight_items,
             analyzer=ANALYZER,
             fragmenter=FRAGMENTER,
             formatter=FORMATTER,
             top=top
         )
         return hl

kallithea/lib/indexers/daemon.py

@@ -1,459 +1,459 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.indexers.daemon
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 A daemon will read from task table and run tasks
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jan 26, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import os
 import sys
 import logging
 import traceback
 
 from shutil import rmtree
 from time import mktime
 
-from os.path import dirname as dn
+from os.path import dirname
 
 # Add location of top level folder to sys.path
-project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
+project_path = dirname(dirname(dirname(dirname(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 
 from kallithea.config.conf import INDEX_EXTENSIONS, INDEX_FILENAMES
 from kallithea.model.scm import ScmModel
 from kallithea.model.db import Repository
 from kallithea.lib.utils2 import safe_unicode, safe_str
 from kallithea.lib.indexers import SCHEMA, IDX_NAME, CHGSETS_SCHEMA, \
     CHGSET_IDX_NAME
 
 from kallithea.lib.vcs.exceptions import ChangesetError, RepositoryError, \
     NodeDoesNotExistError
 
 from whoosh.index import create_in, open_dir, exists_in
 from whoosh.query import *
 from whoosh.qparser import QueryParser
 
 log = logging.getLogger('whoosh_indexer')
 
 
 class WhooshIndexingDaemon(object):
     """
     Daemon for atomic indexing jobs
     """
 
     def __init__(self, indexname=IDX_NAME, index_location=None,
                  repo_location=None, sa=None, repo_list=None,
                  repo_update_list=None):
         self.indexname = indexname
 
         self.index_location = index_location
         if not index_location:
             raise Exception('You have to provide index location')
 
         self.repo_location = repo_location
         if not repo_location:
             raise Exception('You have to provide repositories location')
 
         self.repo_paths = ScmModel(sa).repo_scan(self.repo_location)
 
         #filter repo list
         if repo_list:
             #Fix non-ascii repo names to unicode
             repo_list = map(safe_unicode, repo_list)
             self.filtered_repo_paths = {}
             for repo_name, repo in self.repo_paths.items():
                 if repo_name in repo_list:
                     self.filtered_repo_paths[repo_name] = repo
 
             self.repo_paths = self.filtered_repo_paths
 
         #filter update repo list
         self.filtered_repo_update_paths = {}
         if repo_update_list:
             self.filtered_repo_update_paths = {}
             for repo_name, repo in self.repo_paths.items():
                 if repo_name in repo_update_list:
                     self.filtered_repo_update_paths[repo_name] = repo
             self.repo_paths = self.filtered_repo_update_paths
 
         self.initial = True
         if not os.path.isdir(self.index_location):
             os.makedirs(self.index_location)
             log.info('Cannot run incremental index since it does not '
                      'yet exist running full build')
         elif not exists_in(self.index_location, IDX_NAME):
             log.info('Running full index build as the file content '
                      'index does not exist')
         elif not exists_in(self.index_location, CHGSET_IDX_NAME):
             log.info('Running full index build as the changeset '
                      'index does not exist')
         else:
             self.initial = False
 
     def _get_index_revision(self, repo):
         db_repo = Repository.get_by_repo_name(repo.name_unicode)
         landing_rev = 'tip'
         if db_repo:
             _rev_type, _rev = db_repo.landing_rev
             landing_rev = _rev
         return landing_rev
 
     def _get_index_changeset(self, repo, index_rev=None):
         if not index_rev:
             index_rev = self._get_index_revision(repo)
         cs = repo.get_changeset(index_rev)
         return cs
 
     def get_paths(self, repo):
         """
         recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
         """
         index_paths_ = set()
         try:
             cs = self._get_index_changeset(repo)
             for _topnode, _dirs, files in cs.walk('/'):
                 for f in files:
                     index_paths_.add(os.path.join(safe_str(repo.path), safe_str(f.path)))
 
         except RepositoryError:
             log.debug(traceback.format_exc())
             pass
         return index_paths_
 
     def get_node(self, repo, path, index_rev=None):
         """
         gets a filenode based on given full path. It operates on string for
         hg git compatibility.
 
         :param repo: scm repo instance
         :param path: full path including root location
         :return: FileNode
         """
         # FIXME: paths should be normalized ... or even better: don't include repo.path
         path = safe_str(path)
         repo_path = safe_str(repo.path)
         assert path.startswith(repo_path)
         assert path[len(repo_path)] in (os.path.sep, os.path.altsep)
         node_path = path[len(repo_path) + 1:]
         cs = self._get_index_changeset(repo, index_rev=index_rev)
         node = cs.get_node(node_path)
         return node
 
     def is_indexable_node(self, node):
         """
         Just index the content of chosen files, skipping binary files
         """
         return (node.extension in INDEX_EXTENSIONS or node.name in INDEX_FILENAMES) and \
                not node.is_binary
 
     def get_node_mtime(self, node):
         return mktime(node.last_changeset.date.timetuple())
 
     def add_doc(self, writer, path, repo, repo_name, index_rev=None):
         """
         Adding doc to writer this function itself fetches data from
         the instance of vcs backend
         """
         try:
             node = self.get_node(repo, path, index_rev)
         except (ChangesetError, NodeDoesNotExistError):
             log.debug("couldn't add doc - %s did not have %r at %s", repo, path, index_rev)
             return 0, 0
 
         indexed = indexed_w_content = 0
         if self.is_indexable_node(node):
             u_content = node.content
             if not isinstance(u_content, unicode):
                 log.warning('  >> %s Could not get this content as unicode '
                             'replacing with empty content' % path)
                 u_content = u''
             else:
                 log.debug('    >> %s [WITH CONTENT]', path)
                 indexed_w_content += 1
 
         else:
             log.debug('    >> %s', path)
             # just index file name without it's content
             u_content = u''
             indexed += 1
 
         p = safe_unicode(path)
         writer.add_document(
             fileid=p,
             owner=unicode(repo.contact),
             repository=safe_unicode(repo_name),
             path=p,
             content=u_content,
             modtime=self.get_node_mtime(node),
             extension=node.extension
         )
         return indexed, indexed_w_content
 
     def index_changesets(self, writer, repo_name, repo, start_rev=None):
         """
         Add all changeset in the vcs repo starting at start_rev
         to the index writer
 
         :param writer: the whoosh index writer to add to
         :param repo_name: name of the repository from whence the
           changeset originates including the repository group
         :param repo: the vcs repository instance to index changesets for,
           the presumption is the repo has changesets to index
         :param start_rev=None: the full sha id to start indexing from
           if start_rev is None then index from the first changeset in
           the repo
         """
 
         if start_rev is None:
             start_rev = repo[0].raw_id
 
         log.debug('indexing changesets in %s starting at rev: %s',
                   repo_name, start_rev)
 
         indexed = 0
         cs_iter = repo.get_changesets(start=start_rev)
         total = len(cs_iter)
         for cs in cs_iter:
             log.debug('    >> %s/%s', cs, total)
             writer.add_document(
                 raw_id=unicode(cs.raw_id),
                 owner=unicode(repo.contact),
                 date=cs._timestamp,
                 repository=safe_unicode(repo_name),
                 author=cs.author,
                 message=cs.message,
                 last=cs.last,
                 added=u' '.join([safe_unicode(node.path) for node in cs.added]).lower(),
                 removed=u' '.join([safe_unicode(node.path) for node in cs.removed]).lower(),
                 changed=u' '.join([safe_unicode(node.path) for node in cs.changed]).lower(),
                 parents=u' '.join([cs.raw_id for cs in cs.parents]),
             )
             indexed += 1
 
         log.debug('indexed %d changesets for repo %s', indexed, repo_name)
         return indexed
 
     def index_files(self, file_idx_writer, repo_name, repo):
         """
         Index files for given repo_name
 
         :param file_idx_writer: the whoosh index writer to add to
         :param repo_name: name of the repository we're indexing
         :param repo: instance of vcs repo
         """
         i_cnt = iwc_cnt = 0
         log.debug('building index for %s @revision:%s', repo.path,
                                                 self._get_index_revision(repo))
         index_rev = self._get_index_revision(repo)
         for idx_path in self.get_paths(repo):
             i, iwc = self.add_doc(file_idx_writer, idx_path, repo, repo_name, index_rev)
             i_cnt += i
             iwc_cnt += iwc
 
         log.debug('added %s files %s with content for repo %s',
                   i_cnt + iwc_cnt, iwc_cnt, repo.path)
         return i_cnt, iwc_cnt
 
     def update_changeset_index(self):
         idx = open_dir(self.index_location, indexname=CHGSET_IDX_NAME)
 
         with idx.searcher() as searcher:
             writer = idx.writer()
             writer_is_dirty = False
             try:
                 indexed_total = 0
                 repo_name = None
                 for repo_name, repo in self.repo_paths.items():
                     # skip indexing if there aren't any revs in the repo
                     num_of_revs = len(repo)
                     if num_of_revs < 1:
                         continue
 
                     qp = QueryParser('repository', schema=CHGSETS_SCHEMA)
                     q = qp.parse(u"last:t AND %s" % repo_name)
 
                     results = searcher.search(q)
 
                     # default to scanning the entire repo
                     last_rev = 0
                     start_id = None
 
                     if len(results) > 0:
                         # assuming that there is only one result, if not this
                         # may require a full re-index.
                         start_id = results[0]['raw_id']
                         last_rev = repo.get_changeset(revision=start_id).revision
 
                     # there are new changesets to index or a new repo to index
                     if last_rev == 0 or num_of_revs > last_rev + 1:
                         # delete the docs in the index for the previous
                         # last changeset(s)
                         for hit in results:
                             q = qp.parse(u"last:t AND %s AND raw_id:%s" %
                                             (repo_name, hit['raw_id']))
                             writer.delete_by_query(q)
 
                         # index from the previous last changeset + all new ones
                         indexed_total += self.index_changesets(writer,
                                                 repo_name, repo, start_id)
                         writer_is_dirty = True
                 log.debug('indexed %s changesets for repo %s',
                           indexed_total, repo_name
                 )
             finally:
                 if writer_is_dirty:
                     log.debug('>> COMMITING CHANGES TO CHANGESET INDEX<<')
                     writer.commit(merge=True)
                     log.debug('>>> FINISHED REBUILDING CHANGESET INDEX <<<')
                 else:
                     log.debug('>> NOTHING TO COMMIT TO CHANGESET INDEX<<')
 
     def update_file_index(self):
         log.debug((u'STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s '
                    'AND REPOS %s') % (INDEX_EXTENSIONS, self.repo_paths.keys()))
 
         idx = open_dir(self.index_location, indexname=self.indexname)
         # The set of all paths in the index
         indexed_paths = set()
         # The set of all paths we need to re-index
         to_index = set()
 
         writer = idx.writer()
         writer_is_dirty = False
         try:
             with idx.reader() as reader:
 
                 # Loop over the stored fields in the index
                 for fields in reader.all_stored_fields():
                     indexed_path = fields['path']
                     indexed_repo_path = fields['repository']
                     indexed_paths.add(indexed_path)
 
                     if not indexed_repo_path in self.filtered_repo_update_paths:
                         continue
 
                     repo = self.repo_paths[indexed_repo_path]
 
                     try:
                         node = self.get_node(repo, indexed_path)
                         # Check if this file was changed since it was indexed
                         indexed_time = fields['modtime']
                         mtime = self.get_node_mtime(node)
                         if mtime > indexed_time:
                             # The file has changed, delete it and add it to
                             # the list of files to reindex
                             log.debug(
                                 'adding to reindex list %s mtime: %s vs %s',
                                     indexed_path, mtime, indexed_time
                             )
                             writer.delete_by_term('fileid', indexed_path)
                             writer_is_dirty = True
 
                             to_index.add(indexed_path)
                     except (ChangesetError, NodeDoesNotExistError):
                         # This file was deleted since it was indexed
                         log.debug('removing from index %s', indexed_path)
                         writer.delete_by_term('path', indexed_path)
                         writer_is_dirty = True
 
             # Loop over the files in the filesystem
             # Assume we have a function that gathers the filenames of the
             # documents to be indexed
             ri_cnt_total = 0  # indexed
             riwc_cnt_total = 0  # indexed with content
             for repo_name, repo in self.repo_paths.items():
                 # skip indexing if there aren't any revisions
                 if len(repo) < 1:
                     continue
                 ri_cnt = 0   # indexed
                 riwc_cnt = 0  # indexed with content
                 for path in self.get_paths(repo):
                     path = safe_unicode(path)
                     if path in to_index or path not in indexed_paths:
 
                         # This is either a file that's changed, or a new file
                         # that wasn't indexed before. So index it!
                         i, iwc = self.add_doc(writer, path, repo, repo_name)
                         writer_is_dirty = True
                         log.debug('re indexing %s', path)
                         ri_cnt += i
                         ri_cnt_total += 1
                         riwc_cnt += iwc
                         riwc_cnt_total += iwc
                 log.debug('added %s files %s with content for repo %s',
                              ri_cnt + riwc_cnt, riwc_cnt, repo.path
                 )
             log.debug('indexed %s files in total and %s with content',
                         ri_cnt_total, riwc_cnt_total
             )
         finally:
             if writer_is_dirty:
                 log.debug('>> COMMITING CHANGES TO FILE INDEX <<')
                 writer.commit(merge=True)
                 log.debug('>>> FINISHED REBUILDING FILE INDEX <<<')
             else:
                 log.debug('>> NOTHING TO COMMIT TO FILE INDEX <<')
                 writer.cancel()
 
     def build_indexes(self):
         if os.path.exists(self.index_location):
             log.debug('removing previous index')
             rmtree(self.index_location)
 
         if not os.path.exists(self.index_location):
             os.mkdir(self.index_location)
 
         chgset_idx = create_in(self.index_location, CHGSETS_SCHEMA,
                                indexname=CHGSET_IDX_NAME)
         chgset_idx_writer = chgset_idx.writer()
 
         file_idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)
         file_idx_writer = file_idx.writer()
         log.debug('BUILDING INDEX FOR EXTENSIONS %s '
                   'AND REPOS %s' % (INDEX_EXTENSIONS, self.repo_paths.keys()))
 
         for repo_name, repo in self.repo_paths.items():
             # skip indexing if there aren't any revisions
             if len(repo) < 1:
                 continue
 
             self.index_files(file_idx_writer, repo_name, repo)
             self.index_changesets(chgset_idx_writer, repo_name, repo)
 
         log.debug('>> COMMITING CHANGES <<')
         file_idx_writer.commit(merge=True)
         chgset_idx_writer.commit(merge=True)
         log.debug('>>> FINISHED BUILDING INDEX <<<')
 
     def update_indexes(self):
         self.update_file_index()
         self.update_changeset_index()
 
     def run(self, full_index=False):
         """Run daemon"""
         if full_index or self.initial:
             self.build_indexes()
         else:
             self.update_indexes()

kallithea/lib/paster_commands/cache_keys.py

@@ -1,83 +1,83 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.paster_commands.cache_keys
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 cleanup-keys paster command for Kallithea
 
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: mar 27, 2013
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import os
 import sys
 
 from kallithea.model.meta import Session
 from kallithea.lib.utils import BasePasterCommand
 from kallithea.model.db import CacheInvalidation
 
 # Add location of top level folder to sys.path
-from os.path import dirname as dn
-rc_path = dn(dn(dn(os.path.realpath(__file__))))
+from os.path import dirname
+rc_path = dirname(dirname(dirname(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     group_name = "Kallithea"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Cache keys utils"
 
     def command(self):
         #get SqlAlchemy session
         self._init_session()
 
         _caches = CacheInvalidation.query().order_by(CacheInvalidation.cache_key).all()
         if self.options.show:
             for c_obj in _caches:
                 print 'key:%s active:%s' % (c_obj.cache_key, c_obj.cache_active)
         elif self.options.cleanup:
             for c_obj in _caches:
                 Session().delete(c_obj)
                 print 'Removing key: %s' % (c_obj.cache_key)
                 Session().commit()
         else:
             print 'Nothing done, exiting...'
 
     def update_parser(self):
         self.parser.add_option(
             '--show',
             action='store_true',
             dest='show',
             help=("show existing cache keys with together with status")
         )
 
         self.parser.add_option(
             '--cleanup',
             action="store_true",
             dest="cleanup",
             help="cleanup existing cache keys"
         )

kallithea/lib/paster_commands/cleanup.py

@@ -1,149 +1,149 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.paster_commands.cleanup
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 cleanup-repos paster command for Kallithea
 
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jul 14, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import os
 import sys
 import re
 import shutil
 import datetime
 
 from kallithea.lib.utils import BasePasterCommand, ask_ok, REMOVED_REPO_PAT
 from kallithea.lib.utils2 import safe_str
 from kallithea.model.db import Ui
 
 # Add location of top level folder to sys.path
-from os.path import dirname as dn
-rc_path = dn(dn(dn(os.path.realpath(__file__))))
+from os.path import dirname
+rc_path = dirname(dirname(dirname(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     group_name = "Kallithea"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Cleanup deleted repos"
 
     def _parse_older_than(self, val):
         regex = re.compile(r'((?P<days>\d+?)d)?((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?')
         parts = regex.match(val)
         if not parts:
             return
         parts = parts.groupdict()
         time_params = {}
         for (name, param) in parts.iteritems():
             if param:
                 time_params[name] = int(param)
         return datetime.timedelta(**time_params)
 
     def _extract_date(self, name):
         """
         Extract the date part from rm__<date> pattern of removed repos,
         and convert it to datetime object
 
         :param name:
         """
         date_part = name[4:19]  # 4:19 since we don't parse milliseconds
         return datetime.datetime.strptime(date_part, '%Y%m%d_%H%M%S')
 
     def command(self):
         #get SqlAlchemy session
         self._init_session()
 
         repos_location = Ui.get_repos_location()
         to_remove = []
         for dn_, dirs, f in os.walk(safe_str(repos_location)):
             alldirs = list(dirs)
             del dirs[:]
             if ('.hg' in alldirs or
                 'objects' in alldirs and ('refs' in alldirs or 'packed-refs' in f)):
                 continue
             for loc in alldirs:
                 if REMOVED_REPO_PAT.match(loc):
                     to_remove.append([os.path.join(dn_, loc),
                                       self._extract_date(loc)])
                 else:
                     dirs.append(loc)
 
         #filter older than (if present)!
         now = datetime.datetime.now()
         older_than = self.options.older_than
         if older_than:
             to_remove_filtered = []
             older_than_date = self._parse_older_than(older_than)
             for name, date_ in to_remove:
                 repo_age = now - date_
                 if repo_age > older_than_date:
                     to_remove_filtered.append([name, date_])
 
             to_remove = to_remove_filtered
             print 'Removing %s deleted repos older than %s (%s)' \
                 % (len(to_remove), older_than, older_than_date)
         else:
             print 'Removing all %s deleted repos' % len(to_remove)
         if self.options.dont_ask or not to_remove:
             # don't ask just remove !
             remove = True
         else:
             remove = ask_ok('the following repositories will be deleted completely:\n%s\n'
                             'are you sure you want to remove them [y/n]?'
                             % '\n'.join(['%s removed on %s' % (safe_str(x[0]), safe_str(x[1]))
                                          for x in to_remove]))
 
         if remove:
             for path, date_ in to_remove:
                 print 'Removing repository %s' % path
                 shutil.rmtree(path)
         else:
             print 'Nothing done, exiting...'
 
     def update_parser(self):
         self.parser.add_option(
             '--older-than',
             action='store',
             dest='older_than',
             help=("only remove repos that have been removed "
                  "at least given time ago. "
                  "The default is to remove all removed repositories. "
                  "Possible suffixes: "
                  "d (days), h (hours), m (minutes), s (seconds). "
                  "For example --older-than=30d deletes repositories "
                  "removed more than 30 days ago.")
             )
 
         self.parser.add_option(
             '--dont-ask',
             action="store_true",
             dest="dont_ask",
             help="remove repositories without asking for confirmation."
         )

kallithea/lib/paster_commands/install_iis.py

@@ -1,108 +1,108 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.paster_commands.install_iis
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 IIS installation tools for Kallithea
 """
 
 
 import os
 import sys
 from paste.script.appinstall import AbstractInstallCommand
 from paste.script.command import BadCommand
 
 # Add location of top level folder to sys.path
-from os.path import dirname as dn
-rc_path = dn(dn(dn(os.path.realpath(__file__))))
+from os.path import dirname
+rc_path = dirname(dirname(dirname(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 class Command(AbstractInstallCommand):
     default_verbosity = 1
     max_args = 1
     min_args = 1
     summary = 'Setup IIS given a config file'
     usage = 'CONFIG_FILE'
 
     description = '''
     Script for installing into IIS using isapi-wsgi.
     '''
     parser = AbstractInstallCommand.standard_parser(
         simulate=True, quiet=True, interactive=True)
     parser.add_option('--virtualdir',
                       action='store',
                       dest='virtualdir',
                       default='/',
                       help='The virtual folder to install into on IIS')
 
     def command(self):
         config_spec = self.args[0]
         if not config_spec.startswith('config:'):
             config_spec = 'config:' + config_spec
         config_file = config_spec[len('config:'):].split('#', 1)[0]
         config_file = os.path.join(os.getcwd(), config_file)
         try:
             import isapi_wsgi
         except ImportError:
             raise BadCommand('missing requirement: isapi-wsgi not installed')
 
         file = '''\
 # Created by Kallithea install_iis
 import sys
 
 if hasattr(sys, "isapidllhandle"):
     import win32traceutil
 
 import isapi_wsgi
 import os
 
 def __ExtensionFactory__():
     from paste.deploy import loadapp
     from paste.script.util.logging_config import fileConfig
     fileConfig('%(inifile)s')
     application = loadapp('config:%(inifile)s')
 
     def app(environ, start_response):
         user = environ.get('REMOTE_USER', None)
         if user is not None:
             os.environ['REMOTE_USER'] = user
         return application(environ, start_response)
 
     return isapi_wsgi.ISAPIThreadPoolHandler(app)
 
 if __name__=='__main__':
     from isapi.install import *
     params = ISAPIParameters()
     sm = [ScriptMapParams(Extension="*", Flags=0)]
     vd = VirtualDirParameters(Name="%(virtualdir)s",
                               Description = "Kallithea",
                               ScriptMaps = sm,
                               ScriptMapUpdate = "replace")
     params.VirtualDirs = [vd]
     HandleCommandLine(params)
 '''
 
         outdata = file % {
                 'inifile': config_file.replace('\\', '\\\\'),
                 'virtualdir': self.options.virtualdir
                 }
 
         dispatchfile = os.path.join(os.getcwd(), 'dispatch.py')
         self.ensure_file(dispatchfile, outdata, False)
         print 'Generating %s' % (dispatchfile,)
 
         print ('Run \'python "%s" install\' with administrative privileges '
             'to generate the _dispatch.dll file and install it into the '
             'default web site') % (dispatchfile,)

kallithea/lib/paster_commands/ishell.py

@@ -1,74 +1,74 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.paster_commands.ishell
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 interactive shell paster command for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 4, 2013
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import os
 import sys
 
 from kallithea.lib.utils import BasePasterCommand
 
 # Add location of top level folder to sys.path
-from os.path import dirname as dn
-rc_path = dn(dn(dn(os.path.realpath(__file__))))
+from os.path import dirname
+rc_path = dirname(dirname(dirname(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     group_name = "Kallithea"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Interactive shell"
 
     def command(self):
         #get SqlAlchemy session
         self._init_session()
 
         # imports, used in IPython shell
         import os
         import sys
         import time
         import shutil
         import datetime
         from kallithea.model.db import *
 
         try:
             from IPython import embed
             from IPython.config.loader import Config
             cfg = Config()
             cfg.InteractiveShellEmbed.confirm_exit = False
             embed(config=cfg, banner1="Kallithea IShell.")
         except ImportError:
             print 'IPython installation is required for ishell'
             sys.exit(-1)
 
     def update_parser(self):
         pass

kallithea/lib/paster_commands/make_index.py

@@ -1,109 +1,109 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.paster_commands.make_index
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 make-index paster command for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Aug 17, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import os
 import sys
 
 from string import strip
 from kallithea.model.repo import RepoModel
 from kallithea.lib.utils import BasePasterCommand, load_rcextensions
 
 # Add location of top level folder to sys.path
-from os.path import dirname as dn
-rc_path = dn(dn(dn(os.path.realpath(__file__))))
+from os.path import dirname
+rc_path = dirname(dirname(dirname(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     group_name = "Kallithea"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Creates or updates full text search index"
 
     def command(self):
         #get SqlAlchemy session
         self._init_session()
         from pylons import config
         index_location = config['index_dir']
         load_rcextensions(config['here'])
 
         repo_location = self.options.repo_location \
             if self.options.repo_location else RepoModel().repos_path
         repo_list = map(strip, self.options.repo_list.split(',')) \
             if self.options.repo_list else None
 
         repo_update_list = map(strip, self.options.repo_update_list.split(',')) \
             if self.options.repo_update_list else None
 
         #======================================================================
         # WHOOSH DAEMON
         #======================================================================
         from kallithea.lib.pidlock import LockHeld, DaemonLock
         from kallithea.lib.indexers.daemon import WhooshIndexingDaemon
         try:
-            l = DaemonLock(file_=os.path.join(dn(dn(index_location)),
+            l = DaemonLock(file_=os.path.join(dirname(dirname(index_location)),
                                               'make_index.lock'))
             WhooshIndexingDaemon(index_location=index_location,
                                  repo_location=repo_location,
                                  repo_list=repo_list,
                                  repo_update_list=repo_update_list) \
                 .run(full_index=self.options.full_index)
             l.release()
         except LockHeld:
             sys.exit(1)
 
     def update_parser(self):
         self.parser.add_option('--repo-location',
                           action='store',
                           dest='repo_location',
                           help="Specifies repositories location to index OPTIONAL",
                           )
         self.parser.add_option('--index-only',
                           action='store',
                           dest='repo_list',
                           help="Specifies a comma separated list of repositories "
                                 "to build index on. If not given all repositories "
                                 "are scanned for indexing. OPTIONAL",
                           )
         self.parser.add_option('--update-only',
                           action='store',
                           dest='repo_update_list',
                           help="Specifies a comma separated list of repositories "
                                 "to re-build index on. OPTIONAL",
                           )
         self.parser.add_option('-f',
                           action='store_true',
                           dest='full_index',
                           help="Specifies that index should be made full i.e"
                                 " destroy old and build from scratch",
                           default=False)

kallithea/lib/paster_commands/make_rcextensions.py

@@ -1,80 +1,80 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.paster_commands.make_rcextensions
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 make-rcext paster command for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Mar 6, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import os
 import sys
 import pkg_resources
 
 from kallithea.lib.utils import BasePasterCommand, ask_ok
 
 # Add location of top level folder to sys.path
-from os.path import dirname as dn
-rc_path = dn(dn(dn(os.path.realpath(__file__))))
+from os.path import dirname
+rc_path = dirname(dirname(dirname(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     group_name = "Kallithea"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Write template file for extending Kallithea in Python."
     usage = "CONFIG_FILE"
     description = '''\
         A rcextensions directory with a __init__.py file will be created next to
         the ini file. Local customizations in that file will survive upgrades.
         The file contains instructions on how it can be customized.
         '''
 
     def command(self):
         from pylons import config
 
         here = config['here']
         content = pkg_resources.resource_string(
             'kallithea', os.path.join('config', 'rcextensions', '__init__.py')
         )
         ext_file = os.path.join(here, 'rcextensions', '__init__.py')
         if os.path.exists(ext_file):
             msg = ('Extension file already exists, do you want '
                    'to overwrite it ? [y/n]')
             if not ask_ok(msg):
                 print 'Nothing done, exiting...'
                 return
 
         dirname = os.path.dirname(ext_file)
         if not os.path.isdir(dirname):
             os.makedirs(dirname)
         with open(ext_file, 'wb') as f:
             f.write(content)
             print 'Wrote new extensions file to %s' % ext_file
 
     def update_parser(self):
         pass

kallithea/lib/paster_commands/repo_scan.py

@@ -1,69 +1,69 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.paster_commands.repo_scan
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 repo-scan paster command for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Feb 9, 2013
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import os
 import sys
 
 from kallithea.model.scm import ScmModel
 from kallithea.lib.utils import BasePasterCommand, repo2db_mapper
 
 # Add location of top level folder to sys.path
-from os.path import dirname as dn
-rc_path = dn(dn(dn(os.path.realpath(__file__))))
+from os.path import dirname
+rc_path = dirname(dirname(dirname(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     group_name = "Kallithea"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Rescan default location for new repositories"
 
     def command(self):
         #get SqlAlchemy session
         self._init_session()
         rm_obsolete = self.options.delete_obsolete
         print 'Now scanning root location for new repos ...'
         added, removed = repo2db_mapper(ScmModel().repo_scan(),
                                         remove_obsolete=rm_obsolete)
         added = ', '.join(added) or '-'
         removed = ', '.join(removed) or '-'
         print 'Scan completed added: %s removed: %s' % (added, removed)
 
     def update_parser(self):
         self.parser.add_option(
             '--delete-obsolete',
             action='store_true',
             help="Use this flag do delete repositories that are "
                  "present in Kallithea database but not on the filesystem",
         )

kallithea/lib/paster_commands/setup_db.py

@@ -1,131 +1,131 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.paster_commands.setup_db
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Databaset setup paster command for Kallithea
 """
 
 
 import os
 import sys
 from paste.script.appinstall import AbstractInstallCommand
 from paste.script.command import BadCommand
 from paste.deploy import appconfig
 
 # Add location of top level folder to sys.path
-from os.path import dirname as dn
-rc_path = dn(dn(dn(os.path.realpath(__file__))))
+from os.path import dirname
+rc_path = dirname(dirname(dirname(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 
 class Command(AbstractInstallCommand):
 
     default_verbosity = 1
     max_args = 1
     min_args = 1
     summary = "Setup an application, given a config file"
     usage = "CONFIG_FILE"
     group_name = "Kallithea"
 
     description = """\
         Setup Kallithea according to its configuration file.  This is
         the second part of a two-phase web application installation
         process (the first phase is prepare-app).  The setup process
         consist of things like setting up databases, creating super user
     """
 
     parser = AbstractInstallCommand.standard_parser(
         simulate=True, quiet=True, interactive=True)
     parser.add_option('--user',
                       action='store',
                       dest='username',
                       default=None,
                       help='Admin Username')
     parser.add_option('--email',
                       action='store',
                       dest='email',
                       default=None,
                       help='Admin Email')
     parser.add_option('--password',
                       action='store',
                       dest='password',
                       default=None,
                       help='Admin password min 6 chars')
     parser.add_option('--repos',
                       action='store',
                       dest='repos_location',
                       default=None,
                       help='Absolute path to repositories location')
     parser.add_option('--name',
                       action='store',
                       dest='section_name',
                       default=None,
                       help='The name of the section to set up (default: app:main)')
     parser.add_option('--force-yes',
                        action='store_true',
                        dest='force_ask',
                        default=None,
                        help='Force yes to every question')
     parser.add_option('--force-no',
                        action='store_false',
                        dest='force_ask',
                        default=None,
                        help='Force no to every question')
     parser.add_option('--public-access',
                        action='store_true',
                        dest='public_access',
                        default=None,
                        help='Enable public access on this installation (default)')
     parser.add_option('--no-public-access',
                        action='store_false',
                        dest='public_access',
                        default=None,
                        help='Disable public access on this installation ')
 
     def command(self):
         config_spec = self.args[0]
         section = self.options.section_name
         if section is None:
             if '#' in config_spec:
                 config_spec, section = config_spec.split('#', 1)
             else:
                 section = 'main'
         if not ':' in section:
             plain_section = section
             section = 'app:' + section
         else:
             plain_section = section.split(':', 1)[0]
         if not config_spec.startswith('config:'):
             config_spec = 'config:' + config_spec
         if plain_section != 'main':
             config_spec += '#' + plain_section
         config_file = config_spec[len('config:'):].split('#', 1)[0]
         config_file = os.path.join(os.getcwd(), config_file)
         self.logging_file_config(config_file)
         conf = appconfig(config_spec, relative_to=os.getcwd())
         ep_name = conf.context.entry_point_name
         ep_group = conf.context.protocol
         dist = conf.context.distribution
         if dist is None:
             raise BadCommand(
                 "The section %r is not the application (probably a filter). "
                 "You should add #section_name, where section_name is the "
                 "section that configures your application" % plain_section)
         installer = self.get_installer(dist, ep_group, ep_name)
         installer.setup_config(
             self, config_file, section, self.sysconfig_install_vars(installer))
         self.call_sysconfig_functions(
             'post_setup_hook', installer, config_file)

kallithea/lib/paster_commands/update_repoinfo.py

@@ -1,86 +1,86 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.paster_commands.update_repoinfo
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 update-repoinfo paster command for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jul 14, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import os
 import sys
 import string
 
 from kallithea.lib.utils import BasePasterCommand
 from kallithea.model.db import Repository
 from kallithea.model.repo import RepoModel
 from kallithea.model.meta import Session
 
 # Add location of top level folder to sys.path
-from os.path import dirname as dn
-rc_path = dn(dn(dn(os.path.realpath(__file__))))
+from os.path import dirname
+rc_path = dirname(dirname(dirname(os.path.realpath(__file__))))
 sys.path.append(rc_path)
 
 
 class Command(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     group_name = "Kallithea"
     takes_config_file = -1
     parser = BasePasterCommand.standard_parser(verbose=True)
     summary = "Updates repositories caches for last changeset"
 
     def command(self):
         #get SqlAlchemy session
         self._init_session()
 
         repo_update_list = map(string.strip,
                                self.options.repo_update_list.split(',')) \
                                if self.options.repo_update_list else None
 
         if repo_update_list is not None:
             repo_list = list(Repository.query() \
                 .filter(Repository.repo_name.in_(repo_update_list)))
         else:
             repo_list = Repository.getAll()
         RepoModel.update_repoinfo(repositories=repo_list)
         Session().commit()
 
         if self.options.invalidate_cache:
             for r in repo_list:
                 r.set_invalidate()
         print 'Updated cache for %s repositories' % (len(repo_list))
 
     def update_parser(self):
         self.parser.add_option('--update-only',
                            action='store',
                            dest='repo_update_list',
                            help="Specifies a comma separated list of repositories "
                                 "to update last commit info for. OPTIONAL")
         self.parser.add_option('--invalidate-cache',
                            action='store_true',
                            dest='invalidate_cache',
                            help="Trigger cache invalidation event for repos. "
                                 "OPTIONAL")

kallithea/lib/utils.py

@@ -1,875 +1,875 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.utils
 ~~~~~~~~~~~~~~~~~~~
 
 Utilities library for Kallithea
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 18, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 import os
 import re
 import logging
 import datetime
 import traceback
 import paste
 import beaker
 import tarfile
 import shutil
 import decorator
 import warnings
 from os.path import abspath
-from os.path import dirname as dn
+from os.path import dirname
 
 from paste.script.command import Command, BadCommand
 
 from webhelpers.text import collapse, remove_formatting, strip_tags
 from beaker.cache import _cache_decorate
 
 from kallithea import BRAND
 
 from kallithea.lib.vcs.utils.hgcompat import ui, config
 from kallithea.lib.vcs.utils.helpers import get_scm
 from kallithea.lib.vcs.exceptions import VCSError
 
 from kallithea.model import meta
 from kallithea.model.db import Repository, User, Ui, \
     UserLog, RepoGroup, Setting, UserGroup
 from kallithea.model.meta import Session
 from kallithea.model.repo_group import RepoGroupModel
 from kallithea.lib.utils2 import safe_str, safe_unicode, get_current_authuser
 from kallithea.lib.vcs.utils.fakemod import create_module
 
 log = logging.getLogger(__name__)
 
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}_.*')
 
 
 def recursive_replace(str_, replace=' '):
     """
     Recursive replace of given sign to just one instance
 
     :param str_: given string
     :param replace: char to find and replace multiple instances
 
     Examples::
     >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
     'Mighty-Mighty-Bo-sstones'
     """
 
     if str_.find(replace * 2) == -1:
         return str_
     else:
         str_ = str_.replace(replace * 2, replace)
         return recursive_replace(str_, replace)
 
 
 def repo_name_slug(value):
     """
     Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
 
     slug = remove_formatting(value)
     slug = strip_tags(slug)
 
     for c in """`?=[]\;'"<>,/~!@#$%^&*()+{}|: """:
         slug = slug.replace(c, '-')
     slug = recursive_replace(slug, '-')
     slug = collapse(slug, '-')
     return slug
 
 
 #==============================================================================
 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 #==============================================================================
 def get_repo_slug(request):
     _repo = request.environ['pylons.routes_dict'].get('repo_name')
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 
 
 def get_repo_group_slug(request):
     _group = request.environ['pylons.routes_dict'].get('group_name')
     if _group:
         _group = _group.rstrip('/')
     return _group
 
 
 def get_user_group_slug(request):
     _group = request.environ['pylons.routes_dict'].get('id')
     _group = UserGroup.get(_group)
     if _group:
         return _group.users_group_name
     return None
 
 
 def _extract_id_from_repo_name(repo_name):
     if repo_name.startswith('/'):
         repo_name = repo_name.lstrip('/')
     by_id_match = re.match(r'^_(\d{1,})', repo_name)
     if by_id_match:
         return by_id_match.groups()[0]
 
 
 def get_repo_by_id(repo_name):
     """
     Extracts repo_name by id from special urls. Example url is _11/repo_name
 
     :param repo_name:
     :return: repo_name if matched else None
     """
     _repo_id = _extract_id_from_repo_name(repo_name)
     if _repo_id:
         from kallithea.model.db import Repository
         repo = Repository.get(_repo_id)
         if repo:
             # TODO: return repo instead of reponame? or would that be a layering violation?
             return repo.repo_name
     return None
 
 
 def action_logger(user, action, repo, ipaddr='', sa=None, commit=False):
     """
     Action logger for various actions made by users
 
     :param user: user that made this action, can be a unique username string or
         object containing user_id attribute
     :param action: action to log, should be on of predefined unique actions for
         easy translations
     :param repo: string name of repository or object containing repo_id,
         that action was made on
     :param ipaddr: optional IP address from what the action was made
     :param sa: optional sqlalchemy session
 
     """
 
     if not sa:
         sa = meta.Session()
     # if we don't get explicit IP address try to get one from registered user
     # in tmpl context var
     if not ipaddr:
         ipaddr = getattr(get_current_authuser(), 'ip_addr', '')
 
     if getattr(user, 'user_id', None):
         user_obj = User.get(user.user_id)
     elif isinstance(user, basestring):
         user_obj = User.get_by_username(user)
     else:
         raise Exception('You have to provide a user object or a username')
 
     if getattr(repo, 'repo_id', None):
         repo_obj = Repository.get(repo.repo_id)
         repo_name = repo_obj.repo_name
     elif isinstance(repo, basestring):
         repo_name = repo.lstrip('/')
         repo_obj = Repository.get_by_repo_name(repo_name)
     else:
         repo_obj = None
         repo_name = u''
 
     user_log = UserLog()
     user_log.user_id = user_obj.user_id
     user_log.username = user_obj.username
     user_log.action = safe_unicode(action)
 
     user_log.repository = repo_obj
     user_log.repository_name = repo_name
 
     user_log.action_date = datetime.datetime.now()
     user_log.user_ip = ipaddr
     sa.add(user_log)
 
     log.info('Logging action:%s on %s by user:%s ip:%s',
              action, safe_unicode(repo), user_obj, ipaddr)
     if commit:
         sa.commit()
 
 
 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
     """
     Scans given path for repos and return (name,(type,path)) tuple
 
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
 
     # remove ending slash for better results
     path = safe_str(path.rstrip(os.sep))
     log.debug('now scanning in %s location recursive:%s...', path, recursive)
 
     def _get_repos(p):
         if not os.access(p, os.R_OK) or not os.access(p, os.X_OK):
             log.warning('ignoring repo path without access: %s', p)
             return
         if not os.access(p, os.W_OK):
             log.warning('repo path without write access: %s', p)
         for dirpath in os.listdir(p):
             if os.path.isfile(os.path.join(p, dirpath)):
                 continue
             cur_path = os.path.join(p, dirpath)
 
             # skip removed repos
             if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
                 continue
 
             #skip .<something> dirs TODO: rly? then we should prevent creating them ...
             if dirpath.startswith('.'):
                 continue
 
             try:
                 scm_info = get_scm(cur_path)
                 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
             except VCSError:
                 if not recursive:
                     continue
                 #check if this dir contains other repos for recursive scan
                 rec_path = os.path.join(p, dirpath)
                 if not os.path.islink(rec_path) and os.path.isdir(rec_path):
                     for inner_scm in _get_repos(rec_path):
                         yield inner_scm
 
     return _get_repos(path)
 
 
 def is_valid_repo(repo_name, base_path, scm=None):
     """
     Returns True if given path is a valid repository False otherwise.
     If scm param is given also compare if given scm is the same as expected
     from scm parameter
 
     :param repo_name:
     :param base_path:
     :param scm:
 
     :return True: if given path is a valid repository
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
 
     try:
         scm_ = get_scm(full_path)
         if scm:
             return scm_[0] == scm
         return True
     except VCSError:
         return False
 
 
 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
     """
     Returns True if given path is a repository group False otherwise
 
     :param repo_name:
     :param base_path:
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
 
     # check if it's not a repo
     if is_valid_repo(repo_group_name, base_path):
         return False
 
     try:
         # we need to check bare git repos at higher level
         # since we might match branches/hooks/info/objects or possible
         # other things inside bare git repo
         get_scm(os.path.dirname(full_path))
         return False
     except VCSError:
         pass
 
     # check if it's a valid path
     if skip_path_check or os.path.isdir(full_path):
         return True
 
     return False
 
 
 def ask_ok(prompt, retries=4, complaint='Yes or no please!'):
     while True:
         ok = raw_input(prompt)
         if ok in ('y', 'ye', 'yes'):
             return True
         if ok in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise IOError
         print complaint
 
 #propagated from mercurial documentation
 ui_sections = ['alias', 'auth',
                 'decode/encode', 'defaults',
                 'diff', 'email',
                 'extensions', 'format',
                 'merge-patterns', 'merge-tools',
                 'hooks', 'http_proxy',
                 'smtp', 'patch',
                 'paths', 'profiling',
                 'server', 'trusted',
                 'ui', 'web', ]
 
 
 def make_ui(read_from='file', path=None, checkpaths=True, clear_session=True):
     """
     A function that will read python rc files or database
     and make an mercurial ui object from read options
 
     :param path: path to mercurial config file
     :param checkpaths: check the path
     :param read_from: read from 'file' or 'db'
     """
 
     baseui = ui.ui()
 
     # clean the baseui object
     baseui._ocfg = config.config()
     baseui._ucfg = config.config()
     baseui._tcfg = config.config()
 
     if read_from == 'file':
         if not os.path.isfile(path):
             log.debug('hgrc file is not present at %s, skipping...', path)
             return False
         log.debug('reading hgrc from %s', path)
         cfg = config.config()
         cfg.read(path)
         for section in ui_sections:
             for k, v in cfg.items(section):
                 log.debug('settings ui from file: [%s] %s=%s', section, k, v)
                 baseui.setconfig(safe_str(section), safe_str(k), safe_str(v))
 
     elif read_from == 'db':
         sa = meta.Session()
         ret = sa.query(Ui).all()
 
         hg_ui = ret
         for ui_ in hg_ui:
             if ui_.ui_active:
                 ui_val = safe_str(ui_.ui_value)
                 if ui_.ui_section == 'hooks' and BRAND != 'kallithea' and ui_val.startswith('python:' + BRAND + '.lib.hooks.'):
                     ui_val = ui_val.replace('python:' + BRAND + '.lib.hooks.', 'python:kallithea.lib.hooks.')
                 log.debug('settings ui from db: [%s] %s=%s', ui_.ui_section,
                           ui_.ui_key, ui_val)
                 baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),
                                  ui_val)
             if ui_.ui_key == 'push_ssl':
                 # force set push_ssl requirement to False, kallithea
                 # handles that
                 baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),
                                  False)
         if clear_session:
             meta.Session.remove()
 
         # prevent interactive questions for ssh password / passphrase
         ssh = baseui.config('ui', 'ssh', default='ssh')
         baseui.setconfig('ui', 'ssh', '%s -oBatchMode=yes -oIdentitiesOnly=yes' % ssh)
 
     return baseui
 
 
 def set_app_settings(config):
     """
     Updates pylons config with new settings from database
 
     :param config:
     """
     hgsettings = Setting.get_app_settings()
 
     for k, v in hgsettings.items():
         config[k] = v
 
 
 def set_vcs_config(config):
     """
     Patch VCS config with some Kallithea specific stuff
 
     :param config: kallithea.CONFIG
     """
     from kallithea.lib.vcs import conf
     from kallithea.lib.utils2 import aslist
     conf.settings.BACKENDS = {
         'hg': 'kallithea.lib.vcs.backends.hg.MercurialRepository',
         'git': 'kallithea.lib.vcs.backends.git.GitRepository',
     }
 
     conf.settings.GIT_EXECUTABLE_PATH = config.get('git_path', 'git')
     conf.settings.GIT_REV_FILTER = config.get('git_rev_filter', '--all').strip()
     conf.settings.DEFAULT_ENCODINGS = aslist(config.get('default_encoding',
                                                         'utf8'), sep=',')
 
 
 def set_indexer_config(config):
     """
     Update Whoosh index mapping
 
     :param config: kallithea.CONFIG
     """
     from kallithea.config import conf
 
     log.debug('adding extra into INDEX_EXTENSIONS')
     conf.INDEX_EXTENSIONS.extend(re.split('\s+', config.get('index.extensions', '')))
 
     log.debug('adding extra into INDEX_FILENAMES')
     conf.INDEX_FILENAMES.extend(re.split('\s+', config.get('index.filenames', '')))
 
 
 def map_groups(path):
     """
     Given a full path to a repository, create all nested groups that this
     repo is inside. This function creates parent-child relationships between
     groups and creates default perms for all new groups.
 
     :param paths: full path to repository
     """
     sa = meta.Session()
     groups = path.split(Repository.url_sep())
     parent = None
     group = None
 
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = RepoGroupModel(sa)
     owner = User.get_first_admin()
     for lvl, group_name in enumerate(groups):
         group_name = u'/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
 
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
 
         if group is None:
             log.debug('creating group level: %s group_name: %s',
                       lvl, group_name)
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             group.user = owner
             sa.add(group)
             perm_obj = rgm._create_default_perms(group)
             sa.add(perm_obj)
             sa.flush()
 
         parent = group
     return group
 
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False,
                    install_git_hooks=False, user=None, overwrite_git_hooks=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also check for db entries
     that are not in initial_repo_list and removes them.
 
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     :param install_git_hooks: if this is True, also check and install git hook
         for a repo if missing
     :param overwrite_git_hooks: if this is True, overwrite any existing git hooks
         that may be encountered (even if user-deployed)
     """
     from kallithea.model.repo import RepoModel
     from kallithea.model.scm import ScmModel
     sa = meta.Session()
     repo_model = RepoModel()
     if user is None:
         user = User.get_first_admin()
     added = []
 
     ##creation defaults
     defs = Setting.get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
 
     for name, repo in initial_repo_list.items():
         group = map_groups(name)
         unicode_name = safe_unicode(name)
         db_repo = repo_model.get_by_repo_name(unicode_name)
         # found repo that is on filesystem not in Kallithea database
         if not db_repo:
             log.info('repository %s not found, creating now', name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
 
             new_repo = repo_model._create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repo_group=getattr(group, 'group_id', None),
                 owner=user,
                 enable_locking=enable_locking,
                 enable_downloads=enable_downloads,
                 enable_statistics=enable_statistics,
                 private=private,
                 state=Repository.STATE_CREATED
             )
             sa.commit()
             # we added that repo just now, and make sure it has githook
             # installed, and updated server info
             if new_repo.repo_type == 'git':
                 git_repo = new_repo.scm_instance
                 ScmModel().install_git_hooks(git_repo)
                 # update repository server-info
                 log.debug('Running update server info')
                 git_repo._update_server_info()
             new_repo.update_changeset_cache()
         elif install_git_hooks:
             if db_repo.repo_type == 'git':
                 ScmModel().install_git_hooks(db_repo.scm_instance, force_create=overwrite_git_hooks)
 
     removed = []
     # remove from database those repositories that are not in the filesystem
     unicode_initial_repo_list = set(safe_unicode(name) for name in initial_repo_list)
     for repo in sa.query(Repository).all():
         if repo.repo_name not in unicode_initial_repo_list:
             if remove_obsolete:
                 log.debug("Removing non-existing repository found in db `%s`",
                           repo.repo_name)
                 try:
                     RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
                     sa.commit()
                 except Exception:
                     #don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
             removed.append(repo.repo_name)
     return added, removed
 
 
 # set cache regions for beaker so celery can utilise it
 def add_cache(settings):
     cache_settings = {'regions': None}
     for key in settings.keys():
         for prefix in ['beaker.cache.', 'cache.']:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 cache_settings[name] = settings[key].strip()
     if cache_settings['regions']:
         for region in cache_settings['regions'].split(','):
             region = region.strip()
             region_settings = {}
             for key, value in cache_settings.items():
                 if key.startswith(region):
                     region_settings[key.split('.')[1]] = value
             region_settings['expire'] = int(region_settings.get('expire',
                                                                 60))
             region_settings.setdefault('lock_dir',
                                        cache_settings.get('lock_dir'))
             region_settings.setdefault('data_dir',
                                        cache_settings.get('data_dir'))
 
             if 'type' not in region_settings:
                 region_settings['type'] = cache_settings.get('type',
                                                              'memory')
             beaker.cache.cache_regions[region] = region_settings
 
 
 def load_rcextensions(root_path):
     import kallithea
     from kallithea.config import conf
 
     path = os.path.join(root_path, 'rcextensions', '__init__.py')
     if os.path.isfile(path):
         rcext = create_module('rc', path)
         EXT = kallithea.EXTENSIONS = rcext
         log.debug('Found rcextensions now loading %s...', rcext)
 
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
 
         #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)
 
         if getattr(EXT, 'INDEX_EXTENSIONS', []):
             log.debug('settings custom INDEX_EXTENSIONS')
             conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])
 
         #ADDITIONAL MAPPINGS
         log.debug('adding extra into INDEX_EXTENSIONS')
         conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))
 
         # auto check if the module is not missing any data, set to default if is
         # this will help autoupdate new feature of rcext module
         #from kallithea.config import rcextensions
         #for k in dir(rcextensions):
         #    if not k.startswith('_') and not hasattr(EXT, k):
         #        setattr(EXT, k, getattr(rcextensions, k))
 
 
 def get_custom_lexer(extension):
     """
     returns a custom lexer if it's defined in rcextensions module, or None
     if there's no custom lexer defined
     """
     import kallithea
     from pygments import lexers
     #check if we didn't define this extension as other lexer
     if kallithea.EXTENSIONS and extension in kallithea.EXTENSIONS.EXTRA_LEXERS:
         _lexer_name = kallithea.EXTENSIONS.EXTRA_LEXERS[extension]
         return lexers.get_lexer_by_name(_lexer_name)
 
 
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config, full_index):
     """
     Makes default test index
 
     :param config: test config
     :param full_index:
     """
 
     from kallithea.lib.indexers.daemon import WhooshIndexingDaemon
     from kallithea.lib.pidlock import DaemonLock, LockHeld
 
     repo_location = repo_location
 
     index_location = os.path.join(config['app_conf']['index_dir'])
     if not os.path.exists(index_location):
         os.makedirs(index_location)
 
     try:
-        l = DaemonLock(file_=os.path.join(dn(index_location), 'make_index.lock'))
+        l = DaemonLock(file_=os.path.join(dirname(index_location), 'make_index.lock'))
         WhooshIndexingDaemon(index_location=index_location,
                              repo_location=repo_location) \
             .run(full_index=full_index)
         l.release()
     except LockHeld:
         pass
 
 
 def create_test_env(repos_test_path, config):
     """
     Makes a fresh database and
     install test repository into tmp dir
     """
     from kallithea.lib.db_manage import DbManage
     from kallithea.tests import HG_REPO, GIT_REPO, TESTS_TMP_PATH
 
     # PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     log.debug('making test db %s', dbconf)
 
     # create test dir if it doesn't exist
     if not os.path.isdir(repos_test_path):
         log.debug('Creating testdir %s', repos_test_path)
         os.makedirs(repos_test_path)
 
     dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],
                         tests=True)
     dbmanage.create_tables(override=True)
     # for tests dynamically set new root paths based on generated content
     dbmanage.create_settings(dbmanage.config_prompt(repos_test_path))
     dbmanage.create_default_user()
     dbmanage.admin_prompt()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
     # PART TWO make test repo
     log.debug('making test vcs repositories')
 
     idx_path = config['app_conf']['index_dir']
     data_path = config['app_conf']['cache_dir']
 
     #clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s', idx_path)
         shutil.rmtree(idx_path)
 
     if data_path and os.path.exists(data_path):
         log.debug('remove %s', data_path)
         shutil.rmtree(data_path)
 
     #CREATE DEFAULT TEST REPOS
-    cur_dir = dn(dn(abspath(__file__)))
+    cur_dir = dirname(dirname(abspath(__file__)))
     tar = tarfile.open(os.path.join(cur_dir, 'tests', 'fixtures', "vcs_test_hg.tar.gz"))
     tar.extractall(os.path.join(TESTS_TMP_PATH, HG_REPO))
     tar.close()
 
-    cur_dir = dn(dn(abspath(__file__)))
+    cur_dir = dirname(dirname(abspath(__file__)))
     tar = tarfile.open(os.path.join(cur_dir, 'tests', 'fixtures', "vcs_test_git.tar.gz"))
     tar.extractall(os.path.join(TESTS_TMP_PATH, GIT_REPO))
     tar.close()
 
     #LOAD VCS test stuff
     from kallithea.tests.vcs import setup_package
     setup_package()
 
 
 #==============================================================================
 # PASTER COMMANDS
 #==============================================================================
 class BasePasterCommand(Command):
     """
     Abstract Base Class for paster commands.
 
     The celery commands are somewhat aggressive about loading
     celery.conf, and since our module sets the `CELERY_LOADER`
     environment variable to our loader, we have to bootstrap a bit and
     make sure we've had a chance to load the pylons config off of the
     command line, otherwise everything fails.
     """
     min_args = 1
     min_args_error = "Please provide a paster config file as an argument."
     takes_config_file = 1
     requires_config_file = True
 
     def run(self, args):
         """
         Overrides Command.run
 
         Checks for a config file argument and loads it.
         """
         if len(args) < self.min_args:
             raise BadCommand(
                 self.min_args_error % {'min_args': self.min_args,
                                        'actual_args': len(args)})
 
         # Decrement because we're going to lob off the first argument.
         # @@ This is hacky
         self.min_args -= 1
         self.bootstrap_config(args[0])
         self.update_parser()
         return super(BasePasterCommand, self).run(args[1:])
 
     def update_parser(self):
         """
         Abstract method.  Allows for the class's parser to be updated
         before the superclass's `run` method is called.  Necessary to
         allow options/arguments to be passed through to the underlying
         celery command.
         """
         raise NotImplementedError("Abstract Method.")
 
     def bootstrap_config(self, conf):
         """
         Loads the pylons configuration.
         """
         from pylons import config as pylonsconfig
 
         self.path_to_ini_file = os.path.realpath(conf)
         conf = paste.deploy.appconfig('config:' + self.path_to_ini_file)
         pylonsconfig.init_app(conf.global_conf, conf.local_conf)
 
     def _init_session(self):
         """
         Inits SqlAlchemy Session
         """
         logging.config.fileConfig(self.path_to_ini_file)
 
         from pylons import config
         from kallithea.model import init_model
         from kallithea.lib.utils2 import engine_from_config
         add_cache(config)
         engine = engine_from_config(config, 'sqlalchemy.db1.')
         init_model(engine)
 
 
 def check_git_version():
     """
     Checks what version of git is installed in system, and issues a warning
     if it's too old for Kallithea to work properly.
     """
     from kallithea import BACKENDS
     from kallithea.lib.vcs.backends.git.repository import GitRepository
     from kallithea.lib.vcs.conf import settings
     from distutils.version import StrictVersion
 
     if 'git' not in BACKENDS:
         return None
 
     stdout, stderr = GitRepository._run_git_command(['--version'], _bare=True,
                                                     _safe=True)
 
     m = re.search("\d+.\d+.\d+", stdout)
     if m:
         ver = StrictVersion(m.group(0))
     else:
         ver = StrictVersion('0.0.0')
 
     req_ver = StrictVersion('1.7.4')
 
     log.debug('Git executable: "%s" version %s detected: %s',
               settings.GIT_EXECUTABLE_PATH, ver, stdout)
     if stderr:
         log.warning('Error detecting git version: %r', stderr)
     elif ver < req_ver:
         log.warning('Kallithea detected git version %s, which is too old '
                     'for the system to function properly. '
                     'Please upgrade to version %s or later.' % (ver, req_ver))
     return ver
 
 
 @decorator.decorator
 def jsonify(func, *args, **kwargs):
     """Action decorator that formats output for JSON
 
     Given a function that will return content, this decorator will turn
     the result into JSON, with a content-type of 'application/json' and
     output it.
 
     """
     from pylons.decorators.util import get_pylons
     from kallithea.lib.compat import json
     pylons = get_pylons(args)
     pylons.response.headers['Content-Type'] = 'application/json; charset=utf-8'
     data = func(*args, **kwargs)
     if isinstance(data, (list, tuple)):
         msg = "JSON responses with Array envelopes are susceptible to " \
               "cross-site data leak attacks, see " \
               "http://wiki.pylonshq.com/display/pylonsfaq/Warnings"
         warnings.warn(msg, Warning, 2)
         log.warning(msg)
     log.debug("Returning JSON wrapped action output")
     return json.dumps(data, encoding='utf-8')
 
 
 def conditional_cache(region, prefix, condition, func):
     """
 
     Conditional caching function use like::
         def _c(arg):
             #heavy computation function
             return data
 
         # depending from condition the compute is wrapped in cache or not
         compute = conditional_cache('short_term', 'cache_desc', condition=True, func=func)
         return compute(arg)
 
     :param region: name of cache region
     :param prefix: cache region prefix
     :param condition: condition for cache to be triggered, and return data cached
     :param func: wrapped heavy function to compute
 
     """
     wrapped = func
     if condition:
         log.debug('conditional_cache: True, wrapping call of '
                   'func: %s into %s region cache' % (region, func))
         wrapped = _cache_decorate((prefix,), None, None, region)(func)
 
     return wrapped

kallithea/tests/fixture.py

@@ -1,299 +1,299 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
 Helpers for fixture generation
 """
 import os
 import time
 from kallithea.tests import *
 from kallithea.model.db import Repository, User, RepoGroup, UserGroup
 from kallithea.model.meta import Session
 from kallithea.model.repo import RepoModel
 from kallithea.model.user import UserModel
 from kallithea.model.repo_group import RepoGroupModel
 from kallithea.model.user_group import UserGroupModel
 from kallithea.model.gist import GistModel
 from kallithea.model.scm import ScmModel
 from kallithea.lib.vcs.backends.base import EmptyChangeset
+from os.path import dirname
 
-dn = os.path.dirname
-FIXTURES = os.path.join(dn(dn(os.path.abspath(__file__))), 'tests', 'fixtures')
+FIXTURES = os.path.join(dirname(dirname(os.path.abspath(__file__))), 'tests', 'fixtures')
 
 
 def error_function(*args, **kwargs):
     raise Exception('Total Crash !')
 
 
 class Fixture(object):
 
     def __init__(self):
         pass
 
     def anon_access(self, status):
         """
         Context manager for controlling anonymous access.
         Anon access will be set and committed, but restored again when exiting the block.
 
         Usage:
 
         fixture = Fixture()
         with fixture.anon_access(False):
             stuff
         """
 
         class context(object):
             def __enter__(self):
                 anon = User.get_default_user()
                 self._before = anon.active
                 anon.active = status
                 Session().add(anon)
                 Session().commit()
                 time.sleep(1.5)  # hack: wait for beaker sql_cache_short to expire
 
             def __exit__(self, exc_type, exc_val, exc_tb):
                 anon = User.get_default_user()
                 anon.active = self._before
                 Session().add(anon)
                 Session().commit()
 
         return context()
 
     def _get_repo_create_params(self, **custom):
         defs = dict(
             repo_name=None,
             repo_type='hg',
             clone_uri='',
             repo_group=u'-1',
             repo_description=u'DESC',
             repo_private=False,
             repo_landing_rev='rev:tip',
             repo_copy_permissions=False,
             repo_state=Repository.STATE_CREATED,
         )
         defs.update(custom)
         if 'repo_name_full' not in custom:
             defs.update({'repo_name_full': defs['repo_name']})
 
         # fix the repo name if passed as repo_name_full
         if defs['repo_name']:
             defs['repo_name'] = defs['repo_name'].split('/')[-1]
 
         return defs
 
     def _get_group_create_params(self, **custom):
         defs = dict(
             group_name=None,
             group_description=u'DESC',
             group_parent_id=None,
             perms_updates=[],
             perms_new=[],
             enable_locking=False,
             recursive=False
         )
         defs.update(custom)
 
         return defs
 
     def _get_user_create_params(self, name, **custom):
         defs = dict(
             username=name,
             password='qweqwe',
             email='[e-mail]' % name,
             firstname=u'TestUser',
             lastname=u'Test',
             active=True,
             admin=False,
             extern_type='internal',
             extern_name=None
         )
         defs.update(custom)
 
         return defs
 
     def _get_user_group_create_params(self, name, **custom):
         defs = dict(
             users_group_name=name,
             user_group_description=u'DESC',
             users_group_active=True,
             user_group_data={},
         )
         defs.update(custom)
 
         return defs
 
     def create_repo(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             r = Repository.get_by_repo_name(name)
             if r:
                 return r
 
         if isinstance(kwargs.get('repo_group'), RepoGroup):
             kwargs['repo_group'] = kwargs['repo_group'].group_id
 
         form_data = self._get_repo_create_params(repo_name=name, **kwargs)
         cur_user = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         RepoModel().create(form_data, cur_user)
         Session().commit()
         ScmModel().mark_for_invalidation(name)
         return Repository.get_by_repo_name(name)
 
     def create_fork(self, repo_to_fork, fork_name, **kwargs):
         repo_to_fork = Repository.get_by_repo_name(repo_to_fork)
 
         form_data = self._get_repo_create_params(repo_name=fork_name,
                                             fork_parent_id=repo_to_fork,
                                             repo_type=repo_to_fork.repo_type,
                                             **kwargs)
         form_data['update_after_clone'] = False
 
         #TODO: fix it !!
         form_data['description'] = form_data['repo_description']
         form_data['private'] = form_data['repo_private']
         form_data['landing_rev'] = form_data['repo_landing_rev']
 
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         RepoModel().create_fork(form_data, cur_user=owner)
         Session().commit()
         ScmModel().mark_for_invalidation(fork_name)
         r = Repository.get_by_repo_name(fork_name)
         assert r
         return r
 
     def destroy_repo(self, repo_name, **kwargs):
         RepoModel().delete(repo_name, **kwargs)
         Session().commit()
 
     def create_repo_group(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             gr = RepoGroup.get_by_group_name(group_name=name)
             if gr:
                 return gr
         form_data = self._get_group_create_params(group_name=name, **kwargs)
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         gr = RepoGroupModel().create(
             group_name=form_data['group_name'],
             group_description=form_data['group_name'],
             owner=owner, parent=form_data['group_parent_id'])
         Session().commit()
         gr = RepoGroup.get_by_group_name(gr.group_name)
         return gr
 
     def destroy_repo_group(self, repogroupid):
         RepoGroupModel().delete(repogroupid)
         Session().commit()
 
     def create_user(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             user = User.get_by_username(name)
             if user:
                 return user
         form_data = self._get_user_create_params(name, **kwargs)
         user = UserModel().create(form_data)
         Session().commit()
         user = User.get_by_username(user.username)
         return user
 
     def destroy_user(self, userid):
         UserModel().delete(userid)
         Session().commit()
 
     def create_user_group(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             gr = UserGroup.get_by_group_name(group_name=name)
             if gr:
                 return gr
         form_data = self._get_user_group_create_params(name, **kwargs)
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         user_group = UserGroupModel().create(
             name=form_data['users_group_name'],
             description=form_data['user_group_description'],
             owner=owner, active=form_data['users_group_active'],
             group_data=form_data['user_group_data'])
         Session().commit()
         user_group = UserGroup.get_by_group_name(user_group.users_group_name)
         return user_group
 
     def destroy_user_group(self, usergroupid):
         UserGroupModel().delete(user_group=usergroupid, force=True)
         Session().commit()
 
     def create_gist(self, **kwargs):
         form_data = {
             'description': u'new-gist',
             'owner': TEST_USER_ADMIN_LOGIN,
             'gist_type': GistModel.cls.GIST_PUBLIC,
             'lifetime': -1,
             'gist_mapping': {'filename1.txt':{'content':'hello world'},}
         }
         form_data.update(kwargs)
         gist = GistModel().create(
             description=form_data['description'],owner=form_data['owner'],
             gist_mapping=form_data['gist_mapping'], gist_type=form_data['gist_type'],
             lifetime=form_data['lifetime']
         )
         Session().commit()
 
         return gist
 
     def destroy_gists(self, gistid=None):
         for g in GistModel.cls.get_all():
             if gistid:
                 if gistid == g.gist_access_id:
                     GistModel().delete(g)
             else:
                 GistModel().delete(g)
         Session().commit()
 
     def load_resource(self, resource_name, strip=True):
         with open(os.path.join(FIXTURES, resource_name), 'rb') as f:
             source = f.read()
             if strip:
                 source = source.strip()
 
         return source
 
     def commit_change(self, repo, filename, content, message, vcs_type, parent=None, newfile=False):
         repo = Repository.get_by_repo_name(repo)
         _cs = parent
         if not parent:
             _cs = EmptyChangeset(alias=vcs_type)
 
         if newfile:
             nodes = {
                 filename: {
                     'content': content
                 }
             }
             cs = ScmModel().create_nodes(
                 user=TEST_USER_ADMIN_LOGIN, repo=repo,
                 message=message,
                 nodes=nodes,
                 parent_cs=_cs,
                 author=TEST_USER_ADMIN_LOGIN,
             )
         else:
             cs = ScmModel().commit_change(
                 repo=repo.scm_instance, repo_name=repo.repo_name,
                 cs=parent, user=TEST_USER_ADMIN_LOGIN,
                 author=TEST_USER_ADMIN_LOGIN,
                 message=message,
                 content=content,
                 f_path=filename
             )
         return cs

kallithea/tests/functional/test_admin.py

@@ -1,149 +1,149 b''
 import os
 import csv
 import datetime
 from kallithea.tests import *
 from kallithea.model.db import UserLog
 from kallithea.model.meta import Session
 from kallithea.lib.utils2 import safe_unicode
+from os.path import dirname
 
-dn = os.path.dirname
-FIXTURES = os.path.join(dn(dn(os.path.abspath(__file__))), 'fixtures')
+FIXTURES = os.path.join(dirname(dirname(os.path.abspath(__file__))), 'fixtures')
 
 
 class TestAdminController(TestController):
 
     @classmethod
     def setup_class(cls):
         UserLog.query().delete()
         Session().commit()
 
         def strptime(val):
             fmt = '%Y-%m-%d %H:%M:%S'
             if '.' not in val:
                 return datetime.datetime.strptime(val, fmt)
 
             nofrag, frag = val.split(".")
             date = datetime.datetime.strptime(nofrag, fmt)
 
             frag = frag[:6]  # truncate to microseconds
             frag += (6 - len(frag)) * '0'  # add 0s
             return date.replace(microsecond=int(frag))
 
         with open(os.path.join(FIXTURES, 'journal_dump.csv')) as f:
             for row in csv.DictReader(f):
                 ul = UserLog()
                 for k, v in row.iteritems():
                     v = safe_unicode(v)
                     if k == 'action_date':
                         v = strptime(v)
                     if k in ['user_id', 'repository_id']:
                         # nullable due to FK problems
                         v = None
                     setattr(ul, k, v)
                 Session().add(ul)
             Session().commit()
 
     @classmethod
     def teardown_class(cls):
         UserLog.query().delete()
         Session().commit()
 
     def test_index(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index'))
         response.mustcontain('Admin Journal')
 
     def test_filter_all_entries(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',))
         response.mustcontain('2034 Entries')
 
     def test_filter_journal_filter_exact_match_on_repository(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='repository:xxx'))
         response.mustcontain('3 Entries')
 
     def test_filter_journal_filter_exact_match_on_repository_CamelCase(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='repository:XxX'))
         response.mustcontain('3 Entries')
 
     def test_filter_journal_filter_wildcard_on_repository(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='repository:*test*'))
         response.mustcontain('862 Entries')
 
     def test_filter_journal_filter_prefix_on_repository(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='repository:test*'))
         response.mustcontain('257 Entries')
 
     def test_filter_journal_filter_prefix_on_repository_CamelCase(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='repository:Test*'))
         response.mustcontain('257 Entries')
 
     def test_filter_journal_filter_prefix_on_repository_and_user(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='repository:test* AND username:demo'))
         response.mustcontain('130 Entries')
 
     def test_filter_journal_filter_prefix_on_repository_or_other_repo(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='repository:test* OR repository:xxx'))
         response.mustcontain('260 Entries')  # 257 + 3
 
     def test_filter_journal_filter_exact_match_on_username(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='username:demo'))
         response.mustcontain('1087 Entries')
 
     def test_filter_journal_filter_exact_match_on_username_camelCase(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='username:DemO'))
         response.mustcontain('1087 Entries')
 
     def test_filter_journal_filter_wildcard_on_username(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='username:*test*'))
         response.mustcontain('100 Entries')
 
     def test_filter_journal_filter_prefix_on_username(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='username:demo*'))
         response.mustcontain('1101 Entries')
 
     def test_filter_journal_filter_prefix_on_user_or_other_user(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='username:demo OR username:volcan'))
         response.mustcontain('1095 Entries')  # 1087 + 8
 
     def test_filter_journal_filter_wildcard_on_action(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='action:*pull_request*'))
         response.mustcontain('187 Entries')
 
     def test_filter_journal_filter_on_date(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='date:20121010'))
         response.mustcontain('47 Entries')
 
     def test_filter_journal_filter_on_date_2(self):
         self.log_user()
         response = self.app.get(url(controller='admin/admin', action='index',
                                     filter='date:20121020'))
         response.mustcontain('17 Entries')

kallithea/tests/scripts/manual_test_concurrency.py

@@ -1,221 +1,221 b''
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.tests.scripts.manual_test_concurrency
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Test suite for making push/pull operations
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Dec 30, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 
 """
 
 import os
 import sys
 import shutil
 import logging
-from os.path import dirname as dn
+from os.path import dirname
 
 from tempfile import _RandomNameSequence
 from subprocess import Popen, PIPE
 
 from paste.deploy import appconfig
 from sqlalchemy import engine_from_config
 
 from kallithea.lib.utils import add_cache
 from kallithea.model import init_model
 from kallithea.model import meta
 from kallithea.model.db import User, Repository, Ui
 from kallithea.lib.auth import get_crypt_password
 
 from kallithea.tests import HG_REPO
 from kallithea.config.environment import load_environment
 
-rel_path = dn(dn(dn(dn(os.path.abspath(__file__)))))
+rel_path = dirname(dirname(dirname(dirname(os.path.abspath(__file__)))))
 conf = appconfig('config:development.ini', relative_to=rel_path)
 load_environment(conf.global_conf, conf.local_conf)
 
 add_cache(conf)
 
 USER = TEST_USER_ADMIN_LOGIN
 PASS = TEST_USER_ADMIN_PASS
 HOST = 'server.local'
 METHOD = 'pull'
 DEBUG = True
 log = logging.getLogger(__name__)
 
 
 class Command(object):
 
     def __init__(self, cwd):
         self.cwd = cwd
 
     def execute(self, cmd, *args):
         """Runs command on the system with given ``args``.
         """
 
         command = cmd + ' ' + ' '.join(args)
         log.debug('Executing %s', command)
         if DEBUG:
             print command
         p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, cwd=self.cwd)
         stdout, stderr = p.communicate()
         if DEBUG:
             print stdout, stderr
         return stdout, stderr
 
 
 def get_session():
     engine = engine_from_config(conf, 'sqlalchemy.db1.')
     init_model(engine)
     sa = meta.Session
     return sa
 
 
 def create_test_user(force=True):
     print 'creating test user'
     sa = get_session()
 
     user = sa.query(User).filter(User.username == USER).scalar()
 
     if force and user is not None:
         print 'removing current user'
         for repo in sa.query(Repository).filter(Repository.user == user).all():
             sa.delete(repo)
         sa.delete(user)
         sa.commit()
 
     if user is None or force:
         print 'creating new one'
         new_usr = User()
         new_usr.username = USER
         new_usr.password = get_crypt_password(PASS)
         new_usr.email = '[e-mail]'
         new_usr.name = 'test'
         new_usr.lastname = 'lasttestname'
         new_usr.active = True
         new_usr.admin = True
         sa.add(new_usr)
         sa.commit()
 
     print 'done'
 
 
 def create_test_repo(force=True):
     print 'creating test repo'
     from kallithea.model.repo import RepoModel
     sa = get_session()
 
     user = sa.query(User).filter(User.username == USER).scalar()
     if user is None:
         raise Exception('user not found')
 
     repo = sa.query(Repository).filter(Repository.repo_name == HG_REPO).scalar()
 
     if repo is None:
         print 'repo not found creating'
 
         form_data = {'repo_name': HG_REPO,
                      'repo_type': 'hg',
                      'private':False,
                      'clone_uri': '' }
         rm = RepoModel(sa)
         rm.base_path = '/home/hg'
         rm.create(form_data, user)
 
     print 'done'
 
 
 def set_anonymous_access(enable=True):
     sa = get_session()
     user = sa.query(User).filter(User.username == 'default').one()
     user.active = enable
     sa.add(user)
     sa.commit()
 
 
 def get_anonymous_access():
     sa = get_session()
     return sa.query(User).filter(User.username == 'default').one().active
 
 
 #==============================================================================
 # TESTS
 #==============================================================================
 def test_clone_with_credentials(no_errors=False, repo=HG_REPO, method=METHOD,
                                 seq=None, backend='hg'):
     cwd = path = os.path.join(Ui.get_by_key('paths', '/').ui_value, repo)
 
     if seq is None:
         seq = _RandomNameSequence().next()
 
     try:
         shutil.rmtree(path, ignore_errors=True)
         os.makedirs(path)
         #print 'made dirs %s' % os.path.join(path)
     except OSError:
         raise
 
     clone_url = 'http://%(user)s:%(pass)s@%(host)s/%(cloned_repo)s' % \
                   {'user': USER,
                    'pass': PASS,
                    'host': HOST,
                    'cloned_repo': repo, }
 
     dest = path + seq
     if method == 'pull':
         stdout, stderr = Command(cwd).execute(backend, method, '--cwd', dest, clone_url)
     else:
         stdout, stderr = Command(cwd).execute(backend, method, clone_url, dest)
         print stdout,'sdasdsadsa'
         if not no_errors:
             if backend == 'hg':
                 assert """adding file changes""" in stdout, 'no messages about cloning'
                 assert """abort""" not in stderr , 'got error from clone'
             elif backend == 'git':
                 assert """Cloning into""" in stdout, 'no messages about cloning'
 
 if __name__ == '__main__':
     try:
         create_test_user(force=False)
         seq = None
         import time
 
         try:
             METHOD = sys.argv[3]
         except IndexError:
             pass
 
         try:
             backend = sys.argv[4]
         except IndexError:
             backend = 'hg'
 
         if METHOD == 'pull':
             seq = _RandomNameSequence().next()
             test_clone_with_credentials(repo=sys.argv[1], method='clone',
                                         seq=seq, backend=backend)
         s = time.time()
         for i in range(1, int(sys.argv[2]) + 1):
             print 'take', i
             test_clone_with_credentials(repo=sys.argv[1], method=METHOD,
                                         seq=seq, backend=backend)
         print 'time taken %.3f' % (time.time() - s)
     except Exception as e:
         sys.exit('stop on %s' % e)

kallithea/tests/scripts/manual_test_crawler.py

@@ -1,190 +1,190 b''
 #!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.tests.scripts.manual_test_crawler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Test for crawling a project for memory usage
 This should be runned just as regular script together
 with a watch script that will show memory usage.
 
 watch -n1 ./kallithea/tests/mem_watch
 
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 21, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 
 
 import cookielib
 import urllib
 import urllib2
 import time
 import os
 import sys
 import tempfile
-from os.path import dirname as dn
+from os.path import dirname
 
 __here__ = os.path.abspath(__file__)
-__root__ = dn(dn(dn(__here__)))
+__root__ = dirname(dirname(dirname(__here__)))
 sys.path.append(__root__)
 
 from kallithea.lib import vcs
 from kallithea.lib.compat import OrderedSet
 from kallithea.lib.vcs.exceptions import RepositoryError
 
 PASES = 3
 HOST = 'http://127.0.0.1'
 PORT = 5000
 BASE_URI = '%s:%s/' % (HOST, PORT)
 
 if len(sys.argv) == 2:
     BASE_URI = sys.argv[1]
 
 if not BASE_URI.endswith('/'):
     BASE_URI += '/'
 
 print 'Crawling @ %s' % BASE_URI
 BASE_URI += '%s'
 PROJECT_PATH = os.path.join('/', 'home', 'username', 'repos')
 PROJECTS = [
     #'linux-magx-pbranch',
     'CPython',
     'kallithea',
 ]
 
 
 cj = cookielib.FileCookieJar(os.path.join(tempfile.gettempdir(), 'rc_test_cookie.txt'))
 o = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
 o.addheaders = [
     ('User-agent', 'kallithea-crawler'),
     ('Accept-Language', 'en - us, en;q = 0.5')
 ]
 
 urllib2.install_opener(o)
 
 
 def _get_repo(proj):
     if isinstance(proj, basestring):
         repo = vcs.get_repo(os.path.join(PROJECT_PATH, proj))
         proj = proj
     else:
         repo = proj
         proj = repo.name
 
     return repo, proj
 
 
 def test_changelog_walk(proj, pages=100):
     repo, proj = _get_repo(proj)
 
     total_time = 0
     for i in range(1, pages):
 
         page = '/'.join((proj, 'changelog',))
 
         full_uri = (BASE_URI % page) + '?' + urllib.urlencode({'page': i})
         s = time.time()
         f = o.open(full_uri)
 
         assert f.url == full_uri, 'URL:%s does not match %s' % (f.url, full_uri)
 
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print 'visited %s size:%s req:%s ms' % (full_uri, size, e)
 
     print 'total_time', total_time
     print 'average on req', total_time / float(pages)
 
 
 def test_changeset_walk(proj, limit=None):
     repo, proj = _get_repo(proj)
 
     print 'processing', os.path.join(PROJECT_PATH, proj)
     total_time = 0
 
     cnt = 0
     for i in repo:
         cnt += 1
         raw_cs = '/'.join((proj, 'changeset', i.raw_id))
         if limit and limit == cnt:
             break
 
         full_uri = (BASE_URI % raw_cs)
         print '%s visiting %s\%s' % (cnt, full_uri, i)
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print '%s visited %s\%s size:%s req:%s ms' % (cnt, full_uri, i, size, e)
 
     print 'total_time', total_time
     print 'average on req', total_time / float(cnt)
 
 
 def test_files_walk(proj, limit=100):
     repo, proj = _get_repo(proj)
 
     print 'processing', os.path.join(PROJECT_PATH, proj)
     total_time = 0
 
     paths_ = OrderedSet([''])
     try:
         tip = repo.get_changeset('tip')
         for topnode, dirs, files in tip.walk('/'):
 
             for dir in dirs:
                 paths_.add(dir.path)
                 for f in dir:
                     paths_.add(f.path)
 
             for f in files:
                 paths_.add(f.path)
 
     except RepositoryError as e:
         pass
 
     cnt = 0
     for f in paths_:
         cnt += 1
         if limit and limit == cnt:
             break
 
         file_path = '/'.join((proj, 'files', 'tip', f))
         full_uri = (BASE_URI % file_path)
         print '%s visiting %s' % (cnt, full_uri)
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print '%s visited OK size:%s req:%s ms' % (cnt, size, e)
 
     print 'total_time', total_time
     print 'average on req', total_time / float(cnt)
 
 if __name__ == '__main__':
     for path in PROJECTS:
         repo = vcs.get_repo(os.path.join(PROJECT_PATH, path))
         for i in range(PASES):
             print 'PASS %s/%s' % (i, PASES)
             test_changelog_walk(repo, pages=80)
             test_changeset_walk(repo, limit=100)
             test_files_walk(repo, limit=100)