Hacking for git support,and new faster repo scan

docs/changelog.rst

@@ -1,36 +1,42 b''
 .. _changelog:
 
 Changelog
 =========
 
+1.1.0 (**XXXX-XX-XX**)
+----------------------
+- git support
+- performance upgrade for cached repos list
+
+
 1.0.0 (**2010-10-xx**)
 ----------------------
 
 - security bugfix simplehg wasn't checking for permissions on commands
   other than pull or push.
 - fixed doubled messages after push or pull in admin journal
 - templating and css corrections, fixed repo switcher on chrome,updated titles
 - admin menu accessible from options menu on repository view
 - permissions cached queries
 
 1.0.0rc4  (**2010-10-12**)
 --------------------------
 
 - fixed python2.5 missing simplejson imports (thanks to Jens Bäckman)
 - removed cache_manager settings from sqlalchemy meta
 - added sqlalchemy cache settings to ini files
 - validated password length and added second try of failure on paster setup-app
 - fixed setup database destroy prompt even when there was no db
 
 
 1.0.0rc3 (**2010-10-11**)
 -------------------------
 
 - fixed i18n during installation.
 
 1.0.0rc2 (**2010-10-11**)
 -------------------------
 
 - Disabled dirsize in file browser, it's causing nasty bug when dir renames 
   occure. After vcs is fixed it'll be put back again.
 - templating/css rewrites, optimized css.

rhodecode/__init__.py

@@ -1,35 +1,35 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # RhodeCode, a web based repository management based on pylons
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 # 
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on April 9, 2010
 RhodeCode, a web based repository management based on pylons
 versioning implementation: http://semver.org/
 @author: marcink
 """
 
-VERSION = (1, 0, 0, 'rc4')
+VERSION = (1, 1, 0, 'beta')
 
 __version__ = '.'.join((str(each) for each in VERSION[:4]))
 
 def get_version():
     """
     Returns shorter version (digit parts only) as string.
     """
     return '.'.join((str(each) for each in VERSION[:3]))

rhodecode/config/environment.py

@@ -1,79 +1,79 b''
 """Pylons environment configuration"""
 from mako.lookup import TemplateLookup
 from pylons.configuration import PylonsConfig
 from pylons.error import handle_mako_error
 from rhodecode.config.routing import make_map
 from rhodecode.lib.auth import set_available_permissions, set_base_path
 from rhodecode.lib.utils import repo2db_mapper, make_ui, set_rhodecode_config
 from rhodecode.model import init_model
 from rhodecode.model.hg import _get_repos_cached_initial
 from sqlalchemy import engine_from_config
 import logging
 import os
 import rhodecode.lib.app_globals as app_globals
 import rhodecode.lib.helpers
 
 log = logging.getLogger(__name__)
 
 def load_environment(global_conf, app_conf, initial=False):
     """Configure the Pylons environment via the ``pylons.config``
     object
     """
     config = PylonsConfig()
     
     # Pylons paths
     root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
     paths = dict(root=root,
                  controllers=os.path.join(root, 'controllers'),
                  static_files=os.path.join(root, 'public'),
                  templates=[os.path.join(root, 'templates')])
 
     # Initialize config with the basic options
     config.init_app(global_conf, app_conf, package='rhodecode', paths=paths)
 
     config['routes.map'] = make_map(config)
     config['pylons.app_globals'] = app_globals.Globals(config)
     config['pylons.h'] = rhodecode.lib.helpers
     
     # Setup cache object as early as possible
     import pylons
     pylons.cache._push_object(config['pylons.app_globals'].cache)
     
     # Create the Mako TemplateLookup, with the default auto-escaping
     config['pylons.app_globals'].mako_lookup = TemplateLookup(
         directories=paths['templates'],
         error_handler=handle_mako_error,
         module_directory=os.path.join(app_conf['cache_dir'], 'templates'),
         input_encoding='utf-8', default_filters=['escape'],
         imports=['from webhelpers.html import escape'])
 
     #sets the c attribute access when don't existing attribute are accessed
     config['pylons.strict_tmpl_context'] = True
     test = os.path.split(config['__file__'])[-1] == 'test.ini'
     if test:
         from rhodecode.lib.utils import create_test_env, create_test_index
         create_test_env('/tmp', config)
-        create_test_index('/tmp/*', True)
+        create_test_index('/tmp', True)
         
     #MULTIPLE DB configs
     # Setup the SQLAlchemy database engine
     if config['debug'] and not test:
         #use query time debugging.
         from rhodecode.lib.timerproxy import TimerProxy
         sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.',
                                                             proxy=TimerProxy())
     else:
         sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.')
 
     init_model(sa_engine_db1)
     #init baseui
     config['pylons.app_globals'].baseui = make_ui('db')
     
     repo2db_mapper(_get_repos_cached_initial(config['pylons.app_globals'], initial))
     set_available_permissions(config)
     set_base_path(config)
     set_rhodecode_config(config)
     # CONFIGURATION OPTIONS HERE (note: all config options will override
     # any Pylons config options)
     
     return config

rhodecode/lib/app_globals.py

@@ -1,31 +1,31 b''
 """The application's Globals object"""
 
 from beaker.cache import CacheManager
 from beaker.util import parse_cache_config_options
 from vcs.utils.lazy import LazyProperty
 
 class Globals(object):
     """Globals acts as a container for objects available throughout the
     life of the application
 
     """
 
     def __init__(self, config):
         """One instance of Globals is created during application
         initialization and is available during requests via the
         'app_globals' variable
 
         """
         self.cache = CacheManager(**parse_cache_config_options(config))
         self.available_permissions = None   # propagated after init_model
         self.baseui = None                  # propagated after init_model        
         
     @LazyProperty
     def paths(self):
         if self.baseui:
             return self.baseui.configitems('paths')
     
     @LazyProperty
     def base_path(self):
         if self.baseui:
-            return self.paths[0][1].replace('*', '')            
+            return self.paths[0][1]

rhodecode/lib/celerylib/tasks.py

@@ -1,329 +1,336 b''
 from celery.decorators import task
 
 from operator import itemgetter
 from pylons.i18n.translation import _
 from rhodecode.lib.celerylib import run_task, locked_task
 from rhodecode.lib.helpers import person
 from rhodecode.lib.smtp_mailer import SmtpMailer
 from rhodecode.lib.utils import OrderedDict
 from time import mktime
 from vcs.backends.hg import MercurialRepository
+from vcs.backends.git import GitRepository
+import os
 import traceback
+from vcs.backends import get_repo
+from vcs.utils.helpers import get_scm
 
 try:
     import json
 except ImportError:
     #python 2.5 compatibility
     import simplejson as json
 
 try:
     from celeryconfig import PYLONS_CONFIG as config
     celery_on = True
 except ImportError:
     #if celeryconfig is not present let's just load our pylons
     #config instead
     from pylons import config
     celery_on = False
 
 
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
 
 def get_session():
     if celery_on:
         from sqlalchemy import engine_from_config
         from sqlalchemy.orm import sessionmaker, scoped_session
         engine = engine_from_config(dict(config.items('app:main')), 'sqlalchemy.db1.')
         sa = scoped_session(sessionmaker(bind=engine))
     else:
         #If we don't use celery reuse our current application Session
         from rhodecode.model.meta import Session
         sa = Session()
 
     return sa
 
 def get_hg_settings():
     from rhodecode.model.db import RhodeCodeSettings
     sa = get_session()
     ret = sa.query(RhodeCodeSettings).all()
 
     if not ret:
         raise Exception('Could not get application settings !')
     settings = {}
     for each in ret:
         settings['rhodecode_' + each.app_settings_name] = each.app_settings_value
 
     return settings
 
 def get_hg_ui_settings():
     from rhodecode.model.db import RhodeCodeUi
     sa = get_session()
     ret = sa.query(RhodeCodeUi).all()
 
     if not ret:
         raise Exception('Could not get application ui settings !')
     settings = {}
     for each in ret:
         k = each.ui_key
         v = each.ui_value
         if k == '/':
             k = 'root_path'
 
         if k.find('.') != -1:
             k = k.replace('.', '_')
 
         if each.ui_section == 'hooks':
             v = each.ui_active
 
         settings[each.ui_section + '_' + k] = v
 
     return settings
 
 @task
 @locked_task
 def whoosh_index(repo_location, full_index):
     log = whoosh_index.get_logger()
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     WhooshIndexingDaemon(repo_location=repo_location).run(full_index=full_index)
 
 @task
 @locked_task
 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
     from rhodecode.model.db import Statistics, Repository
     log = get_commits_stats.get_logger()
     author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
 
     commits_by_day_author_aggregate = {}
     commits_by_day_aggregate = {}
-    repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
-    repo = MercurialRepository(repos_path + repo_name)
+    repos_path = get_hg_ui_settings()['paths_root_path']
+    p = os.path.join(repos_path, repo_name)
+    repo = get_repo(get_scm(p)[0], p)
 
     skip_date_limit = True
     parse_limit = 250 #limit for single task changeset parsing optimal for
     last_rev = 0
     last_cs = None
     timegetter = itemgetter('time')
 
     sa = get_session()
 
     dbrepo = sa.query(Repository)\
         .filter(Repository.repo_name == repo_name).scalar()
     cur_stats = sa.query(Statistics)\
         .filter(Statistics.repository == dbrepo).scalar()
     if cur_stats:
         last_rev = cur_stats.stat_on_revision
     if not repo.revisions:
         return True
 
     if last_rev == repo.revisions[-1] and len(repo.revisions) > 1:
         #pass silently without any work if we're not on first revision or current
         #state of parsing revision(from db marker) is the last revision
         return True
 
     if cur_stats:
         commits_by_day_aggregate = OrderedDict(
                                        json.loads(
                                         cur_stats.commit_activity_combined))
         commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity)
 
     log.debug('starting parsing %s', parse_limit)
     lmktime = mktime
 
     for cnt, rev in enumerate(repo.revisions[last_rev:]):
         last_cs = cs = repo.get_changeset(rev)
         k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1],
                           cs.date.timetuple()[2])
         timetupple = [int(x) for x in k.split('-')]
         timetupple.extend([0 for _ in xrange(6)])
         k = lmktime(timetupple)
         if commits_by_day_author_aggregate.has_key(author_key_cleaner(cs.author)):
             try:
                 l = [timegetter(x) for x in commits_by_day_author_aggregate\
                         [author_key_cleaner(cs.author)]['data']]
                 time_pos = l.index(k)
             except ValueError:
                 time_pos = False
 
             if time_pos >= 0 and time_pos is not False:
 
                 datadict = commits_by_day_author_aggregate\
                     [author_key_cleaner(cs.author)]['data'][time_pos]
 
                 datadict["commits"] += 1
                 datadict["added"] += len(cs.added)
                 datadict["changed"] += len(cs.changed)
                 datadict["removed"] += len(cs.removed)
 
             else:
                 if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
 
                     datadict = {"time":k,
                                 "commits":1,
                                 "added":len(cs.added),
                                 "changed":len(cs.changed),
                                 "removed":len(cs.removed),
                                }
                     commits_by_day_author_aggregate\
                         [author_key_cleaner(cs.author)]['data'].append(datadict)
 
         else:
             if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
                 commits_by_day_author_aggregate[author_key_cleaner(cs.author)] = {
                                     "label":author_key_cleaner(cs.author),
                                     "data":[{"time":k,
                                              "commits":1,
                                              "added":len(cs.added),
                                              "changed":len(cs.changed),
                                              "removed":len(cs.removed),
                                              }],
                                     "schema":["commits"],
                                     }
 
         #gather all data by day
         if commits_by_day_aggregate.has_key(k):
             commits_by_day_aggregate[k] += 1
         else:
             commits_by_day_aggregate[k] = 1
 
         if cnt >= parse_limit:
             #don't fetch to much data since we can freeze application
             break
     overview_data = []
     for k, v in commits_by_day_aggregate.items():
         overview_data.append([k, v])
     overview_data = sorted(overview_data, key=itemgetter(0))
     if not commits_by_day_author_aggregate:
         commits_by_day_author_aggregate[author_key_cleaner(repo.contact)] = {
             "label":author_key_cleaner(repo.contact),
             "data":[0, 1],
             "schema":["commits"],
         }
 
     stats = cur_stats if cur_stats else Statistics()
     stats.commit_activity = json.dumps(commits_by_day_author_aggregate)
     stats.commit_activity_combined = json.dumps(overview_data)
 
     log.debug('last revison %s', last_rev)
     leftovers = len(repo.revisions[last_rev:])
     log.debug('revisions to parse %s', leftovers)
 
     if last_rev == 0 or leftovers < parse_limit:
         stats.languages = json.dumps(__get_codes_stats(repo_name))
 
     stats.repository = dbrepo
     stats.stat_on_revision = last_cs.revision
 
     try:
         sa.add(stats)
         sa.commit()
     except:
         log.error(traceback.format_exc())
         sa.rollback()
         return False
     if len(repo.revisions) > 1:
         run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
 
     return True
 
 @task
 def reset_user_password(user_email):
     log = reset_user_password.get_logger()
     from rhodecode.lib import auth
     from rhodecode.model.db import User
 
     try:
         try:
             sa = get_session()
             user = sa.query(User).filter(User.email == user_email).scalar()
             new_passwd = auth.PasswordGenerator().gen_password(8,
                              auth.PasswordGenerator.ALPHABETS_BIG_SMALL)
             if user:
                 user.password = auth.get_crypt_password(new_passwd)
                 sa.add(user)
                 sa.commit()
                 log.info('change password for %s', user_email)
             if new_passwd is None:
                 raise Exception('unable to generate new password')
 
         except:
             log.error(traceback.format_exc())
             sa.rollback()
 
         run_task(send_email, user_email,
                  "Your new rhodecode password",
                  'Your new rhodecode password:%s' % (new_passwd))
         log.info('send new password mail to %s', user_email)
 
 
     except:
         log.error('Failed to update user password')
         log.error(traceback.format_exc())
     return True
 
 @task
 def send_email(recipients, subject, body):
     log = send_email.get_logger()
     email_config = dict(config.items('DEFAULT'))
     mail_from = email_config.get('app_email_from')
     user = email_config.get('smtp_username')
     passwd = email_config.get('smtp_password')
     mail_server = email_config.get('smtp_server')
     mail_port = email_config.get('smtp_port')
     tls = email_config.get('smtp_use_tls')
     ssl = False
 
     try:
         m = SmtpMailer(mail_from, user, passwd, mail_server,
                        mail_port, ssl, tls)
         m.send(recipients, subject, body)
     except:
         log.error('Mail sending failed')
         log.error(traceback.format_exc())
         return False
     return True
 
 @task
 def create_repo_fork(form_data, cur_user):
     import os
     from rhodecode.model.repo import RepoModel
 
     repo_model = RepoModel(get_session())
     repo_model.create(form_data, cur_user, just_db=True, fork=True)
 
     repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
     repo_path = os.path.join(repos_path, form_data['repo_name'])
     repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
 
     MercurialRepository(str(repo_fork_path), True, clone_url=str(repo_path))
 
 
 def __get_codes_stats(repo_name):
     LANGUAGES_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx',
     'aspx', 'asx', 'axd', 'c', 'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el',
     'erl', 'h', 'java', 'js', 'jsp', 'jspx', 'lisp', 'lua', 'm', 'mako', 'ml',
     'pas', 'patch', 'php', 'php3', 'php4', 'phtml', 'pm', 'py', 'rb', 'rst',
     's', 'sh', 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', 'yaws']
 
 
-    repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
-    repo = MercurialRepository(repos_path + repo_name)
+    repos_path = get_hg_ui_settings()['paths_root_path']
+    p = os.path.join(repos_path, repo_name)
+    repo = get_repo(get_scm(p)[0], p)
+
     tip = repo.get_changeset()
 
     code_stats = {}
 
     def aggregate(cs):
         for f in cs[2]:
             k = f.mimetype
             if f.extension in LANGUAGES_EXTENSIONS:
                 if code_stats.has_key(k):
                     code_stats[k] += 1
                 else:
                     code_stats[k] = 1
 
     map(aggregate, tip.walk('/'))
 
     return code_stats or {}
 
 
 
 

rhodecode/lib/db_manage.py

@@ -1,287 +1,287 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # database management for RhodeCode
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
 """
 Created on April 10, 2010
 database management and creation for RhodeCode
 @author: marcink
 """
 
 from os.path import dirname as dn, join as jn
 import os
 import sys
 import uuid
 
 from rhodecode.lib.auth import get_crypt_password
 from rhodecode.lib.utils import ask_ok
 from rhodecode.model import init_model
 from rhodecode.model.db import User, Permission, RhodeCodeUi, RhodeCodeSettings, \
     UserToPerm
 from rhodecode.model import meta
 from sqlalchemy.engine import create_engine
 import logging
 
 log = logging.getLogger(__name__)
 
 class DbManage(object):
     def __init__(self, log_sql, dbname, root, tests=False):
         self.dbname = dbname
         self.tests = tests
         self.root = root
         dburi = 'sqlite:////%s' % jn(self.root, self.dbname)
         engine = create_engine(dburi, echo=log_sql)
         init_model(engine)
         self.sa = meta.Session()
         self.db_exists = False
 
     def check_for_db(self, override):
         db_path = jn(self.root, self.dbname)
         log.info('checking for existing db in %s', db_path)
         if os.path.isfile(db_path):
             self.db_exists = True
             if not override:
                 raise Exception('database already exists')
 
     def create_tables(self, override=False):
         """
         Create a auth database
         """
         self.check_for_db(override)
         if self.db_exists:
             log.info("database exist and it's going to be destroyed")
             if self.tests:
                 destroy = True
             else:
                 destroy = ask_ok('Are you sure to destroy old database ? [y/n]')
             if not destroy:
                 sys.exit()
             if self.db_exists and destroy:
                 os.remove(jn(self.root, self.dbname))
         checkfirst = not override
         meta.Base.metadata.create_all(checkfirst=checkfirst)
         log.info('Created tables for %s', self.dbname)
 
     def admin_prompt(self, second=False):
         if not self.tests:
             import getpass
 
 
             def get_password():
                 password = getpass.getpass('Specify admin password (min 6 chars):')
                 confirm = getpass.getpass('Confirm password:')
 
                 if password != confirm:
                     log.error('passwords mismatch')
                     return False
                 if len(password) < 6:
                     log.error('password is to short use at least 6 characters')
                     return False
 
                 return password
 
             username = raw_input('Specify admin username:')
 
             password = get_password()
             if not password:
                 #second try
                 password = get_password()
                 if not password:
                     sys.exit()
 
             email = raw_input('Specify admin email:')
             self.create_user(username, password, email, True)
         else:
             log.info('creating admin and regular test users')
             self.create_user('test_admin', 'test12', '[e-mail]', True)
             self.create_user('test_regular', 'test12', '[e-mail]', False)
             self.create_user('test_regular2', 'test12', '[e-mail]', False)
 
 
 
     def config_prompt(self, test_repo_path=''):
         log.info('Setting up repositories config')
 
         if not self.tests and not test_repo_path:
             path = raw_input('Specify valid full path to your repositories'
                         ' you can change this later in application settings:')
         else:
             path = test_repo_path
 
         if not os.path.isdir(path):
             log.error('You entered wrong path: %s', path)
             sys.exit()
 
         hooks1 = RhodeCodeUi()
         hooks1.ui_section = 'hooks'
         hooks1.ui_key = 'changegroup.update'
         hooks1.ui_value = 'hg update >&2'
         hooks1.ui_active = False
 
         hooks2 = RhodeCodeUi()
         hooks2.ui_section = 'hooks'
         hooks2.ui_key = 'changegroup.repo_size'
         hooks2.ui_value = 'python:rhodecode.lib.hooks.repo_size'
 
         web1 = RhodeCodeUi()
         web1.ui_section = 'web'
         web1.ui_key = 'push_ssl'
         web1.ui_value = 'false'
 
         web2 = RhodeCodeUi()
         web2.ui_section = 'web'
         web2.ui_key = 'allow_archive'
         web2.ui_value = 'gz zip bz2'
 
         web3 = RhodeCodeUi()
         web3.ui_section = 'web'
         web3.ui_key = 'allow_push'
         web3.ui_value = '*'
 
         web4 = RhodeCodeUi()
         web4.ui_section = 'web'
         web4.ui_key = 'baseurl'
         web4.ui_value = '/'
 
         paths = RhodeCodeUi()
         paths.ui_section = 'paths'
         paths.ui_key = '/'
-        paths.ui_value = os.path.join(path, '*')
+        paths.ui_value = path
 
 
         hgsettings1 = RhodeCodeSettings()
 
         hgsettings1.app_settings_name = 'realm'
         hgsettings1.app_settings_value = 'RhodeCode authentication'
 
         hgsettings2 = RhodeCodeSettings()
         hgsettings2.app_settings_name = 'title'
         hgsettings2.app_settings_value = 'RhodeCode'
 
         try:
             self.sa.add(hooks1)
             self.sa.add(hooks2)
             self.sa.add(web1)
             self.sa.add(web2)
             self.sa.add(web3)
             self.sa.add(web4)
             self.sa.add(paths)
             self.sa.add(hgsettings1)
             self.sa.add(hgsettings2)
             self.sa.commit()
         except:
             self.sa.rollback()
             raise
         log.info('created ui config')
 
     def create_user(self, username, password, email='', admin=False):
         log.info('creating administrator user %s', username)
         new_user = User()
         new_user.username = username
         new_user.password = get_crypt_password(password)
         new_user.name = 'RhodeCode'
         new_user.lastname = 'Admin'
         new_user.email = email
         new_user.admin = admin
         new_user.active = True
 
         try:
             self.sa.add(new_user)
             self.sa.commit()
         except:
             self.sa.rollback()
             raise
 
     def create_default_user(self):
         log.info('creating default user')
         #create default user for handling default permissions.
         def_user = User()
         def_user.username = 'default'
         def_user.password = get_crypt_password(str(uuid.uuid1())[:8])
         def_user.name = 'default'
         def_user.lastname = 'default'
         def_user.email = '[e-mail]'
         def_user.admin = False
         def_user.active = False
         try:
             self.sa.add(def_user)
             self.sa.commit()
         except:
             self.sa.rollback()
             raise
 
     def create_permissions(self):
         #module.(access|create|change|delete)_[name]
         #module.(read|write|owner)
         perms = [('repository.none', 'Repository no access'),
                  ('repository.read', 'Repository read access'),
                  ('repository.write', 'Repository write access'),
                  ('repository.admin', 'Repository admin access'),
                  ('hg.admin', 'Hg Administrator'),
                  ('hg.create.repository', 'Repository create'),
                  ('hg.create.none', 'Repository creation disabled'),
                  ('hg.register.none', 'Register disabled'),
                  ('hg.register.manual_activate', 'Register new user with rhodecode without manual activation'),
                  ('hg.register.auto_activate', 'Register new user with rhodecode without auto activation'),
                 ]
 
         for p in perms:
             new_perm = Permission()
             new_perm.permission_name = p[0]
             new_perm.permission_longname = p[1]
             try:
                 self.sa.add(new_perm)
                 self.sa.commit()
             except:
                 self.sa.rollback()
                 raise
 
     def populate_default_permissions(self):
         log.info('creating default user permissions')
 
         default_user = self.sa.query(User)\
         .filter(User.username == 'default').scalar()
 
         reg_perm = UserToPerm()
         reg_perm.user = default_user
         reg_perm.permission = self.sa.query(Permission)\
         .filter(Permission.permission_name == 'hg.register.manual_activate')\
         .scalar()
 
         create_repo_perm = UserToPerm()
         create_repo_perm.user = default_user
         create_repo_perm.permission = self.sa.query(Permission)\
         .filter(Permission.permission_name == 'hg.create.repository')\
         .scalar()
 
         default_repo_perm = UserToPerm()
         default_repo_perm.user = default_user
         default_repo_perm.permission = self.sa.query(Permission)\
         .filter(Permission.permission_name == 'repository.read')\
         .scalar()
 
         try:
             self.sa.add(reg_perm)
             self.sa.add(create_repo_perm)
             self.sa.add(default_repo_perm)
             self.sa.commit()
         except:
             self.sa.rollback()
             raise
 

rhodecode/lib/helpers.py

@@ -1,383 +1,383 b''
 """Helper functions
 
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
 from pygments.formatters import HtmlFormatter
 from pygments import highlight as code_highlight
 from pylons import url, app_globals as g
 from pylons.i18n.translation import _, ungettext
 from vcs.utils.annotate import annotate_highlight
 from webhelpers.html import literal, HTML, escape
 from webhelpers.html.tools import *
 from webhelpers.html.builder import make_tag
 from webhelpers.html.tags import auto_discovery_link, checkbox, css_classes, \
     end_form, file, form, hidden, image, javascript_link, link_to, link_to_if, \
     link_to_unless, ol, required_legend, select, stylesheet_link, submit, text, \
     password, textarea, title, ul, xml_declaration, radio
 from webhelpers.html.tools import auto_link, button_to, highlight, js_obfuscate, \
     mail_to, strip_links, strip_tags, tag_re
 from webhelpers.number import format_byte_size, format_bit_size
 from webhelpers.pylonslib import Flash as _Flash
 from webhelpers.pylonslib.secure_form import secure_form
 from webhelpers.text import chop_at, collapse, convert_accented_entities, \
     convert_misc_entities, lchop, plural, rchop, remove_formatting, \
     replace_whitespace, urlify, truncate, wrap_paragraphs
 
 #Custom helpers here :)
 class _Link(object):
     '''
     Make a url based on label and url with help of url_for
     :param label:name of link    if not defined url is used
     :param url: the url for link
     '''
 
     def __call__(self, label='', *url_, **urlargs):
         if label is None or '':
             label = url
         link_fn = link_to(label, url(*url_, **urlargs))
         return link_fn
 
 link = _Link()
 
 class _GetError(object):
 
     def __call__(self, field_name, form_errors):
         tmpl = """<span class="error_msg">%s</span>"""
         if form_errors and form_errors.has_key(field_name):
             return literal(tmpl % form_errors.get(field_name))
 
 get_error = _GetError()
 
 def recursive_replace(str, replace=' '):
     """
     Recursive replace of given sign to just one instance
     :param str: given string
     :param replace:char to find and replace multiple instances
         
     Examples::
     >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
     'Mighty-Mighty-Bo-sstones'
     """
 
     if str.find(replace * 2) == -1:
         return str
     else:
         str = str.replace(replace * 2, replace)
         return recursive_replace(str, replace)  
 
 class _ToolTip(object):
     
     def __call__(self, tooltip_title, trim_at=50):
         """
         Special function just to wrap our text into nice formatted autowrapped
         text
         :param tooltip_title:
         """
         
         return wrap_paragraphs(escape(tooltip_title), trim_at)\
                        .replace('\n', '<br/>')
     
     def activate(self):
         """
         Adds tooltip mechanism to the given Html all tooltips have to have 
         set class tooltip and set attribute tooltip_title.
         Then a tooltip will be generated based on that
         All with yui js tooltip
         """
         
         js = '''
         YAHOO.util.Event.onDOMReady(function(){
             function toolTipsId(){
                 var ids = [];
                 var tts = YAHOO.util.Dom.getElementsByClassName('tooltip');
                 
                 for (var i = 0; i < tts.length; i++) {
                     //if element doesn not have and id autgenerate one for tooltip
                     
                     if (!tts[i].id){
                         tts[i].id='tt'+i*100;
                     }
                     ids.push(tts[i].id);
                 }
                 return ids        
             };
             var myToolTips = new YAHOO.widget.Tooltip("tooltip", { 
                 context: toolTipsId(),
                 monitorresize:false,
                 xyoffset :[0,0],
                 autodismissdelay:300000,
                 hidedelay:5,
                 showdelay:20,
             });
             
             //Mouse Over event disabled for new repositories since they dont
             //have last commit message
             myToolTips.contextMouseOverEvent.subscribe(
                 function(type, args) {
                     var context = args[0];
                     var txt = context.getAttribute('tooltip_title');
                     if(txt){                                       
                         return true;
                     }
                     else{
                         return false;
                     }
                 });
             
                             
             // Set the text for the tooltip just before we display it. Lazy method
             myToolTips.contextTriggerEvent.subscribe( 
                  function(type, args) { 
 
                  
                         var context = args[0]; 
                         
                         var txt = context.getAttribute('tooltip_title');
                         this.cfg.setProperty("text", txt);
                         
                         
                         // positioning of tooltip
                         var tt_w = this.element.clientWidth;
                         var tt_h = this.element.clientHeight;
                         
                         var context_w = context.offsetWidth;
                         var context_h = context.offsetHeight;
                         
                         var pos_x = YAHOO.util.Dom.getX(context);
                         var pos_y = YAHOO.util.Dom.getY(context);
 
                         var display_strategy = 'top';
                         var xy_pos = [0,0];
                         switch (display_strategy){
                         
                             case 'top':
                                 var cur_x = (pos_x+context_w/2)-(tt_w/2);
                                 var cur_y = pos_y-tt_h-4;
                                 xy_pos = [cur_x,cur_y];                                
                                 break;
                             case 'bottom':
                                 var cur_x = (pos_x+context_w/2)-(tt_w/2);
                                 var cur_y = pos_y+context_h+4;
                                 xy_pos = [cur_x,cur_y];                                
                                 break;
                             case 'left':
                                 var cur_x = (pos_x-tt_w-4);
                                 var cur_y = pos_y-((tt_h/2)-context_h/2);
                                 xy_pos = [cur_x,cur_y];                                
                                 break;
                             case 'right':
                                 var cur_x = (pos_x+context_w+4);
                                 var cur_y = pos_y-((tt_h/2)-context_h/2);
                                 xy_pos = [cur_x,cur_y];                                
                                 break;
                              default:
                                 var cur_x = (pos_x+context_w/2)-(tt_w/2);
                                 var cur_y = pos_y-tt_h-4;
                                 xy_pos = [cur_x,cur_y];                                
                                 break;                             
                                  
                         }
 
                         this.cfg.setProperty("xy",xy_pos);
 
                   });
                   
             //Mouse out 
             myToolTips.contextMouseOutEvent.subscribe(
                 function(type, args) {
                     var context = args[0];
                     
                 });
         });
         '''         
         return literal(js)
 
 tooltip = _ToolTip()
 
 class _FilesBreadCrumbs(object):
     
     def __call__(self, repo_name, rev, paths):
         url_l = [link_to(repo_name, url('files_home',
                                         repo_name=repo_name,
                                         revision=rev, f_path=''))]
         paths_l = paths.split('/')
         
         for cnt, p in enumerate(paths_l, 1):
             if p != '':
                 url_l.append(link_to(p, url('files_home',
                                             repo_name=repo_name,
                                             revision=rev,
                                             f_path='/'.join(paths_l[:cnt]))))
 
         return literal('/'.join(url_l))
 
 files_breadcrumbs = _FilesBreadCrumbs()
 class CodeHtmlFormatter(HtmlFormatter):
 
     def wrap(self, source, outfile):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source, 1):
             i, t = it
             t = '<div id="#S-%s">%s</div>' % (cnt, t)
             yield i, t
 def pygmentize(filenode, **kwargs):
     """
     pygmentize function using pygments
     :param filenode:
     """
     return literal(code_highlight(filenode.content,
                                   filenode.lexer, CodeHtmlFormatter(**kwargs)))
 
 def pygmentize_annotation(filenode, **kwargs):
     """
     pygmentize function for annotation
     :param filenode:
     """
     
     color_dict = {}
     def gen_color():
         """generator for getting 10k of evenly distibuted colors using hsv color
         and golden ratio.
         """        
         import colorsys
         n = 10000
         golden_ratio = 0.618033988749895
         h = 0.22717784590367374
         #generate 10k nice web friendly colors in the same order
         for c in xrange(n):
             h += golden_ratio
             h %= 1
             HSV_tuple = [h, 0.95, 0.95]
             RGB_tuple = colorsys.hsv_to_rgb(*HSV_tuple)
             yield map(lambda x:str(int(x * 256)), RGB_tuple)           
 
     cgenerator = gen_color()
         
     def get_color_string(cs):
         if color_dict.has_key(cs):
             col = color_dict[cs]
         else:
             col = color_dict[cs] = cgenerator.next()
         return "color: rgb(%s)! important;" % (', '.join(col))
         
     def url_func(changeset):
         tooltip_html = "<div style='font-size:0.8em'><b>Author:</b>" + \
         " %s<br/><b>Date:</b> %s</b><br/><b>Message:</b> %s<br/></div>" 
         
         tooltip_html = tooltip_html % (changeset.author,
                                                changeset.date,
                                                tooltip(changeset.message))
         lnk_format = 'r%-5s:%s' % (changeset.revision,
                                  changeset.short_id)
         uri = link_to(
                 lnk_format,
                 url('changeset_home', repo_name=changeset.repository.name,
                     revision=changeset.short_id),
                 style=get_color_string(changeset.short_id),
                 class_='tooltip',
                 tooltip_title=tooltip_html
               )
         
         uri += '\n'
         return uri   
     return literal(annotate_highlight(filenode, url_func, **kwargs))
       
 def repo_name_slug(value):
     """Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
     slug = remove_formatting(value)
     slug = strip_tags(slug)
     
     for c in """=[]\;'"<>,/~!@#$%^&*()+{}|: """:
         slug = slug.replace(c, '-')
     slug = recursive_replace(slug, '-')
     slug = collapse(slug, '-')
     return slug
 
 def get_changeset_safe(repo, rev):
     from vcs.backends.base import BaseRepository
     from vcs.exceptions import RepositoryError
     if not isinstance(repo, BaseRepository):
         raise Exception('You must pass an Repository '
                         'object as first argument got %s', type(repo))
         
     try:
         cs = repo.get_changeset(rev)
     except RepositoryError:
         from rhodecode.lib.utils import EmptyChangeset
         cs = EmptyChangeset()
     return cs
 
 
 flash = _Flash()
 
 
 #===============================================================================
 # MERCURIAL FILTERS available via h.
 #===============================================================================
 from mercurial import util
 from mercurial.templatefilters import age as _age, person as _person
 
-age = lambda  x:_age(x)
+age = lambda  x:x
 capitalize = lambda x: x.capitalize()
 date = lambda x: util.datestr(x)
 email = util.email
 email_or_none = lambda x: util.email(x) if util.email(x) != x else None
 person = lambda x: _person(x)
 hgdate = lambda  x: "%d %d" % x
 isodate = lambda  x: util.datestr(x, '%Y-%m-%d %H:%M %1%2')
 isodatesec = lambda  x: util.datestr(x, '%Y-%m-%d %H:%M:%S %1%2')
 localdate = lambda  x: (x[0], util.makedate()[1])
-rfc822date = lambda  x: util.datestr(x, "%a, %d %b %Y %H:%M:%S %1%2")
-rfc822date_notz = lambda  x: util.datestr(x, "%a, %d %b %Y %H:%M:%S")
+rfc822date = lambda  x: x#util.datestr(x, "%a, %d %b %Y %H:%M:%S %1%2")
+rfc822date_notz = lambda  x: x#util.datestr(x, "%a, %d %b %Y %H:%M:%S")
 rfc3339date = lambda  x: util.datestr(x, "%Y-%m-%dT%H:%M:%S%1:%2")
 time_ago = lambda x: util.datestr(_age(x), "%a, %d %b %Y %H:%M:%S %1%2")
 
 
 #===============================================================================
 # PERMS
 #===============================================================================
 from rhodecode.lib.auth import HasPermissionAny, HasPermissionAll, \
 HasRepoPermissionAny, HasRepoPermissionAll
 
 #===============================================================================
 # GRAVATAR URL
 #===============================================================================
 import hashlib
 import urllib
 from pylons import request
 
 def gravatar_url(email_address, size=30):
     ssl_enabled = 'https' == request.environ.get('HTTP_X_URL_SCHEME')
     default = 'identicon'
     baseurl_nossl = "http://www.gravatar.com/avatar/"
     baseurl_ssl = "https://secure.gravatar.com/avatar/"
     baseurl = baseurl_ssl if ssl_enabled else baseurl_nossl
         
     
     # construct the url
     gravatar_url = baseurl + hashlib.md5(email_address.lower()).hexdigest() + "?"
     gravatar_url += urllib.urlencode({'d':default, 's':str(size)})
 
     return gravatar_url
 
 def safe_unicode(str):
     """safe unicode function. In case of UnicodeDecode error we try to return
     unicode with errors replace, if this failes we return unicode with 
     string_escape decoding """
     
     try:
         u_str = unicode(str)
     except UnicodeDecodeError:
         try:
             u_str = unicode(str, 'utf-8', 'replace')
         except UnicodeDecodeError:
             #incase we have a decode error just represent as byte string
             u_str = unicode(str(str).encode('string_escape'))
         
     return u_str

rhodecode/lib/indexers/__init__.py

@@ -1,142 +1,196 b''
+import os
+import sys
 from os.path import dirname as dn, join as jn
+
+#to get the rhodecode import
+sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
+
 from rhodecode.config.environment import load_environment
 from rhodecode.model.hg import HgModel
 from shutil import rmtree
 from webhelpers.html.builder import escape
 from vcs.utils.lazy import LazyProperty
 
 from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter
 from whoosh.fields import TEXT, ID, STORED, Schema, FieldType
 from whoosh.index import create_in, open_dir
 from whoosh.formats import Characters
 from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter   
 
-import os
-import sys
 import traceback
 
-#to get the rhodecode import
-sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
-
 
 #LOCATION WE KEEP THE INDEX
 IDX_LOCATION = jn(dn(dn(dn(dn(os.path.abspath(__file__))))), 'data', 'index')
 
 #EXTENSIONS WE WANT TO INDEX CONTENT OFF
 INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
                     'cfg', 'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl',
                     'h', 'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp',
                     'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3',
                     'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql',
                     'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt',
                     'yaws']
 
 #CUSTOM ANALYZER wordsplit + lowercase filter
 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
 
 
 #INDEX SCHEMA DEFINITION
 SCHEMA = Schema(owner=TEXT(),
                 repository=TEXT(stored=True),
                 path=TEXT(stored=True),
                 content=FieldType(format=Characters(ANALYZER),
                              scorable=True, stored=True),
                 modtime=STORED(), extension=TEXT(stored=True))
 
 
 IDX_NAME = 'HG_INDEX'
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n') 
 FRAGMENTER = SimpleFragmenter(200)
                             
+from paste.script import command
+import ConfigParser
+
+class MakeIndex(command.Command):
+
+    max_args = 1
+    min_args = 1
+
+    usage = "CONFIG_FILE"
+    summary = "Creates index for full text search given configuration file"
+    group_name = "Whoosh indexing"
+
+    parser = command.Command.standard_parser(verbose=True)
+#    parser.add_option('--repo-location',
+#                      action='store',
+#                      dest='repo_location',
+#                      help="Specifies repositories location to index",
+#                      )
+    parser.add_option('-f',
+                      action='store_true',
+                      dest='full_index',
+                      help="Specifies that index should be made full i.e"
+                            " destroy old and build from scratch",
+                      default=False)
+    def command(self):
+        config_name = self.args[0]
+
+        p = config_name.split('/')
+        if len(p) == 1:
+            root = '.'
+        else:
+            root = '/'.join(p[:-1])
+        print root
+        config = ConfigParser.ConfigParser({'here':root})
+        config.read(config_name)
+        print dict(config.items('app:main'))['index_dir']
+        index_location = dict(config.items('app:main'))['index_dir']
+        #return
+
+        #=======================================================================
+        # WHOOSH DAEMON
+        #=======================================================================
+        from rhodecode.lib.pidlock import LockHeld, DaemonLock
+        from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
+        try:
+            l = DaemonLock()
+            WhooshIndexingDaemon(index_location=index_location)\
+                .run(full_index=self.options.full_index)
+            l.release()
+        except LockHeld:
+            sys.exit(1)
+
+
 class ResultWrapper(object):
     def __init__(self, search_type, searcher, matcher, highlight_items):
         self.search_type = search_type
         self.searcher = searcher
         self.matcher = matcher
         self.highlight_items = highlight_items
         self.fragment_size = 200 / 2
     
     @LazyProperty
     def doc_ids(self):
         docs_id = []
         while self.matcher.is_active():
             docnum = self.matcher.id()
             chunks = [offsets for offsets in self.get_chunks()]
             docs_id.append([docnum, chunks])
             self.matcher.next()
         return docs_id   
         
     def __str__(self):
         return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids))
 
     def __repr__(self):
         return self.__str__()
 
     def __len__(self):
         return len(self.doc_ids)
 
     def __iter__(self):
         """
         Allows Iteration over results,and lazy generate content
 
         *Requires* implementation of ``__getitem__`` method.
         """
         for docid in self.doc_ids:
             yield self.get_full_content(docid)
 
     def __getslice__(self, i, j):
         """
         Slicing of resultWrapper
         """
         slice = []
         for docid in self.doc_ids[i:j]:
             slice.append(self.get_full_content(docid))
         return slice   
                             
 
     def get_full_content(self, docid):
         res = self.searcher.stored_fields(docid[0])
         f_path = res['path'][res['path'].find(res['repository']) \
                              + len(res['repository']):].lstrip('/')
         
         content_short = self.get_short_content(res, docid[1])
         res.update({'content_short':content_short,
                     'content_short_hl':self.highlight(content_short),
                     'f_path':f_path})
         
         return res        
     
     def get_short_content(self, res, chunks):
         
         return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks])
     
     def get_chunks(self):
         """
         Smart function that implements chunking the content
         but not overlap chunks so it doesn't highlight the same
         close occurrences twice.
-        :param matcher:
-        :param size:
+        @param matcher:
+        @param size:
         """
         memory = [(0, 0)]
         for span in self.matcher.spans():
             start = span.startchar or 0
             end = span.endchar or 0
             start_offseted = max(0, start - self.fragment_size)
             end_offseted = end + self.fragment_size
             
             if start_offseted < memory[-1][1]:
                 start_offseted = memory[-1][1]
             memory.append((start_offseted, end_offseted,))    
             yield (start_offseted, end_offseted,)  
         
     def highlight(self, content, top=5):
         if self.search_type != 'content':
             return ''
         hl = highlight(escape(content),
                  self.highlight_items,
                  analyzer=ANALYZER,
                  fragmenter=FRAGMENTER,
                  formatter=FORMATTER,
                  top=top)
         return hl 

rhodecode/lib/indexers/daemon.py

@@ -1,248 +1,221 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # whoosh indexer daemon for rhodecode
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on Jan 26, 2010
 
 @author: marcink
 A deamon will read from task table and run tasks
 """
 import sys
 import os
 from os.path import dirname as dn
 from os.path import join as jn
 
 #to get the rhodecode import
 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 
-from rhodecode.lib.pidlock import LockHeld, DaemonLock
+
 from rhodecode.model.hg import HgModel
 from rhodecode.lib.helpers import safe_unicode
 from whoosh.index import create_in, open_dir
 from shutil import rmtree
-from rhodecode.lib.indexers import INDEX_EXTENSIONS, IDX_LOCATION, SCHEMA, IDX_NAME
+from rhodecode.lib.indexers import INDEX_EXTENSIONS, SCHEMA, IDX_NAME
 
 from time import mktime
 from vcs.exceptions import ChangesetError, RepositoryError
 
 import logging
 
 log = logging.getLogger('whooshIndexer')
 # create logger
 log.setLevel(logging.DEBUG)
 log.propagate = False
 # create console handler and set level to debug
 ch = logging.StreamHandler()
 ch.setLevel(logging.DEBUG)
 
 # create formatter
 formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 
 # add formatter to ch
 ch.setFormatter(formatter)
 
 # add ch to logger
 log.addHandler(ch)
 
-def scan_paths(root_location):
-    return HgModel.repo_scan('/', root_location, None, True)
+def get_repos_location():
+    return HgModel.get_repos_location()
+
 
 class WhooshIndexingDaemon(object):
     """
     Deamon for atomic jobs
     """
 
-    def __init__(self, indexname='HG_INDEX', repo_location=None):
+    def __init__(self, indexname='HG_INDEX', index_location=None,
+                 repo_location=None):
         self.indexname = indexname
+
+        self.index_location = index_location
+        if not index_location:
+            raise Exception('You have to provide index location')
+
         self.repo_location = repo_location
-        self.repo_paths = scan_paths(self.repo_location)
+        if not repo_location:
+            raise Exception('You have to provide repositories location')
+
+
+
+        self.repo_paths = HgModel.repo_scan('/', self.repo_location, None, True)
         self.initial = False
-        if not os.path.isdir(IDX_LOCATION):
-            os.mkdir(IDX_LOCATION)
+        if not os.path.isdir(self.index_location):
+            os.mkdir(self.index_location)
             log.info('Cannot run incremental index since it does not'
                      ' yet exist running full build')
             self.initial = True
         
     def get_paths(self, repo):
         """
         recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
         """
         index_paths_ = set()
         try:
-            tip = repo.get_changeset()
-            
-            for topnode, dirs, files in tip.walk('/'):
+            for topnode, dirs, files in repo.walk('/', 'tip'):
                 for f in files:
                     index_paths_.add(jn(repo.path, f.path))
                 for dir in dirs:
                     for f in files:
                         index_paths_.add(jn(repo.path, f.path))
                 
         except RepositoryError:
             pass
         return index_paths_        
     
     def get_node(self, repo, path):
         n_path = path[len(repo.path) + 1:]
         node = repo.get_changeset().get_node(n_path)
         return node
     
     def get_node_mtime(self, node):
         return mktime(node.last_changeset.date.timetuple())
     
     def add_doc(self, writer, path, repo):
         """Adding doc to writer"""
         node = self.get_node(repo, path)
 
         #we just index the content of chosen files
         if node.extension in INDEX_EXTENSIONS:
             log.debug('    >> %s [WITH CONTENT]' % path)
             u_content = node.content
         else:
             log.debug('    >> %s' % path)
             #just index file name without it's content
             u_content = u''
         
         writer.add_document(owner=unicode(repo.contact),
                         repository=safe_unicode(repo.name),
                         path=safe_unicode(path),
                         content=u_content,
                         modtime=self.get_node_mtime(node),
                         extension=node.extension)             
 
     
     def build_index(self):
-        if os.path.exists(IDX_LOCATION):
+        if os.path.exists(self.index_location):
             log.debug('removing previous index')
-            rmtree(IDX_LOCATION)
+            rmtree(self.index_location)
             
-        if not os.path.exists(IDX_LOCATION):
-            os.mkdir(IDX_LOCATION)
+        if not os.path.exists(self.index_location):
+            os.mkdir(self.index_location)
         
-        idx = create_in(IDX_LOCATION, SCHEMA, indexname=IDX_NAME)
+        idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)
         writer = idx.writer()
         
         for cnt, repo in enumerate(self.repo_paths.values()):
             log.debug('building index @ %s' % repo.path)
         
             for idx_path in self.get_paths(repo):
                 self.add_doc(writer, idx_path, repo)
         
         log.debug('>> COMMITING CHANGES <<')
         writer.commit(merge=True)
         log.debug('>>> FINISHED BUILDING INDEX <<<')
             
     
     def update_index(self):
         log.debug('STARTING INCREMENTAL INDEXING UPDATE')
             
-        idx = open_dir(IDX_LOCATION, indexname=self.indexname)
+        idx = open_dir(self.index_location, indexname=self.indexname)
         # The set of all paths in the index
         indexed_paths = set()
         # The set of all paths we need to re-index
         to_index = set()
         
         reader = idx.reader()
         writer = idx.writer()
     
         # Loop over the stored fields in the index
         for fields in reader.all_stored_fields():
             indexed_path = fields['path']
             indexed_paths.add(indexed_path)
             
             repo = self.repo_paths[fields['repository']]
             
             try:
                 node = self.get_node(repo, indexed_path)
             except ChangesetError:
                 # This file was deleted since it was indexed
                 log.debug('removing from index %s' % indexed_path)
                 writer.delete_by_term('path', indexed_path)
     
             else:
                 # Check if this file was changed since it was indexed
                 indexed_time = fields['modtime']
                 mtime = self.get_node_mtime(node)
                 if mtime > indexed_time:
                     # The file has changed, delete it and add it to the list of
                     # files to reindex
                     log.debug('adding to reindex list %s' % indexed_path)
                     writer.delete_by_term('path', indexed_path)
                     to_index.add(indexed_path)
     
         # Loop over the files in the filesystem
         # Assume we have a function that gathers the filenames of the
         # documents to be indexed
         for repo in self.repo_paths.values():
             for path in self.get_paths(repo):
                 if path in to_index or path not in indexed_paths:
                     # This is either a file that's changed, or a new file
                     # that wasn't indexed before. So index it!
                     self.add_doc(writer, path, repo)
                     log.debug('re indexing %s' % path)
                     
         log.debug('>> COMMITING CHANGES <<')
         writer.commit(merge=True)
         log.debug('>>> FINISHED REBUILDING INDEX <<<')
         
     def run(self, full_index=False):
         """Run daemon"""
         if full_index or self.initial:
             self.build_index()
         else:
             self.update_index()
-        
-if __name__ == "__main__":
-    arg = sys.argv[1:]
-    if len(arg) != 2:
-        sys.stderr.write('Please specify indexing type [full|incremental]' 
-                         'and path to repositories as script args \n')
-        sys.exit()
-    
-    
-    if arg[0] == 'full':
-        full_index = True
-    elif arg[0] == 'incremental':
-        # False means looking just for changes
-        full_index = False
-    else:
-        sys.stdout.write('Please use [full|incremental]' 
-                         ' as script first arg \n')
-        sys.exit()
-    
-    if not os.path.isdir(arg[1]):
-        sys.stderr.write('%s is not a valid path \n' % arg[1])
-        sys.exit()
-    else:
-        if arg[1].endswith('/'):
-            repo_location = arg[1] + '*'
-        else:
-            repo_location = arg[1] + '/*'
-    
-    try:
-        l = DaemonLock()
-        WhooshIndexingDaemon(repo_location=repo_location)\
-            .run(full_index=full_index)
-        l.release()
-        reload(logging)
-    except LockHeld:
-        sys.exit(1)
-

rhodecode/lib/utils.py

@@ -1,511 +1,533 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # Utilities for RhodeCode
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
+from UserDict import DictMixin
+from mercurial import ui, config, hg
+from mercurial.error import RepoError
+from rhodecode.model import meta
+from rhodecode.model.caching_query import FromCache
+from rhodecode.model.db import Repository, User, RhodeCodeUi, RhodeCodeSettings, \
+    UserLog
+from rhodecode.model.repo import RepoModel
+from rhodecode.model.user import UserModel
+from vcs.backends.base import BaseChangeset
+from vcs.backends.git import GitRepository
+from vcs.backends.hg import MercurialRepository
+from vcs.utils.lazy import LazyProperty
+import datetime
+import logging
+import os
 
 """
 Created on April 18, 2010
 Utilities for RhodeCode
 @author: marcink
 """
-from rhodecode.model.caching_query import FromCache
-from mercurial import ui, config, hg
-from mercurial.error import RepoError
-from rhodecode.model import meta
-from rhodecode.model.user import UserModel
-from rhodecode.model.repo import RepoModel
-from rhodecode.model.db import Repository, User, RhodeCodeUi, RhodeCodeSettings, UserLog
-from vcs.backends.base import BaseChangeset
-from vcs.utils.lazy import LazyProperty
-import logging
-import datetime
-import os
 
 log = logging.getLogger(__name__)
 
 
 def get_repo_slug(request):
     return request.environ['pylons.routes_dict'].get('repo_name')
 
 def is_mercurial(environ):
     """
     Returns True if request's target is mercurial server - header
     ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
     """
     http_accept = environ.get('HTTP_ACCEPT')
     if http_accept and http_accept.startswith('application/mercurial'):
         return True
     return False
 
 def is_git(environ):
     """
     Returns True if request's target is git server. ``HTTP_USER_AGENT`` would
     then have git client version given.
     
     :param environ:
     """
     http_user_agent = environ.get('HTTP_USER_AGENT')
     if http_user_agent.startswith('git'):
         return True
     return False
 
 def action_logger(user, action, repo, ipaddr, sa=None):
     """
     Action logger for various action made by users
     """
 
     if not sa:
         sa = meta.Session()
 
     try:
         if hasattr(user, 'user_id'):
             user_id = user.user_id
         elif isinstance(user, basestring):
             user_id = UserModel(sa).get_by_username(user, cache=False).user_id
         else:
             raise Exception('You have to provide user object or username')
 
         repo_name = repo.lstrip('/')
         user_log = UserLog()
         user_log.user_id = user_id
         user_log.action = action
         user_log.repository_name = repo_name
         user_log.repository = RepoModel(sa).get(repo_name, cache=False)
         user_log.action_date = datetime.datetime.now()
         user_log.user_ip = ipaddr
         sa.add(user_log)
         sa.commit()
 
         log.info('Adding user %s, action %s on %s',
                                         user.username, action, repo)
     except Exception, e:
         sa.rollback()
         log.error('could not log user action:%s', str(e))
 
-def check_repo_dir(paths):
-    repos_path = paths[0][1].split('/')
-    if repos_path[-1] in ['*', '**']:
-        repos_path = repos_path[:-1]
-    if repos_path[0] != '/':
-        repos_path[0] = '/'
-    if not os.path.isdir(os.path.join(*repos_path)):
-        raise Exception('Not a valid repository in %s' % paths[0][1])
+def get_repos(path, recursive=False, initial=False):
+    """
+    Scans given path for repos and return (name,(type,path)) tuple 
+    :param prefix:
+    :param path:
+    :param recursive:
+    :param initial:
+    """
+    from vcs.utils.helpers import get_scm
+    from vcs.exceptions import VCSError
+    scm = get_scm(path)
+    if scm:
+        raise Exception('The given path %s should not be a repository got %s',
+                        path, scm)
+
+    for dirpath in os.listdir(path):
+        try:
+            yield dirpath, get_scm(os.path.join(path, dirpath))
+        except VCSError:
+            pass
+
+if __name__ == '__main__':
+    get_repos('', '/home/marcink/workspace-python')
+
 
 def check_repo_fast(repo_name, base_path):
     if os.path.isdir(os.path.join(base_path, repo_name)):return False
     return True
 
 def check_repo(repo_name, base_path, verify=True):
 
     repo_path = os.path.join(base_path, repo_name)
 
     try:
         if not check_repo_fast(repo_name, base_path):
             return False
         r = hg.repository(ui.ui(), repo_path)
         if verify:
             hg.verify(r)
         #here we hnow that repo exists it was verified
         log.info('%s repo is already created', repo_name)
         return False
     except RepoError:
         #it means that there is no valid repo there...
         log.info('%s repo is free for creation', repo_name)
         return True
 
 def ask_ok(prompt, retries=4, complaint='Yes or no, please!'):
     while True:
         ok = raw_input(prompt)
         if ok in ('y', 'ye', 'yes'): return True
         if ok in ('n', 'no', 'nop', 'nope'): return False
         retries = retries - 1
         if retries < 0: raise IOError
         print complaint
 
 def get_hg_ui_cached():
     try:
         sa = meta.Session
         ret = sa.query(RhodeCodeUi)\
         .options(FromCache("sql_cache_short", "get_hg_ui_settings"))\
         .all()
     except:
         pass
     finally:
         meta.Session.remove()
     return ret
 
 
 def get_hg_settings():
     try:
         sa = meta.Session()
         ret = sa.query(RhodeCodeSettings)\
         .options(FromCache("sql_cache_short", "get_hg_settings"))\
         .all()
     except:
         pass
     finally:
         meta.Session.remove()
 
     if not ret:
         raise Exception('Could not get application settings !')
     settings = {}
     for each in ret:
         settings['rhodecode_' + each.app_settings_name] = each.app_settings_value
 
     return settings
 
 def get_hg_ui_settings():
     try:
         sa = meta.Session()
         ret = sa.query(RhodeCodeUi).all()
     except:
         pass
     finally:
         meta.Session.remove()
 
     if not ret:
         raise Exception('Could not get application ui settings !')
     settings = {}
     for each in ret:
         k = each.ui_key
         v = each.ui_value
         if k == '/':
             k = 'root_path'
 
         if k.find('.') != -1:
             k = k.replace('.', '_')
 
         if each.ui_section == 'hooks':
             v = each.ui_active
 
         settings[each.ui_section + '_' + k] = v
 
     return settings
 
 #propagated from mercurial documentation
 ui_sections = ['alias', 'auth',
                 'decode/encode', 'defaults',
                 'diff', 'email',
                 'extensions', 'format',
                 'merge-patterns', 'merge-tools',
                 'hooks', 'http_proxy',
                 'smtp', 'patch',
                 'paths', 'profiling',
                 'server', 'trusted',
                 'ui', 'web', ]
 
 def make_ui(read_from='file', path=None, checkpaths=True):
     """
     A function that will read python rc files or database
     and make an mercurial ui object from read options
     
     :param path: path to mercurial config file
     :param checkpaths: check the path
     :param read_from: read from 'file' or 'db'
     """
 
     baseui = ui.ui()
 
     if read_from == 'file':
         if not os.path.isfile(path):
             log.warning('Unable to read config file %s' % path)
             return False
         log.debug('reading hgrc from %s', path)
         cfg = config.config()
         cfg.read(path)
         for section in ui_sections:
             for k, v in cfg.items(section):
                 baseui.setconfig(section, k, v)
                 log.debug('settings ui from file[%s]%s:%s', section, k, v)
-        if checkpaths:check_repo_dir(cfg.items('paths'))
-
 
     elif read_from == 'db':
         hg_ui = get_hg_ui_cached()
         for ui_ in hg_ui:
             if ui_.ui_active:
                 log.debug('settings ui from db[%s]%s:%s', ui_.ui_section, ui_.ui_key, ui_.ui_value)
                 baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)
 
 
     return baseui
 
 
 def set_rhodecode_config(config):
     hgsettings = get_hg_settings()
 
     for k, v in hgsettings.items():
         config[k] = v
 
 def invalidate_cache(name, *args):
     """Invalidates given name cache"""
 
     from beaker.cache import region_invalidate
     log.info('INVALIDATING CACHE FOR %s', name)
 
     """propagate our arguments to make sure invalidation works. First
     argument has to be the name of cached func name give to cache decorator
     without that the invalidation would not work"""
     tmp = [name]
     tmp.extend(args)
     args = tuple(tmp)
 
     if name == 'cached_repo_list':
         from rhodecode.model.hg import _get_repos_cached
         region_invalidate(_get_repos_cached, None, *args)
 
     if name == 'full_changelog':
         from rhodecode.model.hg import _full_changelog_cached
         region_invalidate(_full_changelog_cached, None, *args)
 
 class EmptyChangeset(BaseChangeset):
     """
     An dummy empty changeset.
     """
 
     revision = -1
     message = ''
     author = ''
     date = ''
     @LazyProperty
     def raw_id(self):
         """
-        Returns raw string identifing this changeset, useful for web
+        Returns raw string identifying this changeset, useful for web
         representation.
         """
         return '0' * 40
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     def get_file_changeset(self, path):
         return self
 
     def get_file_content(self, path):
         return u''
 
     def get_file_size(self, path):
         return 0
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False):
     """
     maps all found repositories into db
     """
 
     sa = meta.Session()
+    rm = RepoModel(sa)
     user = sa.query(User).filter(User.admin == True).first()
 
-    rm = RepoModel()
+    for name, repo in initial_repo_list.items():
+        if not rm.get(name, cache=False):
+            log.info('repository %s not found creating default', name)
 
-    for name, repo in initial_repo_list.items():
-        if not RepoModel(sa).get(name, cache=False):
-            log.info('repository %s not found creating default', name)
+            if isinstance(repo, MercurialRepository):
+                repo_type = 'hg'
+            if isinstance(repo, GitRepository):
+                repo_type = 'git'
 
             form_data = {
                          'repo_name':name,
+                         'repo_type':repo_type,
                          'description':repo.description if repo.description != 'unknown' else \
                                         'auto description for %s' % name,
                          'private':False
                          }
             rm.create(form_data, user, just_db=True)
 
 
     if remove_obsolete:
         #remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in initial_repo_list.keys():
                 sa.delete(repo)
                 sa.commit()
 
 
     meta.Session.remove()
 
-from UserDict import DictMixin
 
 class OrderedDict(dict, DictMixin):
 
     def __init__(self, *args, **kwds):
         if len(args) > 1:
             raise TypeError('expected at most 1 arguments, got %d' % len(args))
         try:
             self.__end
         except AttributeError:
             self.clear()
         self.update(*args, **kwds)
 
     def clear(self):
         self.__end = end = []
         end += [None, end, end]         # sentinel node for doubly linked list
         self.__map = {}                 # key --> [key, prev, next]
         dict.clear(self)
 
     def __setitem__(self, key, value):
         if key not in self:
             end = self.__end
             curr = end[1]
             curr[2] = end[1] = self.__map[key] = [key, curr, end]
         dict.__setitem__(self, key, value)
 
     def __delitem__(self, key):
         dict.__delitem__(self, key)
         key, prev, next = self.__map.pop(key)
         prev[2] = next
         next[1] = prev
 
     def __iter__(self):
         end = self.__end
         curr = end[2]
         while curr is not end:
             yield curr[0]
             curr = curr[2]
 
     def __reversed__(self):
         end = self.__end
         curr = end[1]
         while curr is not end:
             yield curr[0]
             curr = curr[1]
 
     def popitem(self, last=True):
         if not self:
             raise KeyError('dictionary is empty')
         if last:
             key = reversed(self).next()
         else:
             key = iter(self).next()
         value = self.pop(key)
         return key, value
 
     def __reduce__(self):
         items = [[k, self[k]] for k in self]
         tmp = self.__map, self.__end
         del self.__map, self.__end
         inst_dict = vars(self).copy()
         self.__map, self.__end = tmp
         if inst_dict:
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)
 
     def keys(self):
         return list(self)
 
     setdefault = DictMixin.setdefault
     update = DictMixin.update
     pop = DictMixin.pop
     values = DictMixin.values
     items = DictMixin.items
     iterkeys = DictMixin.iterkeys
     itervalues = DictMixin.itervalues
     iteritems = DictMixin.iteritems
 
     def __repr__(self):
         if not self:
             return '%s()' % (self.__class__.__name__,)
         return '%s(%r)' % (self.__class__.__name__, self.items())
 
     def copy(self):
         return self.__class__(self)
 
     @classmethod
     def fromkeys(cls, iterable, value=None):
         d = cls()
         for key in iterable:
             d[key] = value
         return d
 
     def __eq__(self, other):
         if isinstance(other, OrderedDict):
             return len(self) == len(other) and self.items() == other.items()
         return dict.__eq__(self, other)
 
     def __ne__(self, other):
         return not self == other
 
 
 #===============================================================================
 # TEST FUNCTIONS AND CREATORS
 #===============================================================================
 def create_test_index(repo_location, full_index):
     """Makes default test index
     :param repo_location:
     :param full_index:
     """
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     from rhodecode.lib.pidlock import DaemonLock, LockHeld
     from rhodecode.lib.indexers import IDX_LOCATION
     import shutil
 
     if os.path.exists(IDX_LOCATION):
         shutil.rmtree(IDX_LOCATION)
 
     try:
         l = DaemonLock()
         WhooshIndexingDaemon(repo_location=repo_location)\
             .run(full_index=full_index)
         l.release()
     except LockHeld:
         pass
 
 def create_test_env(repos_test_path, config):
     """Makes a fresh database and 
     install test repository into tmp dir
     """
     from rhodecode.lib.db_manage import DbManage
     import tarfile
     import shutil
     from os.path import dirname as dn, join as jn, abspath
 
     log = logging.getLogger('TestEnvCreator')
     # create logger
     log.setLevel(logging.DEBUG)
     log.propagate = True
     # create console handler and set level to debug
     ch = logging.StreamHandler()
     ch.setLevel(logging.DEBUG)
 
     # create formatter
     formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 
     # add formatter to ch
     ch.setFormatter(formatter)
 
     # add ch to logger
     log.addHandler(ch)
 
     #PART ONE create db
     dbname = config['sqlalchemy.db1.url'].split('/')[-1]
     log.debug('making test db %s', dbname)
 
     dbmanage = DbManage(log_sql=True, dbname=dbname, root=config['here'],
                         tests=True)
     dbmanage.create_tables(override=True)
     dbmanage.config_prompt(repos_test_path)
     dbmanage.create_default_user()
     dbmanage.admin_prompt()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
 
     #PART TWO make test repo
     log.debug('making test vcs repo')
     if os.path.isdir('/tmp/vcs_test'):
         shutil.rmtree('/tmp/vcs_test')
 
     cur_dir = dn(dn(abspath(__file__)))
     tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test.tar.gz"))
     tar.extractall('/tmp')
     tar.close()

rhodecode/model/db.py

@@ -1,139 +1,140 b''
 from rhodecode.model.meta import Base
 from sqlalchemy import *
 from sqlalchemy.orm import relation, backref
 from sqlalchemy.orm.session import Session
 from vcs.utils.lazy import LazyProperty
 import logging
 
 log = logging.getLogger(__name__)
 
 class RhodeCodeSettings(Base):
     __tablename__ = 'rhodecode_settings'
     __table_args__ = (UniqueConstraint('app_settings_name'), {'useexisting':True})
     app_settings_id = Column("app_settings_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     app_settings_name = Column("app_settings_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     app_settings_value = Column("app_settings_value", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
 class RhodeCodeUi(Base):
     __tablename__ = 'rhodecode_ui'
     __table_args__ = {'useexisting':True}
     ui_id = Column("ui_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     ui_section = Column("ui_section", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_key = Column("ui_key", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_value = Column("ui_value", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_active = Column("ui_active", BOOLEAN(), nullable=True, unique=None, default=True)
     
     
 class User(Base): 
     __tablename__ = 'users'
     __table_args__ = (UniqueConstraint('username'), UniqueConstraint('email'), {'useexisting':True})
     user_id = Column("user_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     username = Column("username", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     password = Column("password", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     active = Column("active", BOOLEAN(), nullable=True, unique=None, default=None)
     admin = Column("admin", BOOLEAN(), nullable=True, unique=None, default=False)
     name = Column("name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     lastname = Column("lastname", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     email = Column("email", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     last_login = Column("last_login", DATETIME(timezone=False), nullable=True, unique=None, default=None)
     
     user_log = relation('UserLog')
     user_perms = relation('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id")
     
     @LazyProperty
     def full_contact(self):
         return '%s %s <%s>' % (self.name, self.lastname, self.email)
         
     def __repr__(self):
         return "<User('id:%s:%s')>" % (self.user_id, self.username)
     
     def update_lastlogin(self):
         """Update user lastlogin"""
         import datetime
         
         try:
             session = Session.object_session(self)
             self.last_login = datetime.datetime.now()
             session.add(self)
             session.commit()
             log.debug('updated user %s lastlogin', self.username)
         except Exception:
             session.rollback()        
     
       
 class UserLog(Base): 
     __tablename__ = 'user_logs'
     __table_args__ = {'useexisting':True}
     user_log_id = Column("user_log_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", INTEGER(length=None, convert_unicode=False, assert_unicode=None), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None)
     repository_name = Column("repository_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     user_ip = Column("user_ip", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None) 
     action = Column("action", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action_date = Column("action_date", DATETIME(timezone=False), nullable=True, unique=None, default=None)
     revision = Column('revision', TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     
     user = relation('User')
     repository = relation('Repository')
     
 class Repository(Base):
     __tablename__ = 'repositories'
     __table_args__ = (UniqueConstraint('repo_name'), {'useexisting':True},)
     repo_id = Column("repo_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     repo_name = Column("repo_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
+    repo_type = Column("repo_type", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default=None)
     user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=False, default=None)
     private = Column("private", BOOLEAN(), nullable=True, unique=None, default=None)
     description = Column("description", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     fork_id = Column("fork_id", INTEGER(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=False, default=None)
     
     user = relation('User')
     fork = relation('Repository', remote_side=repo_id)
     repo_to_perm = relation('RepoToPerm', cascade='all')
     
     def __repr__(self):
         return "<Repository('id:%s:%s')>" % (self.repo_id, self.repo_name)
         
 class Permission(Base):
     __tablename__ = 'permissions'
     __table_args__ = {'useexisting':True}
     permission_id = Column("permission_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     permission_name = Column("permission_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     permission_longname = Column("permission_longname", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     
     def __repr__(self):
         return "<Permission('%s:%s')>" % (self.permission_id, self.permission_name)
 
 class RepoToPerm(Base):
     __tablename__ = 'repo_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'repository_id'), {'useexisting':True})
     repo_to_perm_id = Column("repo_to_perm_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", INTEGER(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", INTEGER(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None) 
     
     user = relation('User')
     permission = relation('Permission')
     repository = relation('Repository')
 
 class UserToPerm(Base):
     __tablename__ = 'user_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'permission_id'), {'useexisting':True})
     user_to_perm_id = Column("user_to_perm_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", INTEGER(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
     
     user = relation('User')
     permission = relation('Permission')
 
 class Statistics(Base):
     __tablename__ = 'statistics'
     __table_args__ = (UniqueConstraint('repository_id'), {'useexisting':True})
     stat_id = Column("stat_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     repository_id = Column("repository_id", INTEGER(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=True, default=None)
     stat_on_revision = Column("stat_on_revision", INTEGER(), nullable=False)
     commit_activity = Column("commit_activity", BLOB(), nullable=False)#JSON data
     commit_activity_combined = Column("commit_activity_combined", BLOB(), nullable=False)#JSON data
     languages = Column("languages", BLOB(), nullable=False)#JSON data
     
     repository = relation('Repository')
 

rhodecode/model/forms.py

@@ -1,357 +1,353 b''
 """ this is forms validation classes
 http://formencode.org/module-formencode.validators.html
 for list off all availible validators
 
 we can create our own validators
 
 The table below outlines the options which can be used in a schema in addition to the validators themselves
 pre_validators          []     These validators will be applied before the schema
 chained_validators      []     These validators will be applied after the schema
 allow_extra_fields      False     If True, then it is not an error when keys that aren't associated with a validator are present
 filter_extra_fields     False     If True, then keys that aren't associated with a validator are removed
 if_key_missing          NoDefault If this is given, then any keys that aren't available but are expected will be replaced with this value (and then validated). This does not override a present .if_missing attribute on validators. NoDefault is a special FormEncode class to mean that no default values has been specified and therefore missing keys shouldn't take a default value.
 ignore_key_missing      False     If True, then missing keys will be missing in the result, if the validator doesn't have .if_missing on it already    
   
   
 <name> = formencode.validators.<name of validator>
 <name> must equal form name
 list=[1,2,3,4,5]
 for SELECT use formencode.All(OneOf(list), Int())
     
 """
 from formencode import All
 from formencode.validators import UnicodeString, OneOf, Int, Number, Regex, \
     Email, Bool, StringBoolean
 from pylons import session
 from pylons.i18n.translation import _
 from rhodecode.lib.auth import check_password, get_crypt_password
 from rhodecode.model import meta
 from rhodecode.model.user import UserModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.db import User
 from webhelpers.pylonslib.secure_form import authentication_token
 import formencode
 import logging
 import os
 import rhodecode.lib.helpers as h
 
 log = logging.getLogger(__name__)
 
 #this is needed to translate the messages using _() in validators
 class State_obj(object):
     _ = staticmethod(_)
 
 #===============================================================================
 # VALIDATORS
 #===============================================================================
 class ValidAuthToken(formencode.validators.FancyValidator):
     messages = {'invalid_token':_('Token mismatch')}
 
     def validate_python(self, value, state):
 
         if value != authentication_token():
             raise formencode.Invalid(self.message('invalid_token', state,
                                             search_number=value), value, state)
 
 def ValidUsername(edit, old_data):
     class _ValidUsername(formencode.validators.FancyValidator):
 
         def validate_python(self, value, state):
             if value in ['default', 'new_user']:
                 raise formencode.Invalid(_('Invalid username'), value, state)
             #check if user is unique
             old_un = None
             if edit:
                 old_un = UserModel().get(old_data.get('user_id')).username
 
             if old_un != value or not edit:
                 if UserModel().get_by_username(value, cache=False):
                     raise formencode.Invalid(_('This username already exists') ,
                                              value, state)
 
     return _ValidUsername
 
 class ValidPassword(formencode.validators.FancyValidator):
 
     def to_python(self, value, state):
         if value:
             return get_crypt_password(value)
 
 class ValidAuth(formencode.validators.FancyValidator):
     messages = {
             'invalid_password':_('invalid password'),
             'invalid_login':_('invalid user name'),
             'disabled_account':_('Your acccount is disabled')
 
             }
     #error mapping
     e_dict = {'username':messages['invalid_login'],
               'password':messages['invalid_password']}
     e_dict_disable = {'username':messages['disabled_account']}
 
     def validate_python(self, value, state):
         password = value['password']
         username = value['username']
         user = UserModel().get_by_username(username)
         if user is None:
             raise formencode.Invalid(self.message('invalid_password',
                                      state=State_obj), value, state,
                                      error_dict=self.e_dict)
         if user:
             if user.active:
                 if user.username == username and check_password(password,
                                                                 user.password):
                     return value
                 else:
                     log.warning('user %s not authenticated', username)
                     raise formencode.Invalid(self.message('invalid_password',
                                              state=State_obj), value, state,
                                              error_dict=self.e_dict)
             else:
                 log.warning('user %s is disabled', username)
                 raise formencode.Invalid(self.message('disabled_account',
                                          state=State_obj),
                                          value, state,
                                          error_dict=self.e_dict_disable)
 
 class ValidRepoUser(formencode.validators.FancyValidator):
 
     def to_python(self, value, state):
         sa = meta.Session()
         try:
             self.user_db = sa.query(User)\
                 .filter(User.active == True)\
                 .filter(User.username == value).one()
         except Exception:
             raise formencode.Invalid(_('This username is not valid'),
                                      value, state)
         finally:
             meta.Session.remove()
 
         return self.user_db.user_id
 
 def ValidRepoName(edit, old_data):
     class _ValidRepoName(formencode.validators.FancyValidator):
 
         def to_python(self, value, state):
             slug = h.repo_name_slug(value)
             if slug in ['_admin']:
                 raise formencode.Invalid(_('This repository name is disallowed'),
                                          value, state)
             if old_data.get('repo_name') != value or not edit:
                 if RepoModel().get(slug, cache=False):
                     raise formencode.Invalid(_('This repository already exists') ,
                                              value, state)
             return slug
 
 
     return _ValidRepoName
 
 class ValidPerms(formencode.validators.FancyValidator):
     messages = {'perm_new_user_name':_('This username is not valid')}
 
     def to_python(self, value, state):
         perms_update = []
         perms_new = []
         #build a list of permission to update and new permission to create
         for k, v in value.items():
             if k.startswith('perm_'):
                 if  k.startswith('perm_new_user'):
                     new_perm = value.get('perm_new_user', False)
                     new_user = value.get('perm_new_user_name', False)
                     if new_user and new_perm:
                         if (new_user, new_perm) not in perms_new:
                             perms_new.append((new_user, new_perm))
                 else:
                     usr = k[5:]
                     if usr == 'default':
                         if value['private']:
                             #set none for default when updating to private repo
                             v = 'repository.none'
                     perms_update.append((usr, v))
         value['perms_updates'] = perms_update
         value['perms_new'] = perms_new
         sa = meta.Session
         for k, v in perms_new:
             try:
                 self.user_db = sa.query(User)\
                     .filter(User.active == True)\
                     .filter(User.username == k).one()
             except Exception:
                 msg = self.message('perm_new_user_name',
                                      state=State_obj)
                 raise formencode.Invalid(msg, value, state, error_dict={'perm_new_user_name':msg})
         return value
 
 class ValidSettings(formencode.validators.FancyValidator):
 
     def to_python(self, value, state):
         #settings  form can't edit user
         if value.has_key('user'):
             del['value']['user']
 
         return value
 
 class ValidPath(formencode.validators.FancyValidator):
     def to_python(self, value, state):
-        isdir = os.path.isdir(value.replace('*', ''))
-        if (value.endswith('/*') or value.endswith('/**')) and isdir:
-            return value
-        elif not isdir:
+
+        if not os.path.isdir(value):
             msg = _('This is not a valid path')
-        else:
-            msg = _('You need to specify * or ** at the end of path (ie. /tmp/*)')
-
         raise formencode.Invalid(msg, value, state,
                                      error_dict={'paths_root_path':msg})
+        return value
 
 def UniqSystemEmail(old_data):
     class _UniqSystemEmail(formencode.validators.FancyValidator):
         def to_python(self, value, state):
             if old_data.get('email') != value:
                 sa = meta.Session()
                 try:
                     user = sa.query(User).filter(User.email == value).scalar()
                     if user:
                         raise formencode.Invalid(_("That e-mail address is already taken") ,
                                                  value, state)
                 finally:
                     meta.Session.remove()
 
             return value
 
     return _UniqSystemEmail
 
 class ValidSystemEmail(formencode.validators.FancyValidator):
     def to_python(self, value, state):
         sa = meta.Session
         try:
             user = sa.query(User).filter(User.email == value).scalar()
             if  user is None:
                 raise formencode.Invalid(_("That e-mail address doesn't exist.") ,
                                          value, state)
         finally:
             meta.Session.remove()
 
         return value
 
 #===============================================================================
 # FORMS        
 #===============================================================================
 class LoginForm(formencode.Schema):
     allow_extra_fields = True
     filter_extra_fields = True
     username = UnicodeString(
                              strip=True,
                              min=1,
                              not_empty=True,
                              messages={
                                        'empty':_('Please enter a login'),
                                        'tooShort':_('Enter a value %(min)i characters long or more')}
                             )
 
     password = UnicodeString(
                             strip=True,
                             min=6,
                             not_empty=True,
                             messages={
                                       'empty':_('Please enter a password'),
                                       'tooShort':_('Enter %(min)i characters or more')}
                                 )
 
 
     #chained validators have access to all data
     chained_validators = [ValidAuth]
 
 def UserForm(edit=False, old_data={}):
     class _UserForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         username = All(UnicodeString(strip=True, min=1, not_empty=True), ValidUsername(edit, old_data))
         if edit:
             new_password = All(UnicodeString(strip=True, min=6, not_empty=False), ValidPassword)
             admin = StringBoolean(if_missing=False)
         else:
             password = All(UnicodeString(strip=True, min=6, not_empty=True), ValidPassword)
         active = StringBoolean(if_missing=False)
         name = UnicodeString(strip=True, min=1, not_empty=True)
         lastname = UnicodeString(strip=True, min=1, not_empty=True)
         email = All(Email(not_empty=True), UniqSystemEmail(old_data))
 
     return _UserForm
 
 RegisterForm = UserForm
 
 def PasswordResetForm():
     class _PasswordResetForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         email = All(ValidSystemEmail(), Email(not_empty=True))
     return _PasswordResetForm
 
 def RepoForm(edit=False, old_data={}):
     class _RepoForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         repo_name = All(UnicodeString(strip=True, min=1, not_empty=True), ValidRepoName(edit, old_data))
         description = UnicodeString(strip=True, min=1, not_empty=True)
         private = StringBoolean(if_missing=False)
 
         if edit:
             user = All(Int(not_empty=True), ValidRepoUser)
 
         chained_validators = [ValidPerms]
     return _RepoForm
 
 def RepoForkForm(edit=False, old_data={}):
     class _RepoForkForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         fork_name = All(UnicodeString(strip=True, min=1, not_empty=True), ValidRepoName(edit, old_data))
         description = UnicodeString(strip=True, min=1, not_empty=True)
         private = StringBoolean(if_missing=False)
 
     return _RepoForkForm
 
 def RepoSettingsForm(edit=False, old_data={}):
     class _RepoForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         repo_name = All(UnicodeString(strip=True, min=1, not_empty=True), ValidRepoName(edit, old_data))
         description = UnicodeString(strip=True, min=1, not_empty=True)
         private = StringBoolean(if_missing=False)
 
         chained_validators = [ValidPerms, ValidSettings]
     return _RepoForm
 
 
 def ApplicationSettingsForm():
     class _ApplicationSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         rhodecode_title = UnicodeString(strip=True, min=1, not_empty=True)
         rhodecode_realm = UnicodeString(strip=True, min=1, not_empty=True)
 
     return _ApplicationSettingsForm
 
 def ApplicationUiSettingsForm():
     class _ApplicationUiSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         web_push_ssl = OneOf(['true', 'false'], if_missing='false')
         paths_root_path = All(ValidPath(), UnicodeString(strip=True, min=1, not_empty=True))
         hooks_changegroup_update = OneOf(['True', 'False'], if_missing=False)
         hooks_changegroup_repo_size = OneOf(['True', 'False'], if_missing=False)
 
     return _ApplicationUiSettingsForm
 
 def DefaultPermissionsForm(perms_choices, register_choices, create_choices):
     class _DefaultPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         overwrite_default = OneOf(['true', 'false'], if_missing='false')
         default_perm = OneOf(perms_choices)
         default_register = OneOf(register_choices)
         default_create = OneOf(create_choices)
 
     return _DefaultPermissionsForm

rhodecode/model/hg.py

@@ -1,185 +1,181 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # Model for RhodeCode
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 # 
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on April 9, 2010
 Model for RhodeCode
 @author: marcink
 """
 from beaker.cache import cache_region
 from mercurial import ui
-from mercurial.hgweb.hgwebdir_mod import findrepos
 from rhodecode.lib import helpers as h
 from rhodecode.lib.utils import invalidate_cache
 from rhodecode.lib.auth import HasRepoPermissionAny
 from rhodecode.model import meta
 from rhodecode.model.db import Repository, User
 from sqlalchemy.orm import joinedload
 from vcs.exceptions import RepositoryError, VCSError
 import logging
-import os
 import sys
 log = logging.getLogger(__name__)
 
 try:
     from vcs.backends.hg import MercurialRepository
+    from vcs.backends.git import GitRepository
 except ImportError:
     sys.stderr.write('You have to import vcs module')
     raise Exception('Unable to import vcs')
 
 def _get_repos_cached_initial(app_globals, initial):
     """return cached dict with repos
     """
     g = app_globals
-    return HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui, initial)
+    return HgModel().repo_scan(g.paths[0][1], g.baseui, initial)
 
 @cache_region('long_term', 'cached_repo_list')
 def _get_repos_cached():
     """return cached dict with repos
     """
     log.info('getting all repositories list')
     from pylons import app_globals as g
-    return HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui)
+    return HgModel().repo_scan(g.paths[0][1], g.baseui)
 
 @cache_region('super_short_term', 'cached_repos_switcher_list')
 def _get_repos_switcher_cached(cached_repo_list):
     repos_lst = []
     for repo in [x for x in cached_repo_list.values()]:
         if HasRepoPermissionAny('repository.write', 'repository.read',
                     'repository.admin')(repo.name, 'main page check'):
             repos_lst.append((repo.name, repo.dbrepo.private,))
 
     return sorted(repos_lst, key=lambda k:k[0].lower())
 
 @cache_region('long_term', 'full_changelog')
 def _full_changelog_cached(repo_name):
     log.info('getting full changelog for %s', repo_name)
     return list(reversed(list(HgModel().get_repo(repo_name))))
 
 class HgModel(object):
-    """Mercurial Model
+    """
+    Mercurial Model
     """
 
-    def __init__(self):
-        pass
+    def __init__(self, sa=None):
+        if not sa:
+            self.sa = meta.Session()
+        else:
+            self.sa = sa
 
-    @staticmethod
-    def repo_scan(repos_prefix, repos_path, baseui, initial=False):
+    def repo_scan(self, repos_path, baseui, initial=False):
         """
         Listing of repositories in given path. This path should not be a 
         repository itself. Return a dictionary of repository objects
-        :param repos_path: path to directory it could take syntax with 
-        * or ** for deep recursive displaying repositories
-        """
-        sa = meta.Session()
-        def check_repo_dir(path):
-            """Checks the repository
-            :param path:
+        
+        :param repos_path: path to directory containing repositories
+        :param baseui
+        :param initial: initial scann
             """
-            repos_path = path.split('/')
-            if repos_path[-1] in ['*', '**']:
-                repos_path = repos_path[:-1]
-            if repos_path[0] != '/':
-                repos_path[0] = '/'
-            if not os.path.isdir(os.path.join(*repos_path)):
-                raise RepositoryError('Not a valid repository in %s' % path)
-        if not repos_path.endswith('*'):
-            raise VCSError('You need to specify * or ** at the end of path '
-                            'for recursive scanning')
+        log.info('scanning for repositories in %s', repos_path)
 
-        check_repo_dir(repos_path)
-        log.info('scanning for repositories in %s', repos_path)
-        repos = findrepos([(repos_prefix, repos_path)])
         if not isinstance(baseui, ui.ui):
             baseui = ui.ui()
 
+        from rhodecode.lib.utils import get_repos
+        repos = get_repos(repos_path)
+
+
         repos_list = {}
         for name, path in repos:
             try:
                 #name = name.split('/')[-1]
                 if repos_list.has_key(name):
                     raise RepositoryError('Duplicate repository name %s found in'
                                     ' %s' % (name, path))
                 else:
+                    if path[0] == 'hg':
+                        repos_list[name] = MercurialRepository(path[1], baseui=baseui)
+                        repos_list[name].name = name
 
-                    repos_list[name] = MercurialRepository(path, baseui=baseui)
+                    if path[0] == 'git':
+                        repos_list[name] = GitRepository(path[1])
                     repos_list[name].name = name
 
                     dbrepo = None
                     if not initial:
                         #for initial scann on application first run we don't
                         #have db repos yet.
-                        dbrepo = sa.query(Repository)\
+                        dbrepo = self.sa.query(Repository)\
                             .options(joinedload(Repository.fork))\
                             .filter(Repository.repo_name == name)\
                             .scalar()
 
                     if dbrepo:
                         log.info('Adding db instance to cached list')
                         repos_list[name].dbrepo = dbrepo
                         repos_list[name].description = dbrepo.description
                         if dbrepo.user:
                             repos_list[name].contact = dbrepo.user.full_contact
                         else:
-                            repos_list[name].contact = sa.query(User)\
+                            repos_list[name].contact = self.sa.query(User)\
                             .filter(User.admin == True).first().full_contact
             except OSError:
                 continue
-        meta.Session.remove()
+
         return repos_list
 
     def get_repos(self):
         for name, repo in _get_repos_cached().items():
-            if repo._get_hidden():
+
+            if isinstance(repo, MercurialRepository) and repo._get_hidden():
                 #skip hidden web repository
                 continue
 
             last_change = repo.last_change
             tip = h.get_changeset_safe(repo, 'tip')
 
             tmp_d = {}
             tmp_d['name'] = repo.name
             tmp_d['name_sort'] = tmp_d['name'].lower()
             tmp_d['description'] = repo.description
             tmp_d['description_sort'] = tmp_d['description']
             tmp_d['last_change'] = last_change
             tmp_d['last_change_sort'] = last_change[1] - last_change[0]
             tmp_d['tip'] = tip.short_id
             tmp_d['tip_sort'] = tip.revision
             tmp_d['rev'] = tip.revision
             tmp_d['contact'] = repo.contact
             tmp_d['contact_sort'] = tmp_d['contact']
             tmp_d['repo_archives'] = list(repo._get_archives())
             tmp_d['last_msg'] = tip.message
             tmp_d['repo'] = repo
             yield tmp_d
 
     def get_repo(self, repo_name):
         try:
             repo = _get_repos_cached()[repo_name]
             return repo
         except KeyError:
             #i we're here and we got key errors let's try to invalidate the
             #cahce and try again
             invalidate_cache('cached_repo_list')
             repo = _get_repos_cached()[repo_name]
             return repo
 
 
 

rhodecode/templates/shortlog/shortlog_data.html

@@ -1,63 +1,63 b''
 ## -*- coding: utf-8 -*-
 % if c.repo_changesets:
 <table>
 	<tr>
 		<th class="left">${_('date')}</th>
 		<th class="left">${_('author')}</th>
 		<th class="left">${_('revision')}</th>
 		<th class="left">${_('commit message')}</th>
 		<th class="left">${_('branch')}</th>
 		<th class="left">${_('tags')}</th>
 		<th class="left">${_('links')}</th>
 		
 	</tr>
 %for cnt,cs in enumerate(c.repo_changesets):
 	<tr class="parity${cnt%2}">
-		<td>${h.age(cs._ctx.date())} - ${h.rfc822date_notz(cs._ctx.date())} </td>
+		<td>${h.age(cs.date)} - ${h.rfc822date_notz(cs.date)} </td>
 		<td title="${cs.author}">${h.person(cs.author)}</td>
 		<td>r${cs.revision}:${cs.short_id}</td>
 		<td>
 			${h.link_to(h.truncate(cs.message,60),
 			h.url('changeset_home',repo_name=c.repo_name,revision=cs.short_id),
 			title=cs.message)}
 		</td>
 		<td>
 			<span class="logtags">
 				<span class="branchtag">${cs.branch}</span>
 			</span>
 		</td>
 		<td>
 			<span class="logtags">
 				%for tag in cs.tags:
 					<span class="tagtag">${tag}</span>
 				%endfor
 			</span>
 		</td>
 		<td class="nowrap">
 		${h.link_to(_('changeset'),h.url('changeset_home',repo_name=c.repo_name,revision=cs.short_id))}
 		|
 		${h.link_to(_('files'),h.url('files_home',repo_name=c.repo_name,revision=cs.short_id))}
 		</td>
 	</tr>
 %endfor
 
 </table>
 
 <script type="text/javascript">
   var data_div = 'shortlog_data';
   YAHOO.util.Event.onDOMReady(function(){
     YAHOO.util.Event.addListener(YAHOO.util.Dom.getElementsByClassName('pager_link'),"click",function(){
             YAHOO.util.Dom.setStyle('shortlog_data','opacity','0.3');});});
 </script>
 
 <div class="pagination-wh pagination-left">
 ${c.repo_changesets.pager('$link_previous ~2~ $link_next',     
 onclick="""YAHOO.util.Connect.asyncRequest('GET','$partial_url',{
 success:function(o){YAHOO.util.Dom.get(data_div).innerHTML=o.responseText;
 YAHOO.util.Event.addListener(YAHOO.util.Dom.getElementsByClassName('pager_link'),"click",function(){
         YAHOO.util.Dom.setStyle(data_div,'opacity','0.3');});       
 YAHOO.util.Dom.setStyle(data_div,'opacity','1');}},null); return false;""")}
 </div>
 %else:
     ${_('There are no changes yet')}
 %endif