complete rewrite of paster commands,

used fancy celery-pylons baseCommand to implement better support for paster commands
fixed make-index, and paster celeryd. Moved config's add_cache to utils

rhodecode/lib/celerylib/tasks.py

@@ -1,389 +1,365 b''
 from celery.decorators import task
 
 import os
 import traceback
-import beaker
 from time import mktime
 from operator import itemgetter
 
 from pylons import config
 from pylons.i18n.translation import _
 
 from rhodecode.lib.celerylib import run_task, locked_task, str2bool
 from rhodecode.lib.helpers import person
 from rhodecode.lib.smtp_mailer import SmtpMailer
-from rhodecode.lib.utils import OrderedDict
+from rhodecode.lib.utils import OrderedDict, add_cache
 from rhodecode.model import init_model
 from rhodecode.model import meta
 from rhodecode.model.db import RhodeCodeUi
 
 from vcs.backends import get_repo
 
 from sqlalchemy import engine_from_config
 
-#set cache regions for beaker so celery can utilise it
-def add_cache(settings):
-    cache_settings = {'regions':None}
-    for key in settings.keys():
-        for prefix in ['beaker.cache.', 'cache.']:
-            if key.startswith(prefix):
-                name = key.split(prefix)[1].strip()
-                cache_settings[name] = settings[key].strip()
-    if cache_settings['regions']:
-        for region in cache_settings['regions'].split(','):
-            region = region.strip()
-            region_settings = {}
-            for key, value in cache_settings.items():
-                if key.startswith(region):
-                    region_settings[key.split('.')[1]] = value
-            region_settings['expire'] = int(region_settings.get('expire',
-                                                                60))
-            region_settings.setdefault('lock_dir',
-                                       cache_settings.get('lock_dir'))
-            if 'type' not in region_settings:
-                region_settings['type'] = cache_settings.get('type',
-                                                             'memory')
-            beaker.cache.cache_regions[region] = region_settings
 add_cache(config)
 
 try:
     import json
 except ImportError:
     #python 2.5 compatibility
     import simplejson as json
 
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
 
 CELERY_ON = str2bool(config['app_conf'].get('use_celery'))
 
 def get_session():
     if CELERY_ON:
         engine = engine_from_config(config, 'sqlalchemy.db1.')
         init_model(engine)
     sa = meta.Session()
     return sa
 
 def get_repos_path():
     sa = get_session()
     q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
     return q.ui_value
 
 @task
 @locked_task
 def whoosh_index(repo_location, full_index):
     log = whoosh_index.get_logger()
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     index_location = config['index_dir']
     WhooshIndexingDaemon(index_location=index_location,
                          repo_location=repo_location, sa=get_session())\
                          .run(full_index=full_index)
 
 @task
 @locked_task
 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
     from rhodecode.model.db import Statistics, Repository
     log = get_commits_stats.get_logger()
 
     #for js data compatibilty
     author_key_cleaner = lambda k: person(k).replace('"', "")
 
     commits_by_day_author_aggregate = {}
     commits_by_day_aggregate = {}
     repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
 
     skip_date_limit = True
     parse_limit = 250 #limit for single task changeset parsing optimal for
     last_rev = 0
     last_cs = None
     timegetter = itemgetter('time')
 
     sa = get_session()
 
     dbrepo = sa.query(Repository)\
         .filter(Repository.repo_name == repo_name).scalar()
     cur_stats = sa.query(Statistics)\
         .filter(Statistics.repository == dbrepo).scalar()
     if cur_stats:
         last_rev = cur_stats.stat_on_revision
     if not repo.revisions:
         return True
 
     if last_rev == repo.revisions[-1] and len(repo.revisions) > 1:
         #pass silently without any work if we're not on first revision or 
         #current state of parsing revision(from db marker) is the last revision
         return True
 
     if cur_stats:
         commits_by_day_aggregate = OrderedDict(
                                        json.loads(
                                         cur_stats.commit_activity_combined))
         commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity)
 
     log.debug('starting parsing %s', parse_limit)
     lmktime = mktime
 
     for cnt, rev in enumerate(repo.revisions[last_rev:]):
         last_cs = cs = repo.get_changeset(rev)
         k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1],
                           cs.date.timetuple()[2])
         timetupple = [int(x) for x in k.split('-')]
         timetupple.extend([0 for _ in xrange(6)])
         k = lmktime(timetupple)
         if commits_by_day_author_aggregate.has_key(author_key_cleaner(cs.author)):
             try:
                 l = [timegetter(x) for x in commits_by_day_author_aggregate\
                         [author_key_cleaner(cs.author)]['data']]
                 time_pos = l.index(k)
             except ValueError:
                 time_pos = False
 
             if time_pos >= 0 and time_pos is not False:
 
                 datadict = commits_by_day_author_aggregate\
                     [author_key_cleaner(cs.author)]['data'][time_pos]
 
                 datadict["commits"] += 1
                 datadict["added"] += len(cs.added)
                 datadict["changed"] += len(cs.changed)
                 datadict["removed"] += len(cs.removed)
 
             else:
                 if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
 
                     datadict = {"time":k,
                                 "commits":1,
                                 "added":len(cs.added),
                                 "changed":len(cs.changed),
                                 "removed":len(cs.removed),
                                }
                     commits_by_day_author_aggregate\
                         [author_key_cleaner(cs.author)]['data'].append(datadict)
 
         else:
             if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
                 commits_by_day_author_aggregate[author_key_cleaner(cs.author)] = {
                                     "label":author_key_cleaner(cs.author),
                                     "data":[{"time":k,
                                              "commits":1,
                                              "added":len(cs.added),
                                              "changed":len(cs.changed),
                                              "removed":len(cs.removed),
                                              }],
                                     "schema":["commits"],
                                     }
 
         #gather all data by day
         if commits_by_day_aggregate.has_key(k):
             commits_by_day_aggregate[k] += 1
         else:
             commits_by_day_aggregate[k] = 1
 
         if cnt >= parse_limit:
             #don't fetch to much data since we can freeze application
             break
     overview_data = []
     for k, v in commits_by_day_aggregate.items():
         overview_data.append([k, v])
     overview_data = sorted(overview_data, key=itemgetter(0))
     if not commits_by_day_author_aggregate:
         commits_by_day_author_aggregate[author_key_cleaner(repo.contact)] = {
             "label":author_key_cleaner(repo.contact),
             "data":[0, 1],
             "schema":["commits"],
         }
 
     stats = cur_stats if cur_stats else Statistics()
     stats.commit_activity = json.dumps(commits_by_day_author_aggregate)
     stats.commit_activity_combined = json.dumps(overview_data)
 
     log.debug('last revison %s', last_rev)
     leftovers = len(repo.revisions[last_rev:])
     log.debug('revisions to parse %s', leftovers)
 
     if last_rev == 0 or leftovers < parse_limit:
         stats.languages = json.dumps(__get_codes_stats(repo_name))
 
     stats.repository = dbrepo
     stats.stat_on_revision = last_cs.revision
 
     try:
         sa.add(stats)
         sa.commit()
     except:
         log.error(traceback.format_exc())
         sa.rollback()
         return False
     if len(repo.revisions) > 1:
         run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
 
     return True
 
 @task
 def reset_user_password(user_email):
     log = reset_user_password.get_logger()
     from rhodecode.lib import auth
     from rhodecode.model.db import User
 
     try:
         try:
             sa = get_session()
             user = sa.query(User).filter(User.email == user_email).scalar()
             new_passwd = auth.PasswordGenerator().gen_password(8,
                              auth.PasswordGenerator.ALPHABETS_BIG_SMALL)
             if user:
                 user.password = auth.get_crypt_password(new_passwd)
                 sa.add(user)
                 sa.commit()
                 log.info('change password for %s', user_email)
             if new_passwd is None:
                 raise Exception('unable to generate new password')
 
         except:
             log.error(traceback.format_exc())
             sa.rollback()
 
         run_task(send_email, user_email,
                  "Your new rhodecode password",
                  'Your new rhodecode password:%s' % (new_passwd))
         log.info('send new password mail to %s', user_email)
 
 
     except:
         log.error('Failed to update user password')
         log.error(traceback.format_exc())
 
     return True
 
 @task
 def send_email(recipients, subject, body):
     """
     Sends an email with defined parameters from the .ini files.
     
     
     :param recipients: list of recipients, it this is empty the defined email
         address from field 'email_to' is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     """
     log = send_email.get_logger()
     email_config = config
 
     if not recipients:
         recipients = [email_config.get('email_to')]
 
     mail_from = email_config.get('app_email_from')
     user = email_config.get('smtp_username')
     passwd = email_config.get('smtp_password')
     mail_server = email_config.get('smtp_server')
     mail_port = email_config.get('smtp_port')
     tls = str2bool(email_config.get('smtp_use_tls'))
     ssl = str2bool(email_config.get('smtp_use_ssl'))
 
     try:
         m = SmtpMailer(mail_from, user, passwd, mail_server,
                        mail_port, ssl, tls)
         m.send(recipients, subject, body)
     except:
         log.error('Mail sending failed')
         log.error(traceback.format_exc())
         return False
     return True
 
 @task
 def create_repo_fork(form_data, cur_user):
     from rhodecode.model.repo import RepoModel
     from vcs import get_backend
     log = create_repo_fork.get_logger()
     repo_model = RepoModel(get_session())
     repo_model.create(form_data, cur_user, just_db=True, fork=True)
     repo_name = form_data['repo_name']
     repos_path = get_repos_path()
     repo_path = os.path.join(repos_path, repo_name)
     repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
     alias = form_data['repo_type']
 
     log.info('creating repo fork %s as %s', repo_name, repo_path)
     backend = get_backend(alias)
     backend(str(repo_fork_path), create=True, src_url=str(repo_path))
 
 def __get_codes_stats(repo_name):
     LANGUAGES_EXTENSIONS_MAP = {'scm': 'Scheme', 'asmx': 'VbNetAspx', 'Rout':
     'RConsole', 'rest': 'Rst', 'abap': 'ABAP', 'go': 'Go', 'phtml': 'HtmlPhp',
     'ns2': 'Newspeak', 'xml': 'EvoqueXml', 'sh-session': 'BashSession', 'ads':
     'Ada', 'clj': 'Clojure', 'll': 'Llvm', 'ebuild': 'Bash', 'adb': 'Ada',
     'ada': 'Ada', 'c++-objdump': 'CppObjdump', 'aspx':
     'VbNetAspx', 'ksh': 'Bash', 'coffee': 'CoffeeScript', 'vert': 'GLShader',
     'Makefile.*': 'Makefile', 'di': 'D', 'dpatch': 'DarcsPatch', 'rake':
     'Ruby', 'moo': 'MOOCode', 'erl-sh': 'ErlangShell', 'geo': 'GLShader',
     'pov': 'Povray', 'bas': 'VbNet', 'bat': 'Batch', 'd': 'D', 'lisp':
     'CommonLisp', 'h': 'C', 'rbx': 'Ruby', 'tcl': 'Tcl', 'c++': 'Cpp', 'md':
     'MiniD', '.vimrc': 'Vim', 'xsd': 'Xml', 'ml': 'Ocaml', 'el': 'CommonLisp',
     'befunge': 'Befunge', 'xsl': 'Xslt', 'pyx': 'Cython', 'cfm':
     'ColdfusionHtml', 'evoque': 'Evoque', 'cfg': 'Ini', 'htm': 'Html',
     'Makefile': 'Makefile', 'cfc': 'ColdfusionHtml', 'tex': 'Tex', 'cs':
     'CSharp', 'mxml': 'Mxml', 'patch': 'Diff', 'apache.conf': 'ApacheConf',
     'scala': 'Scala', 'applescript': 'AppleScript', 'GNUmakefile': 'Makefile',
     'c-objdump': 'CObjdump', 'lua': 'Lua', 'apache2.conf': 'ApacheConf', 'rb':
     'Ruby', 'gemspec': 'Ruby', 'rl': 'RagelObjectiveC', 'vala': 'Vala', 'tmpl':
     'Cheetah', 'bf': 'Brainfuck', 'plt': 'Gnuplot', 'G': 'AntlrRuby', 'xslt':
     'Xslt', 'flxh': 'Felix', 'asax': 'VbNetAspx', 'Rakefile': 'Ruby', 'S': 'S',
     'wsdl': 'Xml', 'js': 'Javascript', 'autodelegate': 'Myghty', 'properties':
     'Ini', 'bash': 'Bash', 'c': 'C', 'g': 'AntlrRuby', 'r3': 'Rebol', 's':
     'Gas', 'ashx': 'VbNetAspx', 'cxx': 'Cpp', 'boo': 'Boo', 'prolog': 'Prolog',
     'sqlite3-console': 'SqliteConsole', 'cl': 'CommonLisp', 'cc': 'Cpp', 'pot':
     'Gettext', 'vim': 'Vim', 'pxi': 'Cython', 'yaml': 'Yaml', 'SConstruct':
     'Python', 'diff': 'Diff', 'txt': 'Text', 'cw': 'Redcode', 'pxd': 'Cython',
     'plot': 'Gnuplot', 'java': 'Java', 'hrl': 'Erlang', 'py': 'Python',
     'makefile': 'Makefile', 'squid.conf': 'SquidConf', 'asm': 'Nasm', 'toc':
     'Tex', 'kid': 'Genshi', 'rhtml': 'Rhtml', 'po': 'Gettext', 'pl': 'Prolog',
     'pm': 'Perl', 'hx': 'Haxe', 'ascx': 'VbNetAspx', 'ooc': 'Ooc', 'asy':
     'Asymptote', 'hs': 'Haskell', 'SConscript': 'Python', 'pytb':
     'PythonTraceback', 'myt': 'Myghty', 'hh': 'Cpp', 'R': 'S', 'aux': 'Tex',
     'rst': 'Rst', 'cpp-objdump': 'CppObjdump', 'lgt': 'Logtalk', 'rss': 'Xml',
     'flx': 'Felix', 'b': 'Brainfuck', 'f': 'Fortran', 'rbw': 'Ruby',
     '.htaccess': 'ApacheConf', 'cxx-objdump': 'CppObjdump', 'j': 'ObjectiveJ',
     'mll': 'Ocaml', 'yml': 'Yaml', 'mu': 'MuPAD', 'r': 'Rebol', 'ASM': 'Nasm',
     'erl': 'Erlang', 'mly': 'Ocaml', 'mo': 'Modelica', 'def': 'Modula2', 'ini':
     'Ini', 'control': 'DebianControl', 'vb': 'VbNet', 'vapi': 'Vala', 'pro':
     'Prolog', 'spt': 'Cheetah', 'mli': 'Ocaml', 'as': 'ActionScript3', 'cmd':
     'Batch', 'cpp': 'Cpp', 'io': 'Io', 'tac': 'Python', 'haml': 'Haml', 'rkt':
     'Racket', 'st':'Smalltalk', 'inc': 'Povray', 'pas': 'Delphi', 'cmake':
     'CMake', 'csh':'Tcsh', 'hpp': 'Cpp', 'feature': 'Gherkin', 'html': 'Html',
     'php':'Php', 'php3':'Php', 'php4':'Php', 'php5':'Php', 'xhtml': 'Html',
     'hxx': 'Cpp', 'eclass': 'Bash', 'css': 'Css',
     'frag': 'GLShader', 'd-objdump': 'DObjdump', 'weechatlog': 'IrcLogs',
     'tcsh': 'Tcsh', 'objdump': 'Objdump', 'pyw': 'Python', 'h++': 'Cpp',
     'py3tb': 'Python3Traceback', 'jsp': 'Jsp', 'sql': 'Sql', 'mak': 'Makefile',
     'php': 'Php', 'mao': 'Mako', 'man': 'Groff', 'dylan': 'Dylan', 'sass':
     'Sass', 'cfml': 'ColdfusionHtml', 'darcspatch': 'DarcsPatch', 'tpl':
     'Smarty', 'm': 'ObjectiveC', 'f90': 'Fortran', 'mod': 'Modula2', 'sh':
     'Bash', 'lhs': 'LiterateHaskell', 'sources.list': 'SourcesList', 'axd':
     'VbNetAspx', 'sc': 'Python'}
 
     repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
     tip = repo.get_changeset()
     code_stats = {}
 
     def aggregate(cs):
         for f in cs[2]:
             ext = f.extension
             key = LANGUAGES_EXTENSIONS_MAP.get(ext, ext)
             key = key or ext
             if ext in LANGUAGES_EXTENSIONS_MAP.keys():
                 if code_stats.has_key(key):
                     code_stats[key] += 1
                 else:
                     code_stats[key] = 1
 
     map(aggregate, tip.walk('/'))
 
     return code_stats or {}
 
 
 
 

rhodecode/lib/celerypylons/commands.py

@@ -1,143 +1,90 b''
-import os
-from paste.script.command import Command, BadCommand
-import paste.deploy
-from pylons import config
+from rhodecode.lib.utils import BasePasterCommand, Command
 
 
 __all__ = ['CeleryDaemonCommand', 'CeleryBeatCommand',
            'CAMQPAdminCommand', 'CeleryEventCommand']
 
 
-class CeleryCommand(Command):
-    """
-    Abstract Base Class for celery commands.
-
-    The celery commands are somewhat aggressive about loading
-    celery.conf, and since our module sets the `CELERY_LOADER`
-    environment variable to our loader, we have to bootstrap a bit and
-    make sure we've had a chance to load the pylons config off of the
-    command line, otherwise everything fails.
-    """
-    min_args = 1
-    min_args_error = "Please provide a paster config file as an argument."
-    takes_config_file = 1
-    requires_config_file = True
-
-    def run(self, args):
-        """
-        Overrides Command.run
-        
-        Checks for a config file argument and loads it.
-        """
-        if len(args) < self.min_args:
-            raise BadCommand(
-                self.min_args_error % {'min_args': self.min_args,
-                                       'actual_args': len(args)})
-        # Decrement because we're going to lob off the first argument.
-        # @@ This is hacky
-        self.min_args -= 1
-        self.bootstrap_config(args[0])
-        self.update_parser()
-        return super(CeleryCommand, self).run(args[1:])
-
-    def update_parser(self):
-        """
-        Abstract method.  Allows for the class's parser to be updated
-        before the superclass's `run` method is called.  Necessary to
-        allow options/arguments to be passed through to the underlying
-        celery command.
-        """
-        raise NotImplementedError("Abstract Method.")
-
-    def bootstrap_config(self, conf):
-        """
-        Loads the pylons configuration.
-        """
-        path_to_ini_file = os.path.realpath(conf)
-        conf = paste.deploy.appconfig('config:' + path_to_ini_file)
-        config.init_app(conf.global_conf, conf.local_conf)
-
-
-class CeleryDaemonCommand(CeleryCommand):
+class CeleryDaemonCommand(BasePasterCommand):
     """Start the celery worker
 
     Starts the celery worker that uses a paste.deploy configuration
     file.
     """
     usage = 'CONFIG_FILE [celeryd options...]'
     summary = __doc__.splitlines()[0]
     description = "".join(__doc__.splitlines()[2:])
 
     parser = Command.standard_parser(quiet=True)
 
     def update_parser(self):
         from celery.bin import celeryd
         for x in celeryd.WorkerCommand().get_options():
             self.parser.add_option(x)
 
     def command(self):
         from celery.bin import celeryd
         return celeryd.WorkerCommand().run(**vars(self.options))
 
 
-class CeleryBeatCommand(CeleryCommand):
+class CeleryBeatCommand(BasePasterCommand):
     """Start the celery beat server
 
     Starts the celery beat server using a paste.deploy configuration
     file.
     """
     usage = 'CONFIG_FILE [celerybeat options...]'
     summary = __doc__.splitlines()[0]
     description = "".join(__doc__.splitlines()[2:])
 
     parser = Command.standard_parser(quiet=True)
 
     def update_parser(self):
         from celery.bin import celerybeat
         for x in celerybeat.BeatCommand().get_options():
             self.parser.add_option(x)
 
     def command(self):
         from celery.bin import celerybeat
         return celerybeat.BeatCommand(**vars(self.options))
 
-class CAMQPAdminCommand(CeleryCommand):
+class CAMQPAdminCommand(BasePasterCommand):
     """CAMQP Admin
 
     CAMQP celery admin tool.
     """
     usage = 'CONFIG_FILE [camqadm options...]'
     summary = __doc__.splitlines()[0]
     description = "".join(__doc__.splitlines()[2:])
 
     parser = Command.standard_parser(quiet=True)
 
     def update_parser(self):
         from celery.bin import camqadm
         for x in camqadm.OPTION_LIST:
             self.parser.add_option(x)
 
     def command(self):
         from celery.bin import camqadm
         return camqadm.camqadm(*self.args, **vars(self.options))
 
 
-class CeleryEventCommand(CeleryCommand):
+class CeleryEventCommand(BasePasterCommand):
     """Celery event commandd.
 
     Capture celery events.
     """
     usage = 'CONFIG_FILE [celeryev options...]'
     summary = __doc__.splitlines()[0]
     description = "".join(__doc__.splitlines()[2:])
 
     parser = Command.standard_parser(quiet=True)
 
     def update_parser(self):
         from celery.bin import celeryev
         for x in celeryev.OPTION_LIST:
             self.parser.add_option(x)
 
     def command(self):
         from celery.bin import celeryev
         return celeryev.run_celeryev(**vars(self.options))

rhodecode/lib/indexers/__init__.py

@@ -1,188 +1,193 b''
 import os
 import sys
+import traceback
 from os.path import dirname as dn, join as jn
 
 #to get the rhodecode import
 sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
 
+from rhodecode.model import init_model
+from rhodecode.model.scm import ScmModel
 from rhodecode.config.environment import load_environment
-from rhodecode.model.scm import ScmModel
+from rhodecode.lib.utils import BasePasterCommand, Command, add_cache
+
 from shutil import rmtree
 from webhelpers.html.builder import escape
 from vcs.utils.lazy import LazyProperty
 
+from sqlalchemy import engine_from_config
+
 from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter
 from whoosh.fields import TEXT, ID, STORED, Schema, FieldType
 from whoosh.index import create_in, open_dir
 from whoosh.formats import Characters
 from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter
 
-import traceback
 
 #EXTENSIONS WE WANT TO INDEX CONTENT OFF
 INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
                     'cfg', 'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl',
                     'h', 'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp',
                     'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3',
                     'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql',
                     'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt',
                     'yaws']
 
 #CUSTOM ANALYZER wordsplit + lowercase filter
 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
 
 
 #INDEX SCHEMA DEFINITION
 SCHEMA = Schema(owner=TEXT(),
                 repository=TEXT(stored=True),
                 path=TEXT(stored=True),
                 content=FieldType(format=Characters(ANALYZER),
                              scorable=True, stored=True),
                 modtime=STORED(), extension=TEXT(stored=True))
 
 
 IDX_NAME = 'HG_INDEX'
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = SimpleFragmenter(200)
 
-from paste.script import command
-import ConfigParser
 
-class MakeIndex(command.Command):
+class MakeIndex(BasePasterCommand):
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     summary = "Creates index for full text search given configuration file"
     group_name = "RhodeCode"
     takes_config_file = -1
-    parser = command.Command.standard_parser(verbose=True)
-    parser.add_option('--repo-location',
-                      action='store',
-                      dest='repo_location',
-                      help="Specifies repositories location to index REQUIRED",
-                      )
-    parser.add_option('-f',
-                      action='store_true',
-                      dest='full_index',
-                      help="Specifies that index should be made full i.e"
-                            " destroy old and build from scratch",
-                      default=False)
+    parser = Command.standard_parser(verbose=True)
+
     def command(self):
-        config_name = self.args[0]
-        p = config_name.split('/')
-        root = '.' if len(p) == 1 else '/'.join(p[:-1])
-        config = ConfigParser.ConfigParser({'here':root})
-        config.read(config_name)
 
-        index_location = dict(config.items('app:main'))['index_dir']
+        from pylons import config
+        add_cache(config)
+        engine = engine_from_config(config, 'sqlalchemy.db1.')
+        init_model(engine)
+
+        index_location = config['index_dir']
         repo_location = self.options.repo_location
 
         #======================================================================
         # WHOOSH DAEMON
         #======================================================================
         from rhodecode.lib.pidlock import LockHeld, DaemonLock
         from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
         try:
             l = DaemonLock()
             WhooshIndexingDaemon(index_location=index_location,
                                  repo_location=repo_location)\
                 .run(full_index=self.options.full_index)
             l.release()
         except LockHeld:
             sys.exit(1)
 
+    def update_parser(self):
+        self.parser.add_option('--repo-location',
+                          action='store',
+                          dest='repo_location',
+                          help="Specifies repositories location to index REQUIRED",
+                          )
+        self.parser.add_option('-f',
+                          action='store_true',
+                          dest='full_index',
+                          help="Specifies that index should be made full i.e"
+                                " destroy old and build from scratch",
+                          default=False)
 
 class ResultWrapper(object):
     def __init__(self, search_type, searcher, matcher, highlight_items):
         self.search_type = search_type
         self.searcher = searcher
         self.matcher = matcher
         self.highlight_items = highlight_items
         self.fragment_size = 200 / 2
 
     @LazyProperty
     def doc_ids(self):
         docs_id = []
         while self.matcher.is_active():
             docnum = self.matcher.id()
             chunks = [offsets for offsets in self.get_chunks()]
             docs_id.append([docnum, chunks])
             self.matcher.next()
         return docs_id
 
     def __str__(self):
         return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids))
 
     def __repr__(self):
         return self.__str__()
 
     def __len__(self):
         return len(self.doc_ids)
 
     def __iter__(self):
         """
         Allows Iteration over results,and lazy generate content
 
         *Requires* implementation of ``__getitem__`` method.
         """
         for docid in self.doc_ids:
             yield self.get_full_content(docid)
 
     def __getslice__(self, i, j):
         """
         Slicing of resultWrapper
         """
         slice = []
         for docid in self.doc_ids[i:j]:
             slice.append(self.get_full_content(docid))
         return slice
 
 
     def get_full_content(self, docid):
         res = self.searcher.stored_fields(docid[0])
         f_path = res['path'][res['path'].find(res['repository']) \
                              + len(res['repository']):].lstrip('/')
 
         content_short = self.get_short_content(res, docid[1])
         res.update({'content_short':content_short,
                     'content_short_hl':self.highlight(content_short),
                     'f_path':f_path})
 
         return res
 
     def get_short_content(self, res, chunks):
 
         return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks])
 
     def get_chunks(self):
         """
         Smart function that implements chunking the content
         but not overlap chunks so it doesn't highlight the same
         close occurrences twice.
         @param matcher:
         @param size:
         """
         memory = [(0, 0)]
         for span in self.matcher.spans():
             start = span.startchar or 0
             end = span.endchar or 0
             start_offseted = max(0, start - self.fragment_size)
             end_offseted = end + self.fragment_size
 
             if start_offseted < memory[-1][1]:
                 start_offseted = memory[-1][1]
             memory.append((start_offseted, end_offseted,))
             yield (start_offseted, end_offseted,)
 
     def highlight(self, content, top=5):
         if self.search_type != 'content':
             return ''
         hl = highlight(escape(content),
                  self.highlight_items,
                  analyzer=ANALYZER,
                  fragmenter=FRAGMENTER,
                  formatter=FORMATTER,
                  top=top)
         return hl

rhodecode/lib/utils.py

@@ -1,524 +1,611 b''
 # -*- coding: utf-8 -*-
 """
     package.rhodecode.lib.utils
     ~~~~~~~~~~~~~~
 
     Utilities library for RhodeCode
     
     :created_on: Apr 18, 2010
     :author: marcink
     :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
 import os
 import logging
 import datetime
 import traceback
 import ConfigParser
 
 from UserDict import DictMixin
 
 from mercurial import ui, config, hg
 from mercurial.error import RepoError
 
-from paste.script import command
+import paste
+import beaker
+from paste.script.command import Command, BadCommand
+
 from vcs.backends.base import BaseChangeset
 from vcs.utils.lazy import LazyProperty
 
 from rhodecode.model import meta
 from rhodecode.model.caching_query import FromCache
 from rhodecode.model.db import Repository, User, RhodeCodeUi, UserLog
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.user import UserModel
 
 log = logging.getLogger(__name__)
 
 
 def get_repo_slug(request):
     return request.environ['pylons.routes_dict'].get('repo_name')
 
 def action_logger(user, action, repo, ipaddr='', sa=None):
     """
     Action logger for various actions made by users
     
     :param user: user that made this action, can be a unique username string or
         object containing user_id attribute
     :param action: action to log, should be on of predefined unique actions for
         easy translations
     :param repo: string name of repository or object containing repo_id,
         that action was made on
     :param ipaddr: optional ip address from what the action was made
     :param sa: optional sqlalchemy session
     
     """
 
     if not sa:
         sa = meta.Session()
 
     try:
         um = UserModel()
         if hasattr(user, 'user_id'):
             user_obj = user
         elif isinstance(user, basestring):
             user_obj = um.get_by_username(user, cache=False)
         else:
             raise Exception('You have to provide user object or username')
 
 
         rm = RepoModel()
         if hasattr(repo, 'repo_id'):
             repo_obj = rm.get(repo.repo_id, cache=False)
             repo_name = repo_obj.repo_name
         elif  isinstance(repo, basestring):
             repo_name = repo.lstrip('/')
             repo_obj = rm.get_by_repo_name(repo_name, cache=False)
         else:
             raise Exception('You have to provide repository to action logger')
 
 
         user_log = UserLog()
         user_log.user_id = user_obj.user_id
         user_log.action = action
 
         user_log.repository_id = repo_obj.repo_id
         user_log.repository_name = repo_name
 
         user_log.action_date = datetime.datetime.now()
         user_log.user_ip = ipaddr
         sa.add(user_log)
         sa.commit()
 
         log.info('Adding user %s, action %s on %s', user_obj, action, repo)
     except:
         log.error(traceback.format_exc())
         sa.rollback()
 
 def get_repos(path, recursive=False, initial=False):
     """
     Scans given path for repos and return (name,(type,path)) tuple 
     :param prefix:
     :param path:
     :param recursive:
     :param initial:
     """
     from vcs.utils.helpers import get_scm
     from vcs.exceptions import VCSError
 
     try:
         scm = get_scm(path)
     except:
         pass
     else:
         raise Exception('The given path %s should not be a repository got %s',
                         path, scm)
 
     for dirpath in os.listdir(path):
         try:
             yield dirpath, get_scm(os.path.join(path, dirpath))
         except VCSError:
             pass
 
 def check_repo_fast(repo_name, base_path):
     """
     Check given path for existance of directory
     :param repo_name:
     :param base_path:
     
     :return False: if this directory is present
     """
     if os.path.isdir(os.path.join(base_path, repo_name)):return False
     return True
 
 def check_repo(repo_name, base_path, verify=True):
 
     repo_path = os.path.join(base_path, repo_name)
 
     try:
         if not check_repo_fast(repo_name, base_path):
             return False
         r = hg.repository(ui.ui(), repo_path)
         if verify:
             hg.verify(r)
         #here we hnow that repo exists it was verified
         log.info('%s repo is already created', repo_name)
         return False
     except RepoError:
         #it means that there is no valid repo there...
         log.info('%s repo is free for creation', repo_name)
         return True
 
 def ask_ok(prompt, retries=4, complaint='Yes or no, please!'):
     while True:
         ok = raw_input(prompt)
         if ok in ('y', 'ye', 'yes'): return True
         if ok in ('n', 'no', 'nop', 'nope'): return False
         retries = retries - 1
         if retries < 0: raise IOError
         print complaint
 
 #propagated from mercurial documentation
 ui_sections = ['alias', 'auth',
                 'decode/encode', 'defaults',
                 'diff', 'email',
                 'extensions', 'format',
                 'merge-patterns', 'merge-tools',
                 'hooks', 'http_proxy',
                 'smtp', 'patch',
                 'paths', 'profiling',
                 'server', 'trusted',
                 'ui', 'web', ]
 
 def make_ui(read_from='file', path=None, checkpaths=True):
     """
     A function that will read python rc files or database
     and make an mercurial ui object from read options
     
     :param path: path to mercurial config file
     :param checkpaths: check the path
     :param read_from: read from 'file' or 'db'
     """
 
     baseui = ui.ui()
 
     #clean the baseui object
     baseui._ocfg = config.config()
     baseui._ucfg = config.config()
     baseui._tcfg = config.config()
 
     if read_from == 'file':
         if not os.path.isfile(path):
             log.warning('Unable to read config file %s' % path)
             return False
         log.debug('reading hgrc from %s', path)
         cfg = config.config()
         cfg.read(path)
         for section in ui_sections:
             for k, v in cfg.items(section):
                 log.debug('settings ui from file[%s]%s:%s', section, k, v)
                 baseui.setconfig(section, k, v)
 
 
     elif read_from == 'db':
         sa = meta.Session()
         ret = sa.query(RhodeCodeUi)\
             .options(FromCache("sql_cache_short",
                                "get_hg_ui_settings")).all()
 
         hg_ui = ret
         for ui_ in hg_ui:
             if ui_.ui_active:
                 log.debug('settings ui from db[%s]%s:%s', ui_.ui_section,
                           ui_.ui_key, ui_.ui_value)
                 baseui.setconfig(ui_.ui_section, ui_.ui_key, ui_.ui_value)
 
         meta.Session.remove()
     return baseui
 
 
 def set_rhodecode_config(config):
     """
     Updates pylons config with new settings from database
     :param config:
     """
     from rhodecode.model.settings import SettingsModel
     hgsettings = SettingsModel().get_app_settings()
 
     for k, v in hgsettings.items():
         config[k] = v
 
 def invalidate_cache(cache_key, *args):
     """
     Puts cache invalidation task into db for 
     further global cache invalidation
     """
     from rhodecode.model.scm import ScmModel
 
     if cache_key.startswith('get_repo_cached_'):
         name = cache_key.split('get_repo_cached_')[-1]
         ScmModel().mark_for_invalidation(name)
 
 class EmptyChangeset(BaseChangeset):
     """
     An dummy empty changeset. It's possible to pass hash when creating
     an EmptyChangeset
     """
 
     def __init__(self, cs='0' * 40):
         self._empty_cs = cs
         self.revision = -1
         self.message = ''
         self.author = ''
         self.date = ''
 
     @LazyProperty
     def raw_id(self):
         """
         Returns raw string identifying this changeset, useful for web
         representation.
         """
         return self._empty_cs
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     def get_file_changeset(self, path):
         return self
 
     def get_file_content(self, path):
         return u''
 
     def get_file_size(self, path):
         return 0
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False):
     """
     maps all found repositories into db
     """
 
     sa = meta.Session()
     rm = RepoModel()
     user = sa.query(User).filter(User.admin == True).first()
 
     for name, repo in initial_repo_list.items():
         if not rm.get_by_repo_name(name, cache=False):
             log.info('repository %s not found creating default', name)
 
             form_data = {
                          'repo_name':name,
                          'repo_type':repo.alias,
                          'description':repo.description \
                             if repo.description != 'unknown' else \
                                         '%s repository' % name,
                          'private':False
                          }
             rm.create(form_data, user, just_db=True)
 
     if remove_obsolete:
         #remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in initial_repo_list.keys():
                 sa.delete(repo)
                 sa.commit()
 
 class OrderedDict(dict, DictMixin):
 
     def __init__(self, *args, **kwds):
         if len(args) > 1:
             raise TypeError('expected at most 1 arguments, got %d' % len(args))
         try:
             self.__end
         except AttributeError:
             self.clear()
         self.update(*args, **kwds)
 
     def clear(self):
         self.__end = end = []
         end += [None, end, end]         # sentinel node for doubly linked list
         self.__map = {}                 # key --> [key, prev, next]
         dict.clear(self)
 
     def __setitem__(self, key, value):
         if key not in self:
             end = self.__end
             curr = end[1]
             curr[2] = end[1] = self.__map[key] = [key, curr, end]
         dict.__setitem__(self, key, value)
 
     def __delitem__(self, key):
         dict.__delitem__(self, key)
         key, prev, next = self.__map.pop(key)
         prev[2] = next
         next[1] = prev
 
     def __iter__(self):
         end = self.__end
         curr = end[2]
         while curr is not end:
             yield curr[0]
             curr = curr[2]
 
     def __reversed__(self):
         end = self.__end
         curr = end[1]
         while curr is not end:
             yield curr[0]
             curr = curr[1]
 
     def popitem(self, last=True):
         if not self:
             raise KeyError('dictionary is empty')
         if last:
             key = reversed(self).next()
         else:
             key = iter(self).next()
         value = self.pop(key)
         return key, value
 
     def __reduce__(self):
         items = [[k, self[k]] for k in self]
         tmp = self.__map, self.__end
         del self.__map, self.__end
         inst_dict = vars(self).copy()
         self.__map, self.__end = tmp
         if inst_dict:
             return (self.__class__, (items,), inst_dict)
         return self.__class__, (items,)
 
     def keys(self):
         return list(self)
 
     setdefault = DictMixin.setdefault
     update = DictMixin.update
     pop = DictMixin.pop
     values = DictMixin.values
     items = DictMixin.items
     iterkeys = DictMixin.iterkeys
     itervalues = DictMixin.itervalues
     iteritems = DictMixin.iteritems
 
     def __repr__(self):
         if not self:
             return '%s()' % (self.__class__.__name__,)
         return '%s(%r)' % (self.__class__.__name__, self.items())
 
     def copy(self):
         return self.__class__(self)
 
     @classmethod
     def fromkeys(cls, iterable, value=None):
         d = cls()
         for key in iterable:
             d[key] = value
         return d
 
     def __eq__(self, other):
         if isinstance(other, OrderedDict):
             return len(self) == len(other) and self.items() == other.items()
         return dict.__eq__(self, other)
 
     def __ne__(self, other):
         return not self == other
 
 
+#set cache regions for beaker so celery can utilise it
+def add_cache(settings):
+    cache_settings = {'regions':None}
+    for key in settings.keys():
+        for prefix in ['beaker.cache.', 'cache.']:
+            if key.startswith(prefix):
+                name = key.split(prefix)[1].strip()
+                cache_settings[name] = settings[key].strip()
+    if cache_settings['regions']:
+        for region in cache_settings['regions'].split(','):
+            region = region.strip()
+            region_settings = {}
+            for key, value in cache_settings.items():
+                if key.startswith(region):
+                    region_settings[key.split('.')[1]] = value
+            region_settings['expire'] = int(region_settings.get('expire',
+                                                                60))
+            region_settings.setdefault('lock_dir',
+                                       cache_settings.get('lock_dir'))
+            if 'type' not in region_settings:
+                region_settings['type'] = cache_settings.get('type',
+                                                             'memory')
+            beaker.cache.cache_regions[region] = region_settings
+
+
 #===============================================================================
 # TEST FUNCTIONS AND CREATORS
 #===============================================================================
 def create_test_index(repo_location, full_index):
     """Makes default test index
     :param repo_location:
     :param full_index:
     """
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     from rhodecode.lib.pidlock import DaemonLock, LockHeld
     import shutil
 
     index_location = os.path.join(repo_location, 'index')
     if os.path.exists(index_location):
         shutil.rmtree(index_location)
 
     try:
         l = DaemonLock()
         WhooshIndexingDaemon(index_location=index_location,
                              repo_location=repo_location)\
             .run(full_index=full_index)
         l.release()
     except LockHeld:
         pass
 
 def create_test_env(repos_test_path, config):
     """Makes a fresh database and 
     install test repository into tmp dir
     """
     from rhodecode.lib.db_manage import DbManage
     from rhodecode.tests import HG_REPO, GIT_REPO, NEW_HG_REPO, NEW_GIT_REPO, \
         HG_FORK, GIT_FORK, TESTS_TMP_PATH
     import tarfile
     import shutil
     from os.path import dirname as dn, join as jn, abspath
 
     log = logging.getLogger('TestEnvCreator')
     # create logger
     log.setLevel(logging.DEBUG)
     log.propagate = True
     # create console handler and set level to debug
     ch = logging.StreamHandler()
     ch.setLevel(logging.DEBUG)
 
     # create formatter
     formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 
     # add formatter to ch
     ch.setFormatter(formatter)
 
     # add ch to logger
     log.addHandler(ch)
 
     #PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     log.debug('making test db %s', dbconf)
 
     dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],
                         tests=True)
     dbmanage.create_tables(override=True)
     dbmanage.config_prompt(repos_test_path)
     dbmanage.create_default_user()
     dbmanage.admin_prompt()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
 
     #PART TWO make test repo
     log.debug('making test vcs repositories')
 
     #remove old one from previos tests
     for r in [HG_REPO, GIT_REPO, NEW_HG_REPO, NEW_GIT_REPO, HG_FORK, GIT_FORK]:
 
         if os.path.isdir(jn(TESTS_TMP_PATH, r)):
             log.debug('removing %s', r)
             shutil.rmtree(jn(TESTS_TMP_PATH, r))
 
     #CREATE DEFAULT HG REPOSITORY
     cur_dir = dn(dn(abspath(__file__)))
     tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test_hg.tar.gz"))
     tar.extractall(jn(TESTS_TMP_PATH, HG_REPO))
     tar.close()
 
-class UpgradeDb(command.Command):
+
+#==============================================================================
+# PASTER COMMANDS
+#==============================================================================
+
+class BasePasterCommand(Command):
+    """
+    Abstract Base Class for paster commands.
+
+    The celery commands are somewhat aggressive about loading
+    celery.conf, and since our module sets the `CELERY_LOADER`
+    environment variable to our loader, we have to bootstrap a bit and
+    make sure we've had a chance to load the pylons config off of the
+    command line, otherwise everything fails.
+    """
+    min_args = 1
+    min_args_error = "Please provide a paster config file as an argument."
+    takes_config_file = 1
+    requires_config_file = True
+
+    def run(self, args):
+        """
+        Overrides Command.run
+        
+        Checks for a config file argument and loads it.
+        """
+        if len(args) < self.min_args:
+            raise BadCommand(
+                self.min_args_error % {'min_args': self.min_args,
+                                       'actual_args': len(args)})
+
+        # Decrement because we're going to lob off the first argument.
+        # @@ This is hacky
+        self.min_args -= 1
+        self.bootstrap_config(args[0])
+        self.update_parser()
+        return super(BasePasterCommand, self).run(args[1:])
+
+    def update_parser(self):
+        """
+        Abstract method.  Allows for the class's parser to be updated
+        before the superclass's `run` method is called.  Necessary to
+        allow options/arguments to be passed through to the underlying
+        celery command.
+        """
+        raise NotImplementedError("Abstract Method.")
+
+    def bootstrap_config(self, conf):
+        """
+        Loads the pylons configuration.
+        """
+        from pylons import config as pylonsconfig
+
+        path_to_ini_file = os.path.realpath(conf)
+        conf = paste.deploy.appconfig('config:' + path_to_ini_file)
+        pylonsconfig.init_app(conf.global_conf, conf.local_conf)
+
+
+
+class UpgradeDb(BasePasterCommand):
     """Command used for paster to upgrade our database to newer version
     """
 
     max_args = 1
     min_args = 1
 
     usage = "CONFIG_FILE"
     summary = "Upgrades current db to newer version given configuration file"
     group_name = "RhodeCode"
 
-    parser = command.Command.standard_parser(verbose=True)
+    parser = Command.standard_parser(verbose=True)
 
-    parser.add_option('--sql',
+    def command(self):
+        from pylons import config
+        raise NotImplementedError('Not implemented yet')
+
+
+    def update_parser(self):
+        self.parser.add_option('--sql',
                       action='store_true',
                       dest='just_sql',
                       help="Prints upgrade sql for further investigation",
                       default=False)
-    def command(self):
-        config_name = self.args[0]
-        p = config_name.split('/')
-        root = '.' if len(p) == 1 else '/'.join(p[:-1])
-        config = ConfigParser.ConfigParser({'here':root})
-        config.read(config_name)