complete rewrite of paster commands,

used fancy celery-pylons baseCommand to implement better support for paster commands
fixed make-index, and paster celeryd. Moved config's add_cache to utils

rhodecode/lib/celerylib/tasks.py

@@ -2,7 +2,6 b' from celery.decorators import task'
 
 import os
 import traceback
-import beaker
 from time import mktime
 from operator import itemgetter
 
@@ -12,7 +11,7 b' from pylons.i18n.translation import _'
 from rhodecode.lib.celerylib import run_task, locked_task, str2bool
 from rhodecode.lib.helpers import person
 from rhodecode.lib.smtp_mailer import SmtpMailer
-from rhodecode.lib.utils import OrderedDict
+from rhodecode.lib.utils import OrderedDict, add_cache
 from rhodecode.model import init_model
 from rhodecode.model import meta
 from rhodecode.model.db import RhodeCodeUi
@@ -21,29 +20,6 b' from vcs.backends import get_repo'
 
 from sqlalchemy import engine_from_config
 
-#set cache regions for beaker so celery can utilise it
-def add_cache(settings):
-    cache_settings = {'regions':None}
-    for key in settings.keys():
-        for prefix in ['beaker.cache.', 'cache.']:
-            if key.startswith(prefix):
-                name = key.split(prefix)[1].strip()
-                cache_settings[name] = settings[key].strip()
-    if cache_settings['regions']:
-        for region in cache_settings['regions'].split(','):
-            region = region.strip()
-            region_settings = {}
-            for key, value in cache_settings.items():
-                if key.startswith(region):
-                    region_settings[key.split('.')[1]] = value
-            region_settings['expire'] = int(region_settings.get('expire',
-                                                                60))
-            region_settings.setdefault('lock_dir',
-                                       cache_settings.get('lock_dir'))
-            if 'type' not in region_settings:
-                region_settings['type'] = cache_settings.get('type',
-                                                             'memory')
-            beaker.cache.cache_regions[region] = region_settings
 add_cache(config)
 
 try:

rhodecode/lib/celerypylons/commands.py

@@ -1,64 +1,11 b''
-import os
-from paste.script.command import Command, BadCommand
-import paste.deploy
-from pylons import config
+from rhodecode.lib.utils import BasePasterCommand, Command
 
 
 __all__ = ['CeleryDaemonCommand', 'CeleryBeatCommand',
            'CAMQPAdminCommand', 'CeleryEventCommand']
 
 
-class CeleryCommand(Command):
-    """
-    Abstract Base Class for celery commands.
-
-    The celery commands are somewhat aggressive about loading
-    celery.conf, and since our module sets the `CELERY_LOADER`
-    environment variable to our loader, we have to bootstrap a bit and
-    make sure we've had a chance to load the pylons config off of the
-    command line, otherwise everything fails.
-    """
-    min_args = 1
-    min_args_error = "Please provide a paster config file as an argument."
-    takes_config_file = 1
-    requires_config_file = True
-
-    def run(self, args):
-        """
-        Overrides Command.run
-        
-        Checks for a config file argument and loads it.
-        """
-        if len(args) < self.min_args:
-            raise BadCommand(
-                self.min_args_error % {'min_args': self.min_args,
-                                       'actual_args': len(args)})
-        # Decrement because we're going to lob off the first argument.
-        # @@ This is hacky
-        self.min_args -= 1
-        self.bootstrap_config(args[0])
-        self.update_parser()
-        return super(CeleryCommand, self).run(args[1:])
-
-    def update_parser(self):
-        """
-        Abstract method.  Allows for the class's parser to be updated
-        before the superclass's `run` method is called.  Necessary to
-        allow options/arguments to be passed through to the underlying
-        celery command.
-        """
-        raise NotImplementedError("Abstract Method.")
-
-    def bootstrap_config(self, conf):
-        """
-        Loads the pylons configuration.
-        """
-        path_to_ini_file = os.path.realpath(conf)
-        conf = paste.deploy.appconfig('config:' + path_to_ini_file)
-        config.init_app(conf.global_conf, conf.local_conf)
-
-
-class CeleryDaemonCommand(CeleryCommand):
+class CeleryDaemonCommand(BasePasterCommand):
     """Start the celery worker
 
     Starts the celery worker that uses a paste.deploy configuration
@@ -80,7 +27,7 b' class CeleryDaemonCommand(CeleryCommand)'
         return celeryd.WorkerCommand().run(**vars(self.options))
 
 
-class CeleryBeatCommand(CeleryCommand):
+class CeleryBeatCommand(BasePasterCommand):
     """Start the celery beat server
 
     Starts the celery beat server using a paste.deploy configuration
@@ -101,7 +48,7 b' class CeleryBeatCommand(CeleryCommand):'
         from celery.bin import celerybeat
         return celerybeat.BeatCommand(**vars(self.options))
 
-class CAMQPAdminCommand(CeleryCommand):
+class CAMQPAdminCommand(BasePasterCommand):
     """CAMQP Admin
 
     CAMQP celery admin tool.
@@ -122,7 +69,7 b' class CAMQPAdminCommand(CeleryCommand):'
         return camqadm.camqadm(*self.args, **vars(self.options))
 
 
-class CeleryEventCommand(CeleryCommand):
+class CeleryEventCommand(BasePasterCommand):
     """Celery event commandd.
 
     Capture celery events.

rhodecode/lib/indexers/__init__.py

@@ -1,23 +1,28 b''
 import os
 import sys
+import traceback
 from os.path import dirname as dn, join as jn
 
 #to get the rhodecode import
 sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
 
+from rhodecode.model import init_model
+from rhodecode.model.scm import ScmModel
 from rhodecode.config.environment import load_environment
-from rhodecode.model.scm import ScmModel
+from rhodecode.lib.utils import BasePasterCommand, Command, add_cache
+
 from shutil import rmtree
 from webhelpers.html.builder import escape
 from vcs.utils.lazy import LazyProperty
 
+from sqlalchemy import engine_from_config
+
 from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter
 from whoosh.fields import TEXT, ID, STORED, Schema, FieldType
 from whoosh.index import create_in, open_dir
 from whoosh.formats import Characters
 from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter
 
-import traceback
 
 #EXTENSIONS WE WANT TO INDEX CONTENT OFF
 INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
@@ -45,10 +50,8 b" IDX_NAME = 'HG_INDEX'"
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = SimpleFragmenter(200)
 
-from paste.script import command
-import ConfigParser
 
-class MakeIndex(command.Command):
+class MakeIndex(BasePasterCommand):
 
     max_args = 1
     min_args = 1
@@ -57,26 +60,16 b' class MakeIndex(command.Command):'
     summary = "Creates index for full text search given configuration file"
     group_name = "RhodeCode"
     takes_config_file = -1
-    parser = command.Command.standard_parser(verbose=True)
-    parser.add_option('--repo-location',
-                      action='store',
-                      dest='repo_location',
-                      help="Specifies repositories location to index REQUIRED",
-                      )
-    parser.add_option('-f',
-                      action='store_true',
-                      dest='full_index',
-                      help="Specifies that index should be made full i.e"
-                            " destroy old and build from scratch",
-                      default=False)
+    parser = Command.standard_parser(verbose=True)
+
     def command(self):
-        config_name = self.args[0]
-        p = config_name.split('/')
-        root = '.' if len(p) == 1 else '/'.join(p[:-1])
-        config = ConfigParser.ConfigParser({'here':root})
-        config.read(config_name)
 
-        index_location = dict(config.items('app:main'))['index_dir']
+        from pylons import config
+        add_cache(config)
+        engine = engine_from_config(config, 'sqlalchemy.db1.')
+        init_model(engine)
+
+        index_location = config['index_dir']
         repo_location = self.options.repo_location
 
         #======================================================================
@@ -93,6 +86,18 b' class MakeIndex(command.Command):'
         except LockHeld:
             sys.exit(1)
 
+    def update_parser(self):
+        self.parser.add_option('--repo-location',
+                          action='store',
+                          dest='repo_location',
+                          help="Specifies repositories location to index REQUIRED",
+                          )
+        self.parser.add_option('-f',
+                          action='store_true',
+                          dest='full_index',
+                          help="Specifies that index should be made full i.e"
+                                " destroy old and build from scratch",
+                          default=False)
 
 class ResultWrapper(object):
     def __init__(self, search_type, searcher, matcher, highlight_items):

rhodecode/lib/utils.py

@@ -36,7 +36,10 b' from UserDict import DictMixin'
 from mercurial import ui, config, hg
 from mercurial.error import RepoError
 
-from paste.script import command
+import paste
+import beaker
+from paste.script.command import Command, BadCommand
+
 from vcs.backends.base import BaseChangeset
 from vcs.utils.lazy import LazyProperty
 
@@ -416,6 +419,31 b' class OrderedDict(dict, DictMixin):'
         return not self == other
 
 
+#set cache regions for beaker so celery can utilise it
+def add_cache(settings):
+    cache_settings = {'regions':None}
+    for key in settings.keys():
+        for prefix in ['beaker.cache.', 'cache.']:
+            if key.startswith(prefix):
+                name = key.split(prefix)[1].strip()
+                cache_settings[name] = settings[key].strip()
+    if cache_settings['regions']:
+        for region in cache_settings['regions'].split(','):
+            region = region.strip()
+            region_settings = {}
+            for key, value in cache_settings.items():
+                if key.startswith(region):
+                    region_settings[key.split('.')[1]] = value
+            region_settings['expire'] = int(region_settings.get('expire',
+                                                                60))
+            region_settings.setdefault('lock_dir',
+                                       cache_settings.get('lock_dir'))
+            if 'type' not in region_settings:
+                region_settings['type'] = cache_settings.get('type',
+                                                             'memory')
+            beaker.cache.cache_regions[region] = region_settings
+
+
 #===============================================================================
 # TEST FUNCTIONS AND CREATORS
 #===============================================================================
@@ -498,7 +526,66 b' def create_test_env(repos_test_path, con'
     tar.extractall(jn(TESTS_TMP_PATH, HG_REPO))
     tar.close()
 
-class UpgradeDb(command.Command):
+
+#==============================================================================
+# PASTER COMMANDS
+#==============================================================================
+
+class BasePasterCommand(Command):
+    """
+    Abstract Base Class for paster commands.
+
+    The celery commands are somewhat aggressive about loading
+    celery.conf, and since our module sets the `CELERY_LOADER`
+    environment variable to our loader, we have to bootstrap a bit and
+    make sure we've had a chance to load the pylons config off of the
+    command line, otherwise everything fails.
+    """
+    min_args = 1
+    min_args_error = "Please provide a paster config file as an argument."
+    takes_config_file = 1
+    requires_config_file = True
+
+    def run(self, args):
+        """
+        Overrides Command.run
+        
+        Checks for a config file argument and loads it.
+        """
+        if len(args) < self.min_args:
+            raise BadCommand(
+                self.min_args_error % {'min_args': self.min_args,
+                                       'actual_args': len(args)})
+
+        # Decrement because we're going to lob off the first argument.
+        # @@ This is hacky
+        self.min_args -= 1
+        self.bootstrap_config(args[0])
+        self.update_parser()
+        return super(BasePasterCommand, self).run(args[1:])
+
+    def update_parser(self):
+        """
+        Abstract method.  Allows for the class's parser to be updated
+        before the superclass's `run` method is called.  Necessary to
+        allow options/arguments to be passed through to the underlying
+        celery command.
+        """
+        raise NotImplementedError("Abstract Method.")
+
+    def bootstrap_config(self, conf):
+        """
+        Loads the pylons configuration.
+        """
+        from pylons import config as pylonsconfig
+
+        path_to_ini_file = os.path.realpath(conf)
+        conf = paste.deploy.appconfig('config:' + path_to_ini_file)
+        pylonsconfig.init_app(conf.global_conf, conf.local_conf)
+
+
+
+class UpgradeDb(BasePasterCommand):
     """Command used for paster to upgrade our database to newer version
     """
 
@@ -509,16 +596,16 b' class UpgradeDb(command.Command):'
     summary = "Upgrades current db to newer version given configuration file"
     group_name = "RhodeCode"
 
-    parser = command.Command.standard_parser(verbose=True)
+    parser = Command.standard_parser(verbose=True)
 
-    parser.add_option('--sql',
+    def command(self):
+        from pylons import config
+        raise NotImplementedError('Not implemented yet')
+
+
+    def update_parser(self):
+        self.parser.add_option('--sql',
                       action='store_true',
                       dest='just_sql',
                       help="Prints upgrade sql for further investigation",
                       default=False)
-    def command(self):
-        config_name = self.args[0]
-        p = config_name.split('/')
-        root = '.' if len(p) == 1 else '/'.join(p[:-1])
-        config = ConfigParser.ConfigParser({'here':root})
-        config.read(config_name)