Implemented whoosh index building as paster command.

docs update
fixed manifest.in for missing yui file.
Fixed setup to beta
added base for paster upgrade-db command

MANIFEST.in

@@ -7,7 +7,7 b' recursive-include rhodecode/i18n/ *'
 recursive-include rhodecode/public/css *
 recursive-include rhodecode/public/images *
 #js
-include rhodecode/public/js/yui2.js
+include rhodecode/public/js/yui2a.js
 include rhodecode/public/js/excanvas.min.js
 include rhodecode/public/js/yui.flot.js
 include rhodecode/public/js/graph.js

development.ini

@@ -43,6 +43,7 b' full_stack = true'
 static_files = true
 lang=en
 cache_dir = %(here)s/data
+index_dir = %(here)s/data/index
 
 ####################################
 ###         BEAKER CACHE        ####

docs/changelog.rst

@@ -17,6 +17,7 b' 1.1.0 (**2010-XX-XX**)'
 - introduced new enhanced changelog for merges that shows more accurate results
 - gui optimizations, fixed application width to 1024px
 - numerous small bugfixes
+- whoosh index moved to paster command
 
 1.0.2 (**2010-11-12**)
 ----------------------

docs/setup.rst

@@ -41,21 +41,40 b' Setting up the application'
   remember to update these if needed.
   
     
-Setting up Whoosh
------------------
+Setting up Whoosh full text search
+----------------------------------
+
+Index for whoosh can be build starting from version 1.1 using paster command
+passing repo locations to index, as well as Your config file that stores
+whoosh index files locations. There is possible to pass `-f` to the options
+to enable full index rebuild. Without that indexing will run always in in
+incremental mode.
+
+::
+ paster make-index --repo-location=<location for repos> production.ini  
+
+for full index rebuild You can use
+
+::
+ paster make-index -f --repo-location=<location for repos> production.ini
 
 - For full text search You can either put crontab entry for
 
+This command can be run even from crontab in order to do periodical 
+index builds and keep Your index always up to date. An example entry might 
+look like this
+
 ::
  
- python /var/www/rhodecode/<rhodecode_installation_path>/lib/indexers/daemon.py incremental <put_here_path_to_repos>
+ /path/to/python/bin/paster --repo-location=<location for repos> /path/to/rhodecode/production.ini
   
-When using incremental mode whoosh will check last modification date of each file
-and add it to reindex if newer file is available. Also indexing daemon checks
-for removed files and removes them from index. Sometime You might want to rebuild
-index from scrach, in admin pannel You can check `build from scratch` flag
-and in standalone daemon You can pass `full` instead on incremental to build
-remove previos index and build new one.
+When using incremental(default) mode whoosh will check last modification date 
+of each file and add it to reindex if newer file is available. Also indexing 
+daemon checks for removed files and removes them from index. 
+
+Sometime You might want to rebuild index from scratch. You can do that using 
+the `-f` flag passed to paster command or, in admin panel You can check 
+`build from scratch` flag.
 
 Nginx virtual host example
 --------------------------

production.ini

@@ -43,6 +43,7 b' full_stack = true'
 static_files = false
 lang=en
 cache_dir = %(here)s/data
+index_dir = %(here)s/data/index
 
 ####################################
 ###         BEAKER CACHE        ####

rhodecode/config/deployment.ini_tmpl

@@ -43,6 +43,7 b' full_stack = true'
 static_files = true
 lang=en
 cache_dir = %(here)s/data
+index_dir = %(here)s/data/index
 app_instance_uuid = ${app_instance_uuid}
 
 ####################################

rhodecode/lib/indexers/__init__.py

@@ -59,14 +59,14 b' class MakeIndex(command.Command):'
 
     usage = "CONFIG_FILE"
     summary = "Creates index for full text search given configuration file"
-    group_name = "Whoosh indexing"
-
+    group_name = "RhodeCode"
+    takes_config_file = -1
     parser = command.Command.standard_parser(verbose=True)
-#    parser.add_option('--repo-location',
-#                      action='store',
-#                      dest='repo_location',
-#                      help="Specifies repositories location to index",
-#                      )
+    parser.add_option('--repo-location',
+                      action='store',
+                      dest='repo_location',
+                      help="Specifies repositories location to index REQUIRED",
+                      )
     parser.add_option('-f',
                       action='store_true',
                       dest='full_index',
@@ -75,27 +75,23 b' class MakeIndex(command.Command):'
                       default=False)
     def command(self):
         config_name = self.args[0]
-
         p = config_name.split('/')
-        if len(p) == 1:
-            root = '.'
-        else:
-            root = '/'.join(p[:-1])
-        print root
+        root = '.' if len(p) == 1 else '/'.join(p[:-1])
         config = ConfigParser.ConfigParser({'here':root})
         config.read(config_name)
-        print dict(config.items('app:main'))['index_dir']
+
         index_location = dict(config.items('app:main'))['index_dir']
-        #return
+        repo_location = self.options.repo_location
 
-        #=======================================================================
+        #======================================================================
         # WHOOSH DAEMON
-        #=======================================================================
+        #======================================================================
         from rhodecode.lib.pidlock import LockHeld, DaemonLock
         from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
         try:
             l = DaemonLock()
-            WhooshIndexingDaemon(index_location=index_location)\
+            WhooshIndexingDaemon(index_location=index_location,
+                                 repo_location=repo_location)\
                 .run(full_index=self.options.full_index)
             l.release()
         except LockHeld:

rhodecode/lib/indexers/daemon.py

@@ -78,9 +78,7 b' class WhooshIndexingDaemon(object):'
         if not repo_location:
             raise Exception('You have to provide repositories location')
 
-
-
-        self.repo_paths = HgModel.repo_scan('/', self.repo_location, None, True)
+        self.repo_paths = HgModel().repo_scan(self.repo_location, None, True)
         self.initial = False
         if not os.path.isdir(self.index_location):
             os.mkdir(self.index_location)
@@ -89,8 +87,7 b' class WhooshIndexingDaemon(object):'
             self.initial = True
 
     def get_paths(self, repo):
-        """
-        recursive walk in root dir and return a set of all path in that dir
+        """recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
         """
         index_paths_ = set()
@@ -115,7 +112,8 b' class WhooshIndexingDaemon(object):'
         return mktime(node.last_changeset.date.timetuple())
 
     def add_doc(self, writer, path, repo):
-        """Adding doc to writer"""
+        """Adding doc to writer this function itself fetches data from
+        the instance of vcs backend"""
         node = self.get_node(repo, path)
 
         #we just index the content of chosen files

setup.cfg

@@ -1,5 +1,5 b''
 [egg_info]
-tag_build = rc4
+tag_build = beta
 tag_svn_revision = true
 
 [easy_install]

setup.py

@@ -89,5 +89,10 b' setup('
 
     [paste.app_install]
     main = pylons.util:PylonsInstaller
+
+    [paste.global_paster_command]
+    make-index = rhodecode.lib.indexers:MakeIndex
+    upgrade-db = rhodecode.lib.utils:UpgradeDb
+        
     """,
 )