updated config files,

Implemented content index extensions with whoosh,
fixed analyzer to match more words

development.ini

@@ -1,135 +1,155 b''
 ################################################################################
 ################################################################################
 # pylons_app - Pylons environment configuration                                #
 #                                                                              # 
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 
 [DEFAULT]
 debug = true
 ############################################
 ## Uncomment and replace with the address ##
 ## which should receive any error reports ##
 ############################################
 #email_to = admin@localhost
 #smtp_server = mail.server.com
 #error_email_from = paste_error@localhost
 #smtp_username = 
 #smtp_password = 
 #error_message = 'mercurial crash !'
 
 [server:main]
 ##nr of threads to spawn
 threadpool_workers = 5
 
 ##max request before
 threadpool_max_requests = 2
 
 ##option to use threads of process
 use_threadpool = true
 
 use = egg:Paste#http
 host = 127.0.0.1
 port = 5000
 
 [app:main]
 use = egg:pylons_app
 full_stack = true
 static_files = true
 lang=en
 cache_dir = %(here)s/data
 
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir=/%(here)s/data/cache/data
 beaker.cache.lock_dir=/%(here)s/data/cache/lock
 beaker.cache.regions=super_short_term,short_term,long_term
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
 
+####################################
+###       BEAKER SESSION        ####
+####################################
+## Type of storage used for the session, current types are 
+## “dbm”, “file”, “memcached”, “database”, and “memory”. 
+## The storage uses the Container API 
+##that is also used by the cache system.
+beaker.session.type = file
+
+beaker.session.key = hg-app
+beaker.session.secret = g654dcno0-9873jhgfreyu
+beaker.session.timeout = 36000
+
+##auto save the session to not to use .save()
+beaker.session.auto = False
+
+##true exire at browser close
+#beaker.session.cookie_expires = 3600
+
+    
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 #set debug = false
 
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 sqlalchemy.db1.url = sqlite:///%(here)s/hg_app.db
 #sqlalchemy.db1.echo = False
 #sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, pylons_app, sqlalchemy
 
 [handlers]
 keys = console
 
 [formatters]
 keys = generic,color_formatter
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_routes]
 level = DEBUG
 handlers = console
 qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
 
 [logger_pylons_app]
 level = DEBUG
 handlers = console
 qualname = pylons_app
 propagate = 0
 
 [logger_sqlalchemy]
 level = ERROR
 handlers = console
 qualname = sqlalchemy.engine
 propagate = 0
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
 formatter = color_formatter
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class=pylons_app.lib.colored_formatter.ColorFormatter
 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
  No newline at end of file

production.ini

@@ -1,135 +1,155 b''
 ################################################################################
 ################################################################################
 # pylons_app - Pylons environment configuration                                #
 #                                                                              # 
 # The %(here)s variable will be replaced with the parent directory of this file#
 ################################################################################
 
 [DEFAULT]
 debug = true
 ############################################
 ## Uncomment and replace with the address ##
 ## which should receive any error reports ##
 ############################################
 #email_to = admin@localhost
 #smtp_server = mail.server.com
 #error_email_from = paste_error@localhost
 #smtp_username = 
 #smtp_password = 
 #error_message = 'mercurial crash !'
 
 [server:main]
 ##nr of threads to spawn
 threadpool_workers = 5
 
 ##max request before
 threadpool_max_requests = 2
 
 ##option to use threads of process
 use_threadpool = true
 
 use = egg:Paste#http
 host = 127.0.0.1
 port = 8001
 
 [app:main]
 use = egg:pylons_app
 full_stack = true
 static_files = false
 lang=en
 cache_dir = %(here)s/data
 
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 beaker.cache.data_dir=/%(here)s/data/cache/data
 beaker.cache.lock_dir=/%(here)s/data/cache/lock
 beaker.cache.regions=super_short_term,short_term,long_term
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
     
+####################################
+###       BEAKER SESSION        ####
+####################################
+## Type of storage used for the session, current types are 
+## “dbm”, “file”, “memcached”, “database”, and “memory”. 
+## The storage uses the Container API 
+##that is also used by the cache system.
+beaker.session.type = file
+
+beaker.session.key = hg-app
+beaker.session.secret = g654dcno0-9873jhgfreyu
+beaker.session.timeout = 36000
+
+##auto save the session to not to use .save()
+beaker.session.auto = False
+
+##true exire at browser close
+#beaker.session.cookie_expires = 3600
+
+    
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 set debug = false
 
 ##################################
 ###       LOGVIEW CONFIG       ###
 ##################################
 logview.sqlalchemy = #faa
 logview.pylons.templating = #bfb
 logview.pylons.util = #eee
 
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 sqlalchemy.db1.url = sqlite:///%(here)s/hg_app.db
 #sqlalchemy.db1.echo = False
 #sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, pylons_app, sqlalchemy
 
 [handlers]
 keys = console
 
 [formatters]
 keys = generic,color_formatter
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = INFO
 handlers = console
 
 [logger_routes]
 level = INFO
 handlers = console
 qualname = routes.middleware
 # "level = DEBUG" logs the route matched and routing variables.
 
 [logger_pylons_app]
 level = DEBUG
 handlers = console
 qualname = pylons_app
 propagate = 0
 
 [logger_sqlalchemy]
 level = ERROR
 handlers = console
 qualname = sqlalchemy.engine
 propagate = 0
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
 formatter = color_formatter
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class=pylons_app.lib.colored_formatter.ColorFormatter
 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
  No newline at end of file

pylons_app/lib/indexers/__init__.py

@@ -1,36 +1,41 b''
 import sys
 import os
 from pidlock import LockHeld, DaemonLock
 import traceback
 
 from os.path import dirname as dn
 from os.path import join as jn
 
 #to get the pylons_app import
 sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
 
 from pylons_app.config.environment import load_environment
 from pylons_app.model.hg_model import HgModel
 from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter
 from whoosh.fields import TEXT, ID, STORED, Schema
 from whoosh.index import create_in, open_dir
 from shutil import rmtree
 
 #LOCATION WE KEEP THE INDEX
 IDX_LOCATION = jn(dn(dn(dn(dn(os.path.abspath(__file__))))), 'data', 'index')
 
-#EXTENSION TO SKIP READING CONTENT ON
-EXCLUDE_EXTENSIONS = ['pyc', 'mo', 'png', 'jpg', 'jpeg', 'gif', 'swf',
-                       'dll', 'ttf', 'psd', 'svg', 'pdf', 'bmp', 'dll']
+#EXTENSIONS WE WANT TO INDEX CONTENT OFF
+INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c', 
+                    'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl', 'h', 
+                    'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp', 
+                    'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3', 
+                    'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql', 
+                    'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml','xsl','xslt', 
+                    'yaws']
 
 #CUSTOM ANALYZER wordsplit + lowercase filter
-ANALYZER = RegexTokenizer() | LowercaseFilter()
+ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
 
 #INDEX SCHEMA DEFINITION
 SCHEMA = Schema(owner=TEXT(),
                 repository=TEXT(stored=True),
                 path=ID(stored=True, unique=True),
                 content=TEXT(stored=True, analyzer=ANALYZER),
-                modtime=STORED())
+                modtime=STORED(),extension=TEXT(stored=True))
 
-IDX_NAME = 'HG_INDEX'
+IDX_NAME = 'HG_INDEX'
  No newline at end of file

pylons_app/lib/indexers/daemon.py

@@ -1,181 +1,188 b''
 #!/usr/bin/env python
 # encoding: utf-8
 # whoosh indexer daemon for hg-app
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on Jan 26, 2010
 
 @author: marcink
 A deamon will read from task table and run tasks
 """
 import sys
 import os
 from os.path import dirname as dn
 from os.path import join as jn
 
 #to get the pylons_app import
 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 
 from pidlock import LockHeld, DaemonLock
 import traceback
 from pylons_app.config.environment import load_environment
 from pylons_app.model.hg_model import HgModel
 from whoosh.index import create_in, open_dir
 from shutil import rmtree
-from pylons_app.lib.indexers import ANALYZER, EXCLUDE_EXTENSIONS, IDX_LOCATION, \
+from pylons_app.lib.indexers import ANALYZER, INDEX_EXTENSIONS, IDX_LOCATION, \
 SCHEMA, IDX_NAME
 
 import logging
 import logging.config
 logging.config.fileConfig(jn(project_path, 'development.ini'))
 log = logging.getLogger('whooshIndexer')
 
 def scan_paths(root_location):
     return HgModel.repo_scan('/', root_location, None, True)
 
 class WhooshIndexingDaemon(object):
     """Deamon for atomic jobs"""
 
     def __init__(self, indexname='HG_INDEX', repo_location=None):
         self.indexname = indexname
         self.repo_location = repo_location
     
     def get_paths(self, root_dir):
         """recursive walk in root dir and return a set of all path in that dir
         excluding files in .hg dir"""
         index_paths_ = set()
         for path, dirs, files in os.walk(root_dir):
             if path.find('.hg') == -1:
                 for f in files:
                     index_paths_.add(jn(path, f))
     
         return index_paths_
     
     def add_doc(self, writer, path, repo):
         """Adding doc to writer"""
         
-        #we don't won't to read excluded file extensions just index them
-        if path.split('/')[-1].split('.')[-1].lower() not in EXCLUDE_EXTENSIONS:
+        ext = unicode(path.split('/')[-1].split('.')[-1].lower())
+        #we just index the content of choosen files
+        if ext in INDEX_EXTENSIONS:
+            log.debug('    >> %s [WITH CONTENT]' % path)
             fobj = open(path, 'rb')
             content = fobj.read()
             fobj.close()
             try:
                 u_content = unicode(content)
             except UnicodeDecodeError:
                 #incase we have a decode error just represent as byte string
                 u_content = unicode(str(content).encode('string_escape'))
         else:
+            log.debug('    >> %s' % path)
+            #just index file name without it's content
             u_content = u''    
+                
         writer.add_document(owner=unicode(repo.contact),
                             repository=u"%s" % repo.name,
                             path=u"%s" % path,
                             content=u_content,
-                            modtime=os.path.getmtime(path)) 
+                            modtime=os.path.getmtime(path),
+                            extension=ext) 
     
     def build_index(self):
         if os.path.exists(IDX_LOCATION):
+            log.debug('removing previos index')
             rmtree(IDX_LOCATION)
             
         if not os.path.exists(IDX_LOCATION):
             os.mkdir(IDX_LOCATION)
         
         idx = create_in(IDX_LOCATION, SCHEMA, indexname=IDX_NAME)
         writer = idx.writer()
         
         for cnt, repo in enumerate(scan_paths(self.repo_location).values()):
             log.debug('building index @ %s' % repo.path)
         
             for idx_path in self.get_paths(repo.path):
-                log.debug('    >> %s' % idx_path)
                 self.add_doc(writer, idx_path, repo)
         writer.commit(merge=True)
                 
         log.debug('>>> FINISHED BUILDING INDEX <<<')
             
     
     def update_index(self):
         log.debug('STARTING INCREMENTAL INDEXING UPDATE')
             
         idx = open_dir(IDX_LOCATION, indexname=self.indexname)
         # The set of all paths in the index
         indexed_paths = set()
         # The set of all paths we need to re-index
         to_index = set()
         
         reader = idx.reader()
         writer = idx.writer()
     
         # Loop over the stored fields in the index
         for fields in reader.all_stored_fields():
             indexed_path = fields['path']
             indexed_paths.add(indexed_path)
     
             if not os.path.exists(indexed_path):
                 # This file was deleted since it was indexed
                 log.debug('removing from index %s' % indexed_path)
                 writer.delete_by_term('path', indexed_path)
     
             else:
                 # Check if this file was changed since it
                 # was indexed
                 indexed_time = fields['modtime']
                 
                 mtime = os.path.getmtime(indexed_path)
     
                 if mtime > indexed_time:
     
                     # The file has changed, delete it and add it to the list of
                     # files to reindex
                     log.debug('adding to reindex list %s' % indexed_path)
                     writer.delete_by_term('path', indexed_path)
                     to_index.add(indexed_path)
                     #writer.commit()
     
         # Loop over the files in the filesystem
         # Assume we have a function that gathers the filenames of the
         # documents to be indexed
         for repo in scan_paths(self.repo_location).values():
             for path in self.get_paths(repo.path):
                 if path in to_index or path not in indexed_paths:
                     # This is either a file that's changed, or a new file
                     # that wasn't indexed before. So index it!
                     self.add_doc(writer, path, repo)
                     log.debug('reindexing %s' % path)
     
         writer.commit(merge=True)
         #idx.optimize()
         log.debug('>>> FINISHED <<<')
         
     def run(self, full_index=False):
         """Run daemon"""
         if full_index:
             self.build_index()
         else:
             self.update_index()
         
 if __name__ == "__main__":
-    repo_location = '/home/marcink/python_workspace_dirty/*'
-    
+    repo_location = '/home/marcink/hg_repos/*'
+    full_index = True # False means looking just for changes
     try:
         l = DaemonLock()
-        WhooshIndexingDaemon(repo_location=repo_location).run(full_index=True)
+        WhooshIndexingDaemon(repo_location=repo_location)\
+            .run(full_index=full_index)
         l.release()
     except LockHeld:
         sys.exit(1)