Show More
@@ -52,6 +52,26 b' beaker.cache.short_term.expire=60' | |||
|
52 | 52 | beaker.cache.super_short_term.type=memory |
|
53 | 53 | beaker.cache.super_short_term.expire=10 |
|
54 | 54 | |
|
55 | #################################### | |
|
56 | ### BEAKER SESSION #### | |
|
57 | #################################### | |
|
58 | ## Type of storage used for the session, current types are | |
|
59 | ## βdbmβ, βfileβ, βmemcachedβ, βdatabaseβ, and βmemoryβ. | |
|
60 | ## The storage uses the Container API | |
|
61 | ##that is also used by the cache system. | |
|
62 | beaker.session.type = file | |
|
63 | ||
|
64 | beaker.session.key = hg-app | |
|
65 | beaker.session.secret = g654dcno0-9873jhgfreyu | |
|
66 | beaker.session.timeout = 36000 | |
|
67 | ||
|
68 | ##auto save the session to not to use .save() | |
|
69 | beaker.session.auto = False | |
|
70 | ||
|
71 | ##true exire at browser close | |
|
72 | #beaker.session.cookie_expires = 3600 | |
|
73 | ||
|
74 | ||
|
55 | 75 | ################################################################################ |
|
56 | 76 | ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT* ## |
|
57 | 77 | ## Debug mode will enable the interactive debugging tool, allowing ANYONE to ## |
@@ -52,6 +52,26 b' beaker.cache.short_term.expire=60' | |||
|
52 | 52 | beaker.cache.super_short_term.type=memory |
|
53 | 53 | beaker.cache.super_short_term.expire=10 |
|
54 | 54 | |
|
55 | #################################### | |
|
56 | ### BEAKER SESSION #### | |
|
57 | #################################### | |
|
58 | ## Type of storage used for the session, current types are | |
|
59 | ## βdbmβ, βfileβ, βmemcachedβ, βdatabaseβ, and βmemoryβ. | |
|
60 | ## The storage uses the Container API | |
|
61 | ##that is also used by the cache system. | |
|
62 | beaker.session.type = file | |
|
63 | ||
|
64 | beaker.session.key = hg-app | |
|
65 | beaker.session.secret = g654dcno0-9873jhgfreyu | |
|
66 | beaker.session.timeout = 36000 | |
|
67 | ||
|
68 | ##auto save the session to not to use .save() | |
|
69 | beaker.session.auto = False | |
|
70 | ||
|
71 | ##true exire at browser close | |
|
72 | #beaker.session.cookie_expires = 3600 | |
|
73 | ||
|
74 | ||
|
55 | 75 | ################################################################################ |
|
56 | 76 | ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT* ## |
|
57 | 77 | ## Debug mode will enable the interactive debugging tool, allowing ANYONE to ## |
@@ -19,18 +19,23 b' from shutil import rmtree' | |||
|
19 | 19 | #LOCATION WE KEEP THE INDEX |
|
20 | 20 | IDX_LOCATION = jn(dn(dn(dn(dn(os.path.abspath(__file__))))), 'data', 'index') |
|
21 | 21 | |
|
22 |
#EXTENSION |
|
|
23 | EXCLUDE_EXTENSIONS = ['pyc', 'mo', 'png', 'jpg', 'jpeg', 'gif', 'swf', | |
|
24 | 'dll', 'ttf', 'psd', 'svg', 'pdf', 'bmp', 'dll'] | |
|
22 | #EXTENSIONS WE WANT TO INDEX CONTENT OFF | |
|
23 | INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c', | |
|
24 | 'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl', 'h', | |
|
25 | 'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp', | |
|
26 | 'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3', | |
|
27 | 'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql', | |
|
28 | 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml','xsl','xslt', | |
|
29 | 'yaws'] | |
|
25 | 30 | |
|
26 | 31 | #CUSTOM ANALYZER wordsplit + lowercase filter |
|
27 | ANALYZER = RegexTokenizer() | LowercaseFilter() | |
|
32 | ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter() | |
|
28 | 33 | |
|
29 | 34 | #INDEX SCHEMA DEFINITION |
|
30 | 35 | SCHEMA = Schema(owner=TEXT(), |
|
31 | 36 | repository=TEXT(stored=True), |
|
32 | 37 | path=ID(stored=True, unique=True), |
|
33 | 38 | content=TEXT(stored=True, analyzer=ANALYZER), |
|
34 | modtime=STORED()) | |
|
39 | modtime=STORED(),extension=TEXT(stored=True)) | |
|
35 | 40 | |
|
36 |
IDX_NAME = 'HG_INDEX' |
|
|
41 | IDX_NAME = 'HG_INDEX' No newline at end of file |
@@ -38,7 +38,7 b' from pylons_app.config.environment impor' | |||
|
38 | 38 | from pylons_app.model.hg_model import HgModel |
|
39 | 39 | from whoosh.index import create_in, open_dir |
|
40 | 40 | from shutil import rmtree |
|
41 |
from pylons_app.lib.indexers import ANALYZER, |
|
|
41 | from pylons_app.lib.indexers import ANALYZER, INDEX_EXTENSIONS, IDX_LOCATION, \ | |
|
42 | 42 | SCHEMA, IDX_NAME |
|
43 | 43 | |
|
44 | 44 | import logging |
@@ -70,8 +70,10 b' class WhooshIndexingDaemon(object):' | |||
|
70 | 70 | def add_doc(self, writer, path, repo): |
|
71 | 71 | """Adding doc to writer""" |
|
72 | 72 | |
|
73 | #we don't won't to read excluded file extensions just index them | |
|
74 | if path.split('/')[-1].split('.')[-1].lower() not in EXCLUDE_EXTENSIONS: | |
|
73 | ext = unicode(path.split('/')[-1].split('.')[-1].lower()) | |
|
74 | #we just index the content of choosen files | |
|
75 | if ext in INDEX_EXTENSIONS: | |
|
76 | log.debug(' >> %s [WITH CONTENT]' % path) | |
|
75 | 77 | fobj = open(path, 'rb') |
|
76 | 78 | content = fobj.read() |
|
77 | 79 | fobj.close() |
@@ -81,15 +83,20 b' class WhooshIndexingDaemon(object):' | |||
|
81 | 83 | #incase we have a decode error just represent as byte string |
|
82 | 84 | u_content = unicode(str(content).encode('string_escape')) |
|
83 | 85 | else: |
|
86 | log.debug(' >> %s' % path) | |
|
87 | #just index file name without it's content | |
|
84 | 88 |
u_content = u'' |
|
89 | ||
|
85 | 90 | writer.add_document(owner=unicode(repo.contact), |
|
86 | 91 | repository=u"%s" % repo.name, |
|
87 | 92 | path=u"%s" % path, |
|
88 | 93 | content=u_content, |
|
89 |
modtime=os.path.getmtime(path) |
|
|
94 | modtime=os.path.getmtime(path), | |
|
95 | extension=ext) | |
|
90 | 96 | |
|
91 | 97 | def build_index(self): |
|
92 | 98 | if os.path.exists(IDX_LOCATION): |
|
99 | log.debug('removing previos index') | |
|
93 | 100 | rmtree(IDX_LOCATION) |
|
94 | 101 | |
|
95 | 102 | if not os.path.exists(IDX_LOCATION): |
@@ -102,7 +109,6 b' class WhooshIndexingDaemon(object):' | |||
|
102 | 109 | log.debug('building index @ %s' % repo.path) |
|
103 | 110 | |
|
104 | 111 | for idx_path in self.get_paths(repo.path): |
|
105 | log.debug(' >> %s' % idx_path) | |
|
106 | 112 | self.add_doc(writer, idx_path, repo) |
|
107 | 113 | writer.commit(merge=True) |
|
108 | 114 | |
@@ -170,11 +176,12 b' class WhooshIndexingDaemon(object):' | |||
|
170 | 176 | self.update_index() |
|
171 | 177 | |
|
172 | 178 | if __name__ == "__main__": |
|
173 |
repo_location = '/home/marcink/ |
|
|
174 | ||
|
179 | repo_location = '/home/marcink/hg_repos/*' | |
|
180 | full_index = True # False means looking just for changes | |
|
175 | 181 | try: |
|
176 | 182 | l = DaemonLock() |
|
177 |
WhooshIndexingDaemon(repo_location=repo_location) |
|
|
183 | WhooshIndexingDaemon(repo_location=repo_location)\ | |
|
184 | .run(full_index=full_index) | |
|
178 | 185 | l.release() |
|
179 | 186 | except LockHeld: |
|
180 | 187 | sys.exit(1) |
General Comments 0
You need to be logged in to leave comments.
Login now