##// END OF EJS Templates
updated config files,...
marcink -
r436:28f19fa5 default
parent child Browse files
Show More
@@ -1,135 +1,155 b''
1 ################################################################################
1 ################################################################################
2 ################################################################################
2 ################################################################################
3 # pylons_app - Pylons environment configuration #
3 # pylons_app - Pylons environment configuration #
4 # #
4 # #
5 # The %(here)s variable will be replaced with the parent directory of this file#
5 # The %(here)s variable will be replaced with the parent directory of this file#
6 ################################################################################
6 ################################################################################
7
7
8 [DEFAULT]
8 [DEFAULT]
9 debug = true
9 debug = true
10 ############################################
10 ############################################
11 ## Uncomment and replace with the address ##
11 ## Uncomment and replace with the address ##
12 ## which should receive any error reports ##
12 ## which should receive any error reports ##
13 ############################################
13 ############################################
14 #email_to = admin@localhost
14 #email_to = admin@localhost
15 #smtp_server = mail.server.com
15 #smtp_server = mail.server.com
16 #error_email_from = paste_error@localhost
16 #error_email_from = paste_error@localhost
17 #smtp_username =
17 #smtp_username =
18 #smtp_password =
18 #smtp_password =
19 #error_message = 'mercurial crash !'
19 #error_message = 'mercurial crash !'
20
20
21 [server:main]
21 [server:main]
22 ##nr of threads to spawn
22 ##nr of threads to spawn
23 threadpool_workers = 5
23 threadpool_workers = 5
24
24
25 ##max request before
25 ##max request before
26 threadpool_max_requests = 2
26 threadpool_max_requests = 2
27
27
28 ##option to use threads of process
28 ##option to use threads of process
29 use_threadpool = true
29 use_threadpool = true
30
30
31 use = egg:Paste#http
31 use = egg:Paste#http
32 host = 127.0.0.1
32 host = 127.0.0.1
33 port = 5000
33 port = 5000
34
34
35 [app:main]
35 [app:main]
36 use = egg:pylons_app
36 use = egg:pylons_app
37 full_stack = true
37 full_stack = true
38 static_files = true
38 static_files = true
39 lang=en
39 lang=en
40 cache_dir = %(here)s/data
40 cache_dir = %(here)s/data
41
41
42 ####################################
42 ####################################
43 ### BEAKER CACHE ####
43 ### BEAKER CACHE ####
44 ####################################
44 ####################################
45 beaker.cache.data_dir=/%(here)s/data/cache/data
45 beaker.cache.data_dir=/%(here)s/data/cache/data
46 beaker.cache.lock_dir=/%(here)s/data/cache/lock
46 beaker.cache.lock_dir=/%(here)s/data/cache/lock
47 beaker.cache.regions=super_short_term,short_term,long_term
47 beaker.cache.regions=super_short_term,short_term,long_term
48 beaker.cache.long_term.type=memory
48 beaker.cache.long_term.type=memory
49 beaker.cache.long_term.expire=36000
49 beaker.cache.long_term.expire=36000
50 beaker.cache.short_term.type=memory
50 beaker.cache.short_term.type=memory
51 beaker.cache.short_term.expire=60
51 beaker.cache.short_term.expire=60
52 beaker.cache.super_short_term.type=memory
52 beaker.cache.super_short_term.type=memory
53 beaker.cache.super_short_term.expire=10
53 beaker.cache.super_short_term.expire=10
54
54
55 ####################################
56 ### BEAKER SESSION ####
57 ####################################
58 ## Type of storage used for the session, current types are
59 ## β€œdbm”, β€œfile”, β€œmemcached”, β€œdatabase”, and β€œmemory”.
60 ## The storage uses the Container API
61 ##that is also used by the cache system.
62 beaker.session.type = file
63
64 beaker.session.key = hg-app
65 beaker.session.secret = g654dcno0-9873jhgfreyu
66 beaker.session.timeout = 36000
67
68 ##auto save the session to not to use .save()
69 beaker.session.auto = False
70
71 ##true exire at browser close
72 #beaker.session.cookie_expires = 3600
73
74
55 ################################################################################
75 ################################################################################
56 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT* ##
76 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT* ##
57 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to ##
77 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to ##
58 ## execute malicious code after an exception is raised. ##
78 ## execute malicious code after an exception is raised. ##
59 ################################################################################
79 ################################################################################
60 #set debug = false
80 #set debug = false
61
81
62 ##################################
82 ##################################
63 ### LOGVIEW CONFIG ###
83 ### LOGVIEW CONFIG ###
64 ##################################
84 ##################################
65 logview.sqlalchemy = #faa
85 logview.sqlalchemy = #faa
66 logview.pylons.templating = #bfb
86 logview.pylons.templating = #bfb
67 logview.pylons.util = #eee
87 logview.pylons.util = #eee
68
88
69 #########################################################
89 #########################################################
70 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG ###
90 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG ###
71 #########################################################
91 #########################################################
72 sqlalchemy.db1.url = sqlite:///%(here)s/hg_app.db
92 sqlalchemy.db1.url = sqlite:///%(here)s/hg_app.db
73 #sqlalchemy.db1.echo = False
93 #sqlalchemy.db1.echo = False
74 #sqlalchemy.db1.pool_recycle = 3600
94 #sqlalchemy.db1.pool_recycle = 3600
75 sqlalchemy.convert_unicode = true
95 sqlalchemy.convert_unicode = true
76
96
77 ################################
97 ################################
78 ### LOGGING CONFIGURATION ####
98 ### LOGGING CONFIGURATION ####
79 ################################
99 ################################
80 [loggers]
100 [loggers]
81 keys = root, routes, pylons_app, sqlalchemy
101 keys = root, routes, pylons_app, sqlalchemy
82
102
83 [handlers]
103 [handlers]
84 keys = console
104 keys = console
85
105
86 [formatters]
106 [formatters]
87 keys = generic,color_formatter
107 keys = generic,color_formatter
88
108
89 #############
109 #############
90 ## LOGGERS ##
110 ## LOGGERS ##
91 #############
111 #############
92 [logger_root]
112 [logger_root]
93 level = NOTSET
113 level = NOTSET
94 handlers = console
114 handlers = console
95
115
96 [logger_routes]
116 [logger_routes]
97 level = DEBUG
117 level = DEBUG
98 handlers = console
118 handlers = console
99 qualname = routes.middleware
119 qualname = routes.middleware
100 # "level = DEBUG" logs the route matched and routing variables.
120 # "level = DEBUG" logs the route matched and routing variables.
101
121
102 [logger_pylons_app]
122 [logger_pylons_app]
103 level = DEBUG
123 level = DEBUG
104 handlers = console
124 handlers = console
105 qualname = pylons_app
125 qualname = pylons_app
106 propagate = 0
126 propagate = 0
107
127
108 [logger_sqlalchemy]
128 [logger_sqlalchemy]
109 level = ERROR
129 level = ERROR
110 handlers = console
130 handlers = console
111 qualname = sqlalchemy.engine
131 qualname = sqlalchemy.engine
112 propagate = 0
132 propagate = 0
113
133
114 ##############
134 ##############
115 ## HANDLERS ##
135 ## HANDLERS ##
116 ##############
136 ##############
117
137
118 [handler_console]
138 [handler_console]
119 class = StreamHandler
139 class = StreamHandler
120 args = (sys.stderr,)
140 args = (sys.stderr,)
121 level = NOTSET
141 level = NOTSET
122 formatter = color_formatter
142 formatter = color_formatter
123
143
124 ################
144 ################
125 ## FORMATTERS ##
145 ## FORMATTERS ##
126 ################
146 ################
127
147
128 [formatter_generic]
148 [formatter_generic]
129 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
149 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
130 datefmt = %Y-%m-%d %H:%M:%S
150 datefmt = %Y-%m-%d %H:%M:%S
131
151
132 [formatter_color_formatter]
152 [formatter_color_formatter]
133 class=pylons_app.lib.colored_formatter.ColorFormatter
153 class=pylons_app.lib.colored_formatter.ColorFormatter
134 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
154 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
135 datefmt = %Y-%m-%d %H:%M:%S No newline at end of file
155 datefmt = %Y-%m-%d %H:%M:%S
@@ -1,135 +1,155 b''
1 ################################################################################
1 ################################################################################
2 ################################################################################
2 ################################################################################
3 # pylons_app - Pylons environment configuration #
3 # pylons_app - Pylons environment configuration #
4 # #
4 # #
5 # The %(here)s variable will be replaced with the parent directory of this file#
5 # The %(here)s variable will be replaced with the parent directory of this file#
6 ################################################################################
6 ################################################################################
7
7
8 [DEFAULT]
8 [DEFAULT]
9 debug = true
9 debug = true
10 ############################################
10 ############################################
11 ## Uncomment and replace with the address ##
11 ## Uncomment and replace with the address ##
12 ## which should receive any error reports ##
12 ## which should receive any error reports ##
13 ############################################
13 ############################################
14 #email_to = admin@localhost
14 #email_to = admin@localhost
15 #smtp_server = mail.server.com
15 #smtp_server = mail.server.com
16 #error_email_from = paste_error@localhost
16 #error_email_from = paste_error@localhost
17 #smtp_username =
17 #smtp_username =
18 #smtp_password =
18 #smtp_password =
19 #error_message = 'mercurial crash !'
19 #error_message = 'mercurial crash !'
20
20
21 [server:main]
21 [server:main]
22 ##nr of threads to spawn
22 ##nr of threads to spawn
23 threadpool_workers = 5
23 threadpool_workers = 5
24
24
25 ##max request before
25 ##max request before
26 threadpool_max_requests = 2
26 threadpool_max_requests = 2
27
27
28 ##option to use threads of process
28 ##option to use threads of process
29 use_threadpool = true
29 use_threadpool = true
30
30
31 use = egg:Paste#http
31 use = egg:Paste#http
32 host = 127.0.0.1
32 host = 127.0.0.1
33 port = 8001
33 port = 8001
34
34
35 [app:main]
35 [app:main]
36 use = egg:pylons_app
36 use = egg:pylons_app
37 full_stack = true
37 full_stack = true
38 static_files = false
38 static_files = false
39 lang=en
39 lang=en
40 cache_dir = %(here)s/data
40 cache_dir = %(here)s/data
41
41
42 ####################################
42 ####################################
43 ### BEAKER CACHE ####
43 ### BEAKER CACHE ####
44 ####################################
44 ####################################
45 beaker.cache.data_dir=/%(here)s/data/cache/data
45 beaker.cache.data_dir=/%(here)s/data/cache/data
46 beaker.cache.lock_dir=/%(here)s/data/cache/lock
46 beaker.cache.lock_dir=/%(here)s/data/cache/lock
47 beaker.cache.regions=super_short_term,short_term,long_term
47 beaker.cache.regions=super_short_term,short_term,long_term
48 beaker.cache.long_term.type=memory
48 beaker.cache.long_term.type=memory
49 beaker.cache.long_term.expire=36000
49 beaker.cache.long_term.expire=36000
50 beaker.cache.short_term.type=memory
50 beaker.cache.short_term.type=memory
51 beaker.cache.short_term.expire=60
51 beaker.cache.short_term.expire=60
52 beaker.cache.super_short_term.type=memory
52 beaker.cache.super_short_term.type=memory
53 beaker.cache.super_short_term.expire=10
53 beaker.cache.super_short_term.expire=10
54
55 ####################################
56 ### BEAKER SESSION ####
57 ####################################
58 ## Type of storage used for the session, current types are
59 ## β€œdbm”, β€œfile”, β€œmemcached”, β€œdatabase”, and β€œmemory”.
60 ## The storage uses the Container API
61 ##that is also used by the cache system.
62 beaker.session.type = file
63
64 beaker.session.key = hg-app
65 beaker.session.secret = g654dcno0-9873jhgfreyu
66 beaker.session.timeout = 36000
67
68 ##auto save the session to not to use .save()
69 beaker.session.auto = False
70
71 ##true exire at browser close
72 #beaker.session.cookie_expires = 3600
73
54
74
55 ################################################################################
75 ################################################################################
56 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT* ##
76 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT* ##
57 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to ##
77 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to ##
58 ## execute malicious code after an exception is raised. ##
78 ## execute malicious code after an exception is raised. ##
59 ################################################################################
79 ################################################################################
60 set debug = false
80 set debug = false
61
81
62 ##################################
82 ##################################
63 ### LOGVIEW CONFIG ###
83 ### LOGVIEW CONFIG ###
64 ##################################
84 ##################################
65 logview.sqlalchemy = #faa
85 logview.sqlalchemy = #faa
66 logview.pylons.templating = #bfb
86 logview.pylons.templating = #bfb
67 logview.pylons.util = #eee
87 logview.pylons.util = #eee
68
88
69 #########################################################
89 #########################################################
70 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG ###
90 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG ###
71 #########################################################
91 #########################################################
72 sqlalchemy.db1.url = sqlite:///%(here)s/hg_app.db
92 sqlalchemy.db1.url = sqlite:///%(here)s/hg_app.db
73 #sqlalchemy.db1.echo = False
93 #sqlalchemy.db1.echo = False
74 #sqlalchemy.db1.pool_recycle = 3600
94 #sqlalchemy.db1.pool_recycle = 3600
75 sqlalchemy.convert_unicode = true
95 sqlalchemy.convert_unicode = true
76
96
77 ################################
97 ################################
78 ### LOGGING CONFIGURATION ####
98 ### LOGGING CONFIGURATION ####
79 ################################
99 ################################
80 [loggers]
100 [loggers]
81 keys = root, routes, pylons_app, sqlalchemy
101 keys = root, routes, pylons_app, sqlalchemy
82
102
83 [handlers]
103 [handlers]
84 keys = console
104 keys = console
85
105
86 [formatters]
106 [formatters]
87 keys = generic,color_formatter
107 keys = generic,color_formatter
88
108
89 #############
109 #############
90 ## LOGGERS ##
110 ## LOGGERS ##
91 #############
111 #############
92 [logger_root]
112 [logger_root]
93 level = INFO
113 level = INFO
94 handlers = console
114 handlers = console
95
115
96 [logger_routes]
116 [logger_routes]
97 level = INFO
117 level = INFO
98 handlers = console
118 handlers = console
99 qualname = routes.middleware
119 qualname = routes.middleware
100 # "level = DEBUG" logs the route matched and routing variables.
120 # "level = DEBUG" logs the route matched and routing variables.
101
121
102 [logger_pylons_app]
122 [logger_pylons_app]
103 level = DEBUG
123 level = DEBUG
104 handlers = console
124 handlers = console
105 qualname = pylons_app
125 qualname = pylons_app
106 propagate = 0
126 propagate = 0
107
127
108 [logger_sqlalchemy]
128 [logger_sqlalchemy]
109 level = ERROR
129 level = ERROR
110 handlers = console
130 handlers = console
111 qualname = sqlalchemy.engine
131 qualname = sqlalchemy.engine
112 propagate = 0
132 propagate = 0
113
133
114 ##############
134 ##############
115 ## HANDLERS ##
135 ## HANDLERS ##
116 ##############
136 ##############
117
137
118 [handler_console]
138 [handler_console]
119 class = StreamHandler
139 class = StreamHandler
120 args = (sys.stderr,)
140 args = (sys.stderr,)
121 level = NOTSET
141 level = NOTSET
122 formatter = color_formatter
142 formatter = color_formatter
123
143
124 ################
144 ################
125 ## FORMATTERS ##
145 ## FORMATTERS ##
126 ################
146 ################
127
147
128 [formatter_generic]
148 [formatter_generic]
129 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
149 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
130 datefmt = %Y-%m-%d %H:%M:%S
150 datefmt = %Y-%m-%d %H:%M:%S
131
151
132 [formatter_color_formatter]
152 [formatter_color_formatter]
133 class=pylons_app.lib.colored_formatter.ColorFormatter
153 class=pylons_app.lib.colored_formatter.ColorFormatter
134 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
154 format= %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
135 datefmt = %Y-%m-%d %H:%M:%S No newline at end of file
155 datefmt = %Y-%m-%d %H:%M:%S
@@ -1,36 +1,41 b''
1 import sys
1 import sys
2 import os
2 import os
3 from pidlock import LockHeld, DaemonLock
3 from pidlock import LockHeld, DaemonLock
4 import traceback
4 import traceback
5
5
6 from os.path import dirname as dn
6 from os.path import dirname as dn
7 from os.path import join as jn
7 from os.path import join as jn
8
8
9 #to get the pylons_app import
9 #to get the pylons_app import
10 sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
10 sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))
11
11
12 from pylons_app.config.environment import load_environment
12 from pylons_app.config.environment import load_environment
13 from pylons_app.model.hg_model import HgModel
13 from pylons_app.model.hg_model import HgModel
14 from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter
14 from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter
15 from whoosh.fields import TEXT, ID, STORED, Schema
15 from whoosh.fields import TEXT, ID, STORED, Schema
16 from whoosh.index import create_in, open_dir
16 from whoosh.index import create_in, open_dir
17 from shutil import rmtree
17 from shutil import rmtree
18
18
19 #LOCATION WE KEEP THE INDEX
19 #LOCATION WE KEEP THE INDEX
20 IDX_LOCATION = jn(dn(dn(dn(dn(os.path.abspath(__file__))))), 'data', 'index')
20 IDX_LOCATION = jn(dn(dn(dn(dn(os.path.abspath(__file__))))), 'data', 'index')
21
21
22 #EXTENSION TO SKIP READING CONTENT ON
22 #EXTENSIONS WE WANT TO INDEX CONTENT OFF
23 EXCLUDE_EXTENSIONS = ['pyc', 'mo', 'png', 'jpg', 'jpeg', 'gif', 'swf',
23 INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
24 'dll', 'ttf', 'psd', 'svg', 'pdf', 'bmp', 'dll']
24 'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl', 'h',
25 'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp',
26 'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3',
27 'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql',
28 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml','xsl','xslt',
29 'yaws']
25
30
26 #CUSTOM ANALYZER wordsplit + lowercase filter
31 #CUSTOM ANALYZER wordsplit + lowercase filter
27 ANALYZER = RegexTokenizer() | LowercaseFilter()
32 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
28
33
29 #INDEX SCHEMA DEFINITION
34 #INDEX SCHEMA DEFINITION
30 SCHEMA = Schema(owner=TEXT(),
35 SCHEMA = Schema(owner=TEXT(),
31 repository=TEXT(stored=True),
36 repository=TEXT(stored=True),
32 path=ID(stored=True, unique=True),
37 path=ID(stored=True, unique=True),
33 content=TEXT(stored=True, analyzer=ANALYZER),
38 content=TEXT(stored=True, analyzer=ANALYZER),
34 modtime=STORED())
39 modtime=STORED(),extension=TEXT(stored=True))
35
40
36 IDX_NAME = 'HG_INDEX'
41 IDX_NAME = 'HG_INDEX' No newline at end of file
@@ -1,181 +1,188 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 # whoosh indexer daemon for hg-app
3 # whoosh indexer daemon for hg-app
4 # Copyright (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com>
4 # Copyright (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com>
5 #
5 #
6 # This program is free software; you can redistribute it and/or
6 # This program is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU General Public License
7 # modify it under the terms of the GNU General Public License
8 # as published by the Free Software Foundation; version 2
8 # as published by the Free Software Foundation; version 2
9 # of the License or (at your opinion) any later version of the license.
9 # of the License or (at your opinion) any later version of the license.
10 #
10 #
11 # This program is distributed in the hope that it will be useful,
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
14 # GNU General Public License for more details.
15 #
15 #
16 # You should have received a copy of the GNU General Public License
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
19 # MA 02110-1301, USA.
19 # MA 02110-1301, USA.
20 """
20 """
21 Created on Jan 26, 2010
21 Created on Jan 26, 2010
22
22
23 @author: marcink
23 @author: marcink
24 A deamon will read from task table and run tasks
24 A deamon will read from task table and run tasks
25 """
25 """
26 import sys
26 import sys
27 import os
27 import os
28 from os.path import dirname as dn
28 from os.path import dirname as dn
29 from os.path import join as jn
29 from os.path import join as jn
30
30
31 #to get the pylons_app import
31 #to get the pylons_app import
32 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
32 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
33 sys.path.append(project_path)
33 sys.path.append(project_path)
34
34
35 from pidlock import LockHeld, DaemonLock
35 from pidlock import LockHeld, DaemonLock
36 import traceback
36 import traceback
37 from pylons_app.config.environment import load_environment
37 from pylons_app.config.environment import load_environment
38 from pylons_app.model.hg_model import HgModel
38 from pylons_app.model.hg_model import HgModel
39 from whoosh.index import create_in, open_dir
39 from whoosh.index import create_in, open_dir
40 from shutil import rmtree
40 from shutil import rmtree
41 from pylons_app.lib.indexers import ANALYZER, EXCLUDE_EXTENSIONS, IDX_LOCATION, \
41 from pylons_app.lib.indexers import ANALYZER, INDEX_EXTENSIONS, IDX_LOCATION, \
42 SCHEMA, IDX_NAME
42 SCHEMA, IDX_NAME
43
43
44 import logging
44 import logging
45 import logging.config
45 import logging.config
46 logging.config.fileConfig(jn(project_path, 'development.ini'))
46 logging.config.fileConfig(jn(project_path, 'development.ini'))
47 log = logging.getLogger('whooshIndexer')
47 log = logging.getLogger('whooshIndexer')
48
48
49 def scan_paths(root_location):
49 def scan_paths(root_location):
50 return HgModel.repo_scan('/', root_location, None, True)
50 return HgModel.repo_scan('/', root_location, None, True)
51
51
52 class WhooshIndexingDaemon(object):
52 class WhooshIndexingDaemon(object):
53 """Deamon for atomic jobs"""
53 """Deamon for atomic jobs"""
54
54
55 def __init__(self, indexname='HG_INDEX', repo_location=None):
55 def __init__(self, indexname='HG_INDEX', repo_location=None):
56 self.indexname = indexname
56 self.indexname = indexname
57 self.repo_location = repo_location
57 self.repo_location = repo_location
58
58
59 def get_paths(self, root_dir):
59 def get_paths(self, root_dir):
60 """recursive walk in root dir and return a set of all path in that dir
60 """recursive walk in root dir and return a set of all path in that dir
61 excluding files in .hg dir"""
61 excluding files in .hg dir"""
62 index_paths_ = set()
62 index_paths_ = set()
63 for path, dirs, files in os.walk(root_dir):
63 for path, dirs, files in os.walk(root_dir):
64 if path.find('.hg') == -1:
64 if path.find('.hg') == -1:
65 for f in files:
65 for f in files:
66 index_paths_.add(jn(path, f))
66 index_paths_.add(jn(path, f))
67
67
68 return index_paths_
68 return index_paths_
69
69
70 def add_doc(self, writer, path, repo):
70 def add_doc(self, writer, path, repo):
71 """Adding doc to writer"""
71 """Adding doc to writer"""
72
72
73 #we don't won't to read excluded file extensions just index them
73 ext = unicode(path.split('/')[-1].split('.')[-1].lower())
74 if path.split('/')[-1].split('.')[-1].lower() not in EXCLUDE_EXTENSIONS:
74 #we just index the content of choosen files
75 if ext in INDEX_EXTENSIONS:
76 log.debug(' >> %s [WITH CONTENT]' % path)
75 fobj = open(path, 'rb')
77 fobj = open(path, 'rb')
76 content = fobj.read()
78 content = fobj.read()
77 fobj.close()
79 fobj.close()
78 try:
80 try:
79 u_content = unicode(content)
81 u_content = unicode(content)
80 except UnicodeDecodeError:
82 except UnicodeDecodeError:
81 #incase we have a decode error just represent as byte string
83 #incase we have a decode error just represent as byte string
82 u_content = unicode(str(content).encode('string_escape'))
84 u_content = unicode(str(content).encode('string_escape'))
83 else:
85 else:
84 u_content = u''
86 log.debug(' >> %s' % path)
87 #just index file name without it's content
88 u_content = u''
89
85 writer.add_document(owner=unicode(repo.contact),
90 writer.add_document(owner=unicode(repo.contact),
86 repository=u"%s" % repo.name,
91 repository=u"%s" % repo.name,
87 path=u"%s" % path,
92 path=u"%s" % path,
88 content=u_content,
93 content=u_content,
89 modtime=os.path.getmtime(path))
94 modtime=os.path.getmtime(path),
95 extension=ext)
90
96
91 def build_index(self):
97 def build_index(self):
92 if os.path.exists(IDX_LOCATION):
98 if os.path.exists(IDX_LOCATION):
99 log.debug('removing previos index')
93 rmtree(IDX_LOCATION)
100 rmtree(IDX_LOCATION)
94
101
95 if not os.path.exists(IDX_LOCATION):
102 if not os.path.exists(IDX_LOCATION):
96 os.mkdir(IDX_LOCATION)
103 os.mkdir(IDX_LOCATION)
97
104
98 idx = create_in(IDX_LOCATION, SCHEMA, indexname=IDX_NAME)
105 idx = create_in(IDX_LOCATION, SCHEMA, indexname=IDX_NAME)
99 writer = idx.writer()
106 writer = idx.writer()
100
107
101 for cnt, repo in enumerate(scan_paths(self.repo_location).values()):
108 for cnt, repo in enumerate(scan_paths(self.repo_location).values()):
102 log.debug('building index @ %s' % repo.path)
109 log.debug('building index @ %s' % repo.path)
103
110
104 for idx_path in self.get_paths(repo.path):
111 for idx_path in self.get_paths(repo.path):
105 log.debug(' >> %s' % idx_path)
106 self.add_doc(writer, idx_path, repo)
112 self.add_doc(writer, idx_path, repo)
107 writer.commit(merge=True)
113 writer.commit(merge=True)
108
114
109 log.debug('>>> FINISHED BUILDING INDEX <<<')
115 log.debug('>>> FINISHED BUILDING INDEX <<<')
110
116
111
117
112 def update_index(self):
118 def update_index(self):
113 log.debug('STARTING INCREMENTAL INDEXING UPDATE')
119 log.debug('STARTING INCREMENTAL INDEXING UPDATE')
114
120
115 idx = open_dir(IDX_LOCATION, indexname=self.indexname)
121 idx = open_dir(IDX_LOCATION, indexname=self.indexname)
116 # The set of all paths in the index
122 # The set of all paths in the index
117 indexed_paths = set()
123 indexed_paths = set()
118 # The set of all paths we need to re-index
124 # The set of all paths we need to re-index
119 to_index = set()
125 to_index = set()
120
126
121 reader = idx.reader()
127 reader = idx.reader()
122 writer = idx.writer()
128 writer = idx.writer()
123
129
124 # Loop over the stored fields in the index
130 # Loop over the stored fields in the index
125 for fields in reader.all_stored_fields():
131 for fields in reader.all_stored_fields():
126 indexed_path = fields['path']
132 indexed_path = fields['path']
127 indexed_paths.add(indexed_path)
133 indexed_paths.add(indexed_path)
128
134
129 if not os.path.exists(indexed_path):
135 if not os.path.exists(indexed_path):
130 # This file was deleted since it was indexed
136 # This file was deleted since it was indexed
131 log.debug('removing from index %s' % indexed_path)
137 log.debug('removing from index %s' % indexed_path)
132 writer.delete_by_term('path', indexed_path)
138 writer.delete_by_term('path', indexed_path)
133
139
134 else:
140 else:
135 # Check if this file was changed since it
141 # Check if this file was changed since it
136 # was indexed
142 # was indexed
137 indexed_time = fields['modtime']
143 indexed_time = fields['modtime']
138
144
139 mtime = os.path.getmtime(indexed_path)
145 mtime = os.path.getmtime(indexed_path)
140
146
141 if mtime > indexed_time:
147 if mtime > indexed_time:
142
148
143 # The file has changed, delete it and add it to the list of
149 # The file has changed, delete it and add it to the list of
144 # files to reindex
150 # files to reindex
145 log.debug('adding to reindex list %s' % indexed_path)
151 log.debug('adding to reindex list %s' % indexed_path)
146 writer.delete_by_term('path', indexed_path)
152 writer.delete_by_term('path', indexed_path)
147 to_index.add(indexed_path)
153 to_index.add(indexed_path)
148 #writer.commit()
154 #writer.commit()
149
155
150 # Loop over the files in the filesystem
156 # Loop over the files in the filesystem
151 # Assume we have a function that gathers the filenames of the
157 # Assume we have a function that gathers the filenames of the
152 # documents to be indexed
158 # documents to be indexed
153 for repo in scan_paths(self.repo_location).values():
159 for repo in scan_paths(self.repo_location).values():
154 for path in self.get_paths(repo.path):
160 for path in self.get_paths(repo.path):
155 if path in to_index or path not in indexed_paths:
161 if path in to_index or path not in indexed_paths:
156 # This is either a file that's changed, or a new file
162 # This is either a file that's changed, or a new file
157 # that wasn't indexed before. So index it!
163 # that wasn't indexed before. So index it!
158 self.add_doc(writer, path, repo)
164 self.add_doc(writer, path, repo)
159 log.debug('reindexing %s' % path)
165 log.debug('reindexing %s' % path)
160
166
161 writer.commit(merge=True)
167 writer.commit(merge=True)
162 #idx.optimize()
168 #idx.optimize()
163 log.debug('>>> FINISHED <<<')
169 log.debug('>>> FINISHED <<<')
164
170
165 def run(self, full_index=False):
171 def run(self, full_index=False):
166 """Run daemon"""
172 """Run daemon"""
167 if full_index:
173 if full_index:
168 self.build_index()
174 self.build_index()
169 else:
175 else:
170 self.update_index()
176 self.update_index()
171
177
172 if __name__ == "__main__":
178 if __name__ == "__main__":
173 repo_location = '/home/marcink/python_workspace_dirty/*'
179 repo_location = '/home/marcink/hg_repos/*'
174
180 full_index = True # False means looking just for changes
175 try:
181 try:
176 l = DaemonLock()
182 l = DaemonLock()
177 WhooshIndexingDaemon(repo_location=repo_location).run(full_index=True)
183 WhooshIndexingDaemon(repo_location=repo_location)\
184 .run(full_index=full_index)
178 l.release()
185 l.release()
179 except LockHeld:
186 except LockHeld:
180 sys.exit(1)
187 sys.exit(1)
181
188
General Comments 0
You need to be logged in to leave comments. Login now