# HG changeset patch # User Marcin Kuzminski # Date 2010-09-17 20:54:30 # Node ID a9e50dce30815de072e7d66481cc3e68bf02c402 # Parent 7afbc45aab284bee526eb814cf2bf253a98915dd Removed config names from whoosh and celery, celery is now configured based on the config name it's using on celeryconfig. And whoosh uses it's own logger configured just for whoosh Test creates a fresh whoosh index now, for more accurate checks fixed tests for searching diff --git a/celeryconfig.py b/celeryconfig.py --- a/celeryconfig.py +++ b/celeryconfig.py @@ -1,8 +1,20 @@ # List of modules to import when celery starts. import sys import os +import ConfigParser + +PYLONS_CONFIG_NAME = 'test.ini' + +root = os.getcwd() +config = ConfigParser.ConfigParser({'here':root}) +config.read('%s/%s' % (root, PYLONS_CONFIG_NAME)) +PYLONS_CONFIG = config + + +print config.items('app:main') + sys.path.append(os.getcwd()) -CELERY_IMPORTS = ("pylons_app.lib.celerylib.tasks", ) +CELERY_IMPORTS = ("pylons_app.lib.celerylib.tasks",) ## Result store settings. CELERY_RESULT_BACKEND = "database" @@ -30,4 +42,4 @@ CELERYD_MAX_TASKS_PER_CHILD = 1 #CELERY_ALWAYS_EAGER = True #rabbitmqctl add_user rabbitmq qweqwe #rabbitmqctl add_vhost rabbitmqhost -#rabbitmqctl set_permissions -p rabbitmqhost rabbitmq ".*" ".*" ".*" \ No newline at end of file +#rabbitmqctl set_permissions -p rabbitmqhost rabbitmq ".*" ".*" ".*" diff --git a/pylons_app/lib/celerylib/tasks.py b/pylons_app/lib/celerylib/tasks.py --- a/pylons_app/lib/celerylib/tasks.py +++ b/pylons_app/lib/celerylib/tasks.py @@ -1,7 +1,7 @@ from celery.decorators import task from celery.task.sets import subtask +from celeryconfig import PYLONS_CONFIG as config from datetime import datetime, timedelta -from os.path import dirname as dn from pylons.i18n.translation import _ from pylons_app.lib.celerylib import run_task from pylons_app.lib.helpers import person @@ -9,16 +9,9 @@ from pylons_app.lib.smtp_mailer import S from pylons_app.lib.utils import OrderedDict from time import mktime from vcs.backends.hg import MercurialRepository -import ConfigParser import calendar -import os import traceback - -root = dn(dn(dn(dn(os.path.realpath(__file__))))) -config = ConfigParser.ConfigParser({'here':root}) -config.read('%s/development.ini' % root) - __all__ = ['whoosh_index', 'get_commits_stats', 'reset_user_password', 'send_email'] @@ -91,7 +84,7 @@ def whoosh_index(repo_location, full_ind def get_commits_stats(repo): log = get_commits_stats.get_logger() aggregate = OrderedDict() - repos_path = get_hg_ui_settings()['paths_root_path'].replace('*','') + repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '') repo = MercurialRepository(repos_path + repo) #graph range td = datetime.today() + timedelta(days=1) @@ -205,7 +198,7 @@ def send_email(recipients, subject, body ssl = False try: - m = SmtpMailer(mail_from, user, passwd, mail_server, + m = SmtpMailer(mail_from, user, passwd, mail_server, mail_port, ssl, tls) m.send(recipients, subject, body) except: diff --git a/pylons_app/lib/indexers/daemon.py b/pylons_app/lib/indexers/daemon.py --- a/pylons_app/lib/indexers/daemon.py +++ b/pylons_app/lib/indexers/daemon.py @@ -33,19 +33,30 @@ project_path = dn(dn(dn(dn(os.path.realp sys.path.append(project_path) from pidlock import LockHeld, DaemonLock -import traceback -from pylons_app.config.environment import load_environment from pylons_app.model.hg_model import HgModel from pylons_app.lib.helpers import safe_unicode from whoosh.index import create_in, open_dir from shutil import rmtree -from pylons_app.lib.indexers import ANALYZER, INDEX_EXTENSIONS, IDX_LOCATION, \ -SCHEMA, IDX_NAME +from pylons_app.lib.indexers import INDEX_EXTENSIONS, IDX_LOCATION, SCHEMA, IDX_NAME import logging -import logging.config -logging.config.fileConfig(jn(project_path, 'development.ini')) + log = logging.getLogger('whooshIndexer') +# create logger +log.setLevel(logging.DEBUG) + +# create console handler and set level to debug +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) + +# create formatter +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + +# add formatter to ch +ch.setFormatter(formatter) + +# add ch to logger +log.addHandler(ch) def scan_paths(root_location): return HgModel.repo_scan('/', root_location, None, True) @@ -221,6 +232,7 @@ if __name__ == "__main__": WhooshIndexingDaemon(repo_location=repo_location)\ .run(full_index=full_index) l.release() + reload(logging) except LockHeld: sys.exit(1) diff --git a/pylons_app/tests/__init__.py b/pylons_app/tests/__init__.py --- a/pylons_app/tests/__init__.py +++ b/pylons_app/tests/__init__.py @@ -16,7 +16,9 @@ from routes.util import URLGenerator from webtest import TestApp import os from pylons_app.model import meta +from pylons_app.lib.indexers import IDX_LOCATION import logging +import shutil log = logging.getLogger(__name__) import pylons.test @@ -25,6 +27,23 @@ import pylons.test # Invoke websetup with the current config file #SetupCommand('setup-app').run([pylons.test.pylonsapp.config['__file__']]) +def create_index(repo_location, full_index): + from pylons_app.lib.indexers import daemon + from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon + from pylons_app.lib.indexers.pidlock import DaemonLock, LockHeld + + try: + l = DaemonLock() + WhooshIndexingDaemon(repo_location=repo_location)\ + .run(full_index=full_index) + l.release() + except LockHeld: + pass + +if os.path.exists(IDX_LOCATION): + shutil.rmtree(IDX_LOCATION) + +create_index('/tmp/*', True) environ = {} @@ -36,6 +55,7 @@ class TestController(TestCase): self.app = TestApp(wsgiapp) url._push_object(URLGenerator(config['routes.map'], environ)) self.sa = meta.Session + TestCase.__init__(self, *args, **kwargs) @@ -46,4 +66,5 @@ class TestController(TestCase): assert response.status == '302 Found', 'Wrong response code from login got %s' % response.status assert response.session['hg_app_user'].username == 'test_admin', 'wrong logged in user' return response.follow() - \ No newline at end of file + + diff --git a/pylons_app/tests/functional/test_login.py b/pylons_app/tests/functional/test_login.py --- a/pylons_app/tests/functional/test_login.py +++ b/pylons_app/tests/functional/test_login.py @@ -84,7 +84,7 @@ class TestLoginController(TestController def test_register_ok(self): username = 'test_regular4' password = 'qweqwe' - email = 'marcin@somemail.com' + email = 'marcin@test.com' name = 'testname' lastname = 'testlastname' @@ -100,7 +100,7 @@ class TestLoginController(TestController ret = self.sa.query(User).filter(User.username == 'test_regular4').one() assert ret.username == username , 'field mismatch %s %s' % (ret.username, username) - assert check_password(password,ret.password) == True , 'password mismatch' + assert check_password(password, ret.password) == True , 'password mismatch' assert ret.email == email , 'field mismatch %s %s' % (ret.email, email) assert ret.name == name , 'field mismatch %s %s' % (ret.name, name) assert ret.lastname == lastname , 'field mismatch %s %s' % (ret.lastname, lastname) @@ -108,9 +108,9 @@ class TestLoginController(TestController def test_forgot_password_wrong_mail(self): response = self.app.post(url(controller='login', action='password_reset'), - {'email':'marcin@wrongmail.org',}) + {'email':'marcin@wrongmail.org', }) - assert "That e-mail address doesn't exist" in response.body,'Missing error message about wrong email' + assert "That e-mail address doesn't exist" in response.body, 'Missing error message about wrong email' def test_forgot_password(self): response = self.app.get(url(controller='login', action='password_reset')) @@ -130,7 +130,7 @@ class TestLoginController(TestController 'lastname':lastname}) #register new user for email test response = self.app.post(url(controller='login', action='password_reset'), - {'email':email,}) + {'email':email, }) print response.session['flash'] assert 'You have successfully registered into hg-app' in response.session['flash'][0], 'No flash message about user registration' assert 'Your new password was sent' in response.session['flash'][1], 'No flash message about password reset' diff --git a/pylons_app/tests/functional/test_search.py b/pylons_app/tests/functional/test_search.py --- a/pylons_app/tests/functional/test_search.py +++ b/pylons_app/tests/functional/test_search.py @@ -9,7 +9,7 @@ class TestSearchController(TestControlle self.log_user() response = self.app.get(url(controller='search', action='index')) print response.body - assert 'class="small" id="q" name="q" type="text"' in response.body,'Search box content error' + assert 'class="small" id="q" name="q" type="text"' in response.body, 'Search box content error' # Test response... def test_empty_search(self): @@ -18,12 +18,21 @@ class TestSearchController(TestControlle raise SkipTest('skipped due to existing index') else: self.log_user() - response = self.app.get(url(controller='search', action='index'),{'q':'vcs_test'}) - assert 'There is no index to search in. Please run whoosh indexer' in response.body,'No error message about empty index' + response = self.app.get(url(controller='search', action='index'), {'q':'vcs_test'}) + assert 'There is no index to search in. Please run whoosh indexer' in response.body, 'No error message about empty index' def test_normal_search(self): self.log_user() - response = self.app.get(url(controller='search', action='index'),{'q':'def+repo'}) + response = self.app.get(url(controller='search', action='index'), {'q':'def repo'}) print response.body - assert '9 results' in response.body,'no message about proper search results' + assert '10 results' in response.body, 'no message about proper search results' + assert 'Permission denied' not in response.body, 'Wrong permissions settings for that repo and user' + + def test_repo_search(self): + self.log_user() + response = self.app.get(url(controller='search', action='index'), {'q':'repository:vcs_test def test'}) + print response.body + assert '4 results' in response.body, 'no message about proper search results' + assert 'Permission denied' not in response.body, 'Wrong permissions settings for that repo and user' +