diff --git a/README.rst b/README.rst --- a/README.rst +++ b/README.rst @@ -11,9 +11,12 @@ Fully customizable, with authentication, - full permissions per project read/write/admin access even on mercurial request - mako templates let's you cusmotize look and feel of application. - diffs annotations and source code all colored by pygments. -- mercurial branch graph and yui-flot powered graphs +- mercurial branch graph and yui-flot powered graphs with zooming - admin interface for performing user/permission managments as well as repository managment. +- full text search of source codes with indexing daemons using whoosh + (no external search servers required all in one application) +- async tasks for speed and performance using celery (works without them too) - Additional settings for mercurial web, (hooks editable from admin panel !) also manage paths, archive, remote messages - backup scripts can do backup of whole app and send it over scp to desired location @@ -27,11 +30,11 @@ Fully customizable, with authentication, **Incoming** - code review based on hg-review (when it's stable) -- git support (when vcs can handle it) -- full text search of source codes with indexing daemons using whoosh - (no external search servers required all in one application) -- manage hg ui() per repo, add hooks settings, per repo, and not globally -- other cools stuff that i can figure out +- git support (when vcs can handle it - almost there !) +- commit based wikis +- in server forks +- clonning from remote repositories into hg-app +- other cools stuff that i can figure out (or You can help me figure out) .. note:: This software is still in beta mode. @@ -47,10 +50,10 @@ Installation - create new virtualenv and activate it - highly recommend that you use separate virtual-env for whole application -- download hg app from default (not demo) branch from bitbucket and run +- download hg app from default branch from bitbucket and run 'python setup.py install' this will install all required dependencies needed - run paster setup-app production.ini it should create all needed tables - and an admin account. + and an admin account make sure You specify correct path to repositories. - remember that the given path for mercurial repositories must be write accessible for the application - run paster serve development.ini - or you can use manage-hg_app script. @@ -58,4 +61,9 @@ Installation - use admin account you created to login. - default permissions on each repository is read, and owner is admin. So remember to update these. +- in order to use full power of async tasks, You must install message broker + preferrably rabbitmq and start celeryd daemon. The app should gain some speed + than. For installation instructions + You can visit: http://ask.github.com/celery/getting-started/index.html. All + needed configs are inside hg-app ie. celeryconfig.py \ No newline at end of file diff --git a/celeryconfig.py b/celeryconfig.py new file mode 100644 --- /dev/null +++ b/celeryconfig.py @@ -0,0 +1,74 @@ +# List of modules to import when celery starts. +import sys +import os +import ConfigParser +root = os.getcwd() + +PYLONS_CONFIG_NAME = 'development.ini' + +sys.path.append(root) +config = ConfigParser.ConfigParser({'here':root}) +config.read('%s/%s' % (root, PYLONS_CONFIG_NAME)) +PYLONS_CONFIG = config + +CELERY_IMPORTS = ("pylons_app.lib.celerylib.tasks",) + +## Result store settings. +CELERY_RESULT_BACKEND = "database" +CELERY_RESULT_DBURI = dict(config.items('app:main'))['sqlalchemy.db1.url'] +CELERY_RESULT_SERIALIZER = 'json' + + +BROKER_CONNECTION_MAX_RETRIES = 30 + +## Broker settings. +BROKER_HOST = "localhost" +BROKER_PORT = 5672 +BROKER_VHOST = "rabbitmqhost" +BROKER_USER = "rabbitmq" +BROKER_PASSWORD = "qweqwe" + +## Worker settings +## If you're doing mostly I/O you can have more processes, +## but if mostly spending CPU, try to keep it close to the +## number of CPUs on your machine. If not set, the number of CPUs/cores +## available will be used. +CELERYD_CONCURRENCY = 2 +# CELERYD_LOG_FILE = "celeryd.log" +CELERYD_LOG_LEVEL = "DEBUG" +CELERYD_MAX_TASKS_PER_CHILD = 1 + +#Tasks will never be sent to the queue, but executed locally instead. +CELERY_ALWAYS_EAGER = False + +#=============================================================================== +# EMAIL SETTINGS +#=============================================================================== +pylons_email_config = dict(config.items('DEFAULT')) + +CELERY_SEND_TASK_ERROR_EMAILS = True + +#List of (name, email_address) tuples for the admins that should receive error e-mails. +ADMINS = [('Administrator', pylons_email_config.get('email_to'))] + +#The e-mail address this worker sends e-mails from. Default is "celery@localhost". +SERVER_EMAIL = pylons_email_config.get('error_email_from') + +#The mail server to use. Default is "localhost". +MAIL_HOST = pylons_email_config.get('smtp_server') + +#Username (if required) to log on to the mail server with. +MAIL_HOST_USER = pylons_email_config.get('smtp_username') + +#Password (if required) to log on to the mail server with. +MAIL_HOST_PASSWORD = pylons_email_config.get('smtp_password') + +MAIL_PORT = pylons_email_config.get('smtp_port') + + +#=============================================================================== +# INSTRUCTIONS FOR RABBITMQ +#=============================================================================== +# rabbitmqctl add_user rabbitmq qweqwe +# rabbitmqctl add_vhost rabbitmqhost +# rabbitmqctl set_permissions -p rabbitmqhost rabbitmq ".*" ".*" ".*" diff --git a/development.ini b/development.ini --- a/development.ini +++ b/development.ini @@ -1,32 +1,37 @@ ################################################################################ ################################################################################ -# pylons_app - Pylons environment configuration # +# hg-app - Pylons environment configuration # # # # The %(here)s variable will be replaced with the parent directory of this file# ################################################################################ [DEFAULT] debug = true -############################################ -## Uncomment and replace with the address ## -## which should receive any error reports ## -############################################ +################################################################################ +## Uncomment and replace with the address which should receive ## +## any error reports after application crash ## +## Additionally those settings will be used by hg-app mailing system ## +################################################################################ #email_to = admin@localhost +#error_email_from = paste_error@localhost +#app_email_from = hg-app-noreply@localhost +#error_message = + #smtp_server = mail.server.com -#error_email_from = paste_error@localhost #smtp_username = -#smtp_password = -#error_message = 'mercurial crash !' +#smtp_password = +#smtp_port = +#smtp_use_tls = [server:main] ##nr of threads to spawn threadpool_workers = 5 ##max request before -threadpool_max_requests = 2 +threadpool_max_requests = 6 ##option to use threads of process -use_threadpool = true +use_threadpool = false use = egg:Paste#http host = 127.0.0.1 @@ -56,7 +61,7 @@ beaker.cache.super_short_term.expire=10 ### BEAKER SESSION #### #################################### ## Type of storage used for the session, current types are -## “dbm”, “file”, “memcached”, “database”, and “memory”. +## "dbm", "file", "memcached", "database", and "memory". ## The storage uses the Container API ##that is also used by the cache system. beaker.session.type = file diff --git a/production.ini b/production.ini --- a/production.ini +++ b/production.ini @@ -1,28 +1,33 @@ ################################################################################ ################################################################################ -# pylons_app - Pylons environment configuration # +# hg-app - Pylons environment configuration # # # # The %(here)s variable will be replaced with the parent directory of this file# ################################################################################ [DEFAULT] debug = true -############################################ -## Uncomment and replace with the address ## -## which should receive any error reports ## -############################################ +################################################################################ +## Uncomment and replace with the address which should receive ## +## any error reports after application crash ## +## Additionally those settings will be used by hg-app mailing system ## +################################################################################ #email_to = admin@localhost +#error_email_from = paste_error@localhost +#app_email_from = hg-app-noreply@localhost +#error_message = + #smtp_server = mail.server.com -#error_email_from = paste_error@localhost #smtp_username = #smtp_password = -#error_message = 'mercurial crash !' +#smtp_port = +#smtp_use_tls = false [server:main] ##nr of threads to spawn threadpool_workers = 5 -##max request before +##max request before thread respawn threadpool_max_requests = 2 ##option to use threads of process diff --git a/pylons_app/__init__.py b/pylons_app/__init__.py --- a/pylons_app/__init__.py +++ b/pylons_app/__init__.py @@ -20,10 +20,11 @@ """ Created on April 9, 2010 Hg app, a web based mercurial repository managment based on pylons +versioning implementation: http://semver.org/ @author: marcink """ -VERSION = (0, 8, 2, 'beta') +VERSION = (0, 8, 3, 'beta') __version__ = '.'.join((str(each) for each in VERSION[:4])) diff --git a/pylons_app/config/deployment.ini_tmpl b/pylons_app/config/deployment.ini_tmpl --- a/pylons_app/config/deployment.ini_tmpl +++ b/pylons_app/config/deployment.ini_tmpl @@ -7,16 +7,21 @@ [DEFAULT] debug = true -############################################ -## Uncomment and replace with the address ## -## which should receive any error reports ## -############################################ +################################################################################ +## Uncomment and replace with the address which should receive ## +## any error reports after application crash ## +## Additionally those settings will be used by hg-app mailing system ## +################################################################################ #email_to = admin@localhost +#error_email_from = paste_error@localhost +#app_email_from = hg-app-noreply@localhost +#error_message = + #smtp_server = mail.server.com -#error_email_from = paste_error@localhost #smtp_username = #smtp_password = -#error_message = 'hp-app crash !' +#smtp_port = +#smtp_use_tls = false [server:main] ##nr of threads to spawn diff --git a/pylons_app/config/environment.py b/pylons_app/config/environment.py --- a/pylons_app/config/environment.py +++ b/pylons_app/config/environment.py @@ -49,7 +49,12 @@ def load_environment(global_conf, app_co #sets the c attribute access when don't existing attribute are accessed config['pylons.strict_tmpl_context'] = True - test = os.path.split(config['__file__'])[-1] == 'tests.ini' + test = os.path.split(config['__file__'])[-1] == 'test.ini' + if test: + from pylons_app.lib.utils import create_test_env, create_test_index + create_test_env('/tmp', config) + create_test_index('/tmp/*', True) + #MULTIPLE DB configs # Setup the SQLAlchemy database engine if config['debug'] and not test: diff --git a/pylons_app/config/routing.py b/pylons_app/config/routing.py --- a/pylons_app/config/routing.py +++ b/pylons_app/config/routing.py @@ -110,10 +110,11 @@ def make_map(config): #SEARCH map.connect('search', '/_admin/search', controller='search') - #LOGIN/LOGOUT + #LOGIN/LOGOUT/REGISTER/SIGN IN map.connect('login_home', '/_admin/login', controller='login') map.connect('logout_home', '/_admin/logout', controller='login', action='logout') map.connect('register', '/_admin/register', controller='login', action='register') + map.connect('reset_password', '/_admin/password_reset', controller='login', action='password_reset') #FEEDS map.connect('rss_feed_home', '/{repo_name:.*}/feed/rss', @@ -129,7 +130,7 @@ def make_map(config): controller='changeset', revision='tip', conditions=dict(function=check_repo)) map.connect('raw_changeset_home', '/{repo_name:.*}/raw-changeset/{revision}', - controller='changeset',action='raw_changeset', revision='tip', + controller='changeset', action='raw_changeset', revision='tip', conditions=dict(function=check_repo)) map.connect('summary_home', '/{repo_name:.*}/summary', controller='summary', conditions=dict(function=check_repo)) @@ -147,9 +148,12 @@ def make_map(config): map.connect('files_diff_home', '/{repo_name:.*}/diff/{f_path:.*}', controller='files', action='diff', revision='tip', f_path='', conditions=dict(function=check_repo)) - map.connect('files_raw_home', '/{repo_name:.*}/rawfile/{revision}/{f_path:.*}', + map.connect('files_rawfile_home', '/{repo_name:.*}/rawfile/{revision}/{f_path:.*}', controller='files', action='rawfile', revision='tip', f_path='', conditions=dict(function=check_repo)) + map.connect('files_raw_home', '/{repo_name:.*}/raw/{revision}/{f_path:.*}', + controller='files', action='raw', revision='tip', f_path='', + conditions=dict(function=check_repo)) map.connect('files_annotate_home', '/{repo_name:.*}/annotate/{revision}/{f_path:.*}', controller='files', action='annotate', revision='tip', f_path='', conditions=dict(function=check_repo)) diff --git a/pylons_app/controllers/admin/settings.py b/pylons_app/controllers/admin/settings.py --- a/pylons_app/controllers/admin/settings.py +++ b/pylons_app/controllers/admin/settings.py @@ -38,6 +38,7 @@ from pylons_app.model.forms import UserF ApplicationUiSettingsForm from pylons_app.model.hg_model import HgModel from pylons_app.model.user_model import UserModel +from pylons_app.lib.celerylib import tasks, run_task import formencode import logging import traceback @@ -102,6 +103,12 @@ class SettingsController(BaseController) invalidate_cache('cached_repo_list') h.flash(_('Repositories sucessfully rescanned'), category='success') + if setting_id == 'whoosh': + repo_location = get_hg_ui_settings()['paths_root_path'] + full_index = request.POST.get('full_index', False) + task = run_task(tasks.whoosh_index, repo_location, full_index) + + h.flash(_('Whoosh reindex task scheduled'), category='success') if setting_id == 'global': application_form = ApplicationSettingsForm()() @@ -253,7 +260,8 @@ class SettingsController(BaseController) # url('admin_settings_my_account_update', id=ID) user_model = UserModel() uid = c.hg_app_user.user_id - _form = UserForm(edit=True, old_data={'user_id':uid})() + _form = UserForm(edit=True, old_data={'user_id':uid, + 'email':c.hg_app_user.email})() form_result = {} try: form_result = _form.to_python(dict(request.POST)) @@ -262,7 +270,11 @@ class SettingsController(BaseController) category='success') except formencode.Invalid as errors: - #c.user = self.sa.query(User).get(c.hg_app_user.user_id) + c.user = self.sa.query(User).get(c.hg_app_user.user_id) + c.user_repos = [] + for repo in c.cached_repo_list.values(): + if repo.dbrepo.user.username == c.user.username: + c.user_repos.append(repo) return htmlfill.render( render('admin/users/user_edit_my_account.html'), defaults=errors.value, diff --git a/pylons_app/controllers/admin/users.py b/pylons_app/controllers/admin/users.py --- a/pylons_app/controllers/admin/users.py +++ b/pylons_app/controllers/admin/users.py @@ -98,7 +98,10 @@ class UsersController(BaseController): # method='put') # url('user', id=ID) user_model = UserModel() - _form = UserForm(edit=True, old_data={'user_id':id})() + c.user = user_model.get_user(id) + + _form = UserForm(edit=True, old_data={'user_id':id, + 'email':c.user.email})() form_result = {} try: form_result = _form.to_python(dict(request.POST)) @@ -106,7 +109,6 @@ class UsersController(BaseController): h.flash(_('User updated succesfully'), category='success') except formencode.Invalid as errors: - c.user = user_model.get_user(id) return htmlfill.render( render('admin/users/user_edit.html'), defaults=errors.value, @@ -148,6 +150,8 @@ class UsersController(BaseController): """GET /users/id/edit: Form to edit an existing item""" # url('edit_user', id=ID) c.user = self.sa.query(User).get(id) + if not c.user: + return redirect(url('users')) if c.user.username == 'default': h.flash(_("You can't edit this user since it's" " crucial for entire application"), category='warning') diff --git a/pylons_app/controllers/files.py b/pylons_app/controllers/files.py --- a/pylons_app/controllers/files.py +++ b/pylons_app/controllers/files.py @@ -45,6 +45,7 @@ class FilesController(BaseController): 'repository.admin') def __before__(self): super(FilesController, self).__before__() + c.file_size_limit = 250 * 1024 #limit of file size to display def index(self, repo_name, revision, f_path): hg_model = HgModel() @@ -76,7 +77,6 @@ class FilesController(BaseController): revision=next_rev, f_path=f_path) c.changeset = repo.get_changeset(revision) - c.cur_rev = c.changeset.raw_id c.rev_nr = c.changeset.revision @@ -96,6 +96,14 @@ class FilesController(BaseController): response.content_disposition = 'attachment; filename=%s' \ % f_path.split('/')[-1] return file_node.content + + def raw(self, repo_name, revision, f_path): + hg_model = HgModel() + c.repo = hg_model.get_repo(c.repo_name) + file_node = c.repo.get_changeset(revision).get_node(f_path) + response.content_type = 'text/plain' + + return file_node.content def annotate(self, repo_name, revision, f_path): hg_model = HgModel() diff --git a/pylons_app/controllers/login.py b/pylons_app/controllers/login.py --- a/pylons_app/controllers/login.py +++ b/pylons_app/controllers/login.py @@ -28,7 +28,9 @@ from pylons import request, response, se from pylons.controllers.util import abort, redirect from pylons_app.lib.auth import AuthUser, HasPermissionAnyDecorator from pylons_app.lib.base import BaseController, render -from pylons_app.model.forms import LoginForm, RegisterForm +import pylons_app.lib.helpers as h +from pylons.i18n.translation import _ +from pylons_app.model.forms import LoginForm, RegisterForm, PasswordResetForm from pylons_app.model.user_model import UserModel import formencode import logging @@ -42,7 +44,7 @@ class LoginController(BaseController): def index(self): #redirect if already logged in - c.came_from = request.GET.get('came_from',None) + c.came_from = request.GET.get('came_from', None) if c.hg_app_user.is_authenticated: return redirect(url('hg_home')) @@ -82,7 +84,7 @@ class LoginController(BaseController): return render('/login.html') - @HasPermissionAnyDecorator('hg.admin', 'hg.register.auto_activate', + @HasPermissionAnyDecorator('hg.admin', 'hg.register.auto_activate', 'hg.register.manual_activate') def register(self): user_model = UserModel() @@ -99,6 +101,8 @@ class LoginController(BaseController): form_result = register_form.to_python(dict(request.POST)) form_result['active'] = c.auto_active user_model.create_registration(form_result) + h.flash(_('You have successfully registered into hg-app'), + category='success') return redirect(url('login_home')) except formencode.Invalid as errors: @@ -110,7 +114,29 @@ class LoginController(BaseController): encoding="UTF-8") return render('/register.html') - + + def password_reset(self): + user_model = UserModel() + if request.POST: + + password_reset_form = PasswordResetForm()() + try: + form_result = password_reset_form.to_python(dict(request.POST)) + user_model.reset_password(form_result) + h.flash(_('Your new password was sent'), + category='success') + return redirect(url('login_home')) + + except formencode.Invalid as errors: + return htmlfill.render( + render('/password_reset.html'), + defaults=errors.value, + errors=errors.error_dict or {}, + prefix_error=False, + encoding="UTF-8") + + return render('/password_reset.html') + def logout(self): session['hg_app_user'] = AuthUser() session.save() diff --git a/pylons_app/controllers/search.py b/pylons_app/controllers/search.py --- a/pylons_app/controllers/search.py +++ b/pylons_app/controllers/search.py @@ -26,10 +26,9 @@ from pylons import request, response, se from pylons.controllers.util import abort, redirect from pylons_app.lib.auth import LoginRequired from pylons_app.lib.base import BaseController, render -from pylons_app.lib.indexers import ANALYZER, IDX_LOCATION, SCHEMA, IDX_NAME -from webhelpers.html.builder import escape -from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter, \ - ContextFragmenter +from pylons_app.lib.indexers import IDX_LOCATION, SCHEMA, IDX_NAME, ResultWrapper +from webhelpers.paginate import Page +from webhelpers.util import update_params from pylons.i18n.translation import _ from whoosh.index import open_dir, EmptyIndexError from whoosh.qparser import QueryParser, QueryParserError @@ -45,69 +44,55 @@ class SearchController(BaseController): def __before__(self): super(SearchController, self).__before__() - def index(self): c.formated_results = [] c.runtime = '' - search_items = set() c.cur_query = request.GET.get('q', None) if c.cur_query: cur_query = c.cur_query.lower() - if c.cur_query: + p = int(request.params.get('page', 1)) + highlight_items = set() try: idx = open_dir(IDX_LOCATION, indexname=IDX_NAME) searcher = idx.searcher() - + qp = QueryParser("content", schema=SCHEMA) try: query = qp.parse(unicode(cur_query)) if isinstance(query, Phrase): - search_items.update(query.words) + highlight_items.update(query.words) else: for i in query.all_terms(): - search_items.add(i[1]) - - log.debug(query) - log.debug(search_items) - results = searcher.search(query) - c.runtime = '%s results (%.3f seconds)' \ - % (len(results), results.runtime) + if i[0] == 'content': + highlight_items.add(i[1]) - analyzer = ANALYZER - formatter = HtmlFormatter('span', - between='\n...\n') - - #how the parts are splitted within the same text part - fragmenter = SimpleFragmenter(200) - #fragmenter = ContextFragmenter(search_items) + matcher = query.matcher(searcher) - for res in results: - d = {} - d.update(res) - hl = highlight(escape(res['content']), search_items, - analyzer=analyzer, - fragmenter=fragmenter, - formatter=formatter, - top=5) - f_path = res['path'][res['path'].find(res['repository']) \ - + len(res['repository']):].lstrip('/') - d.update({'content_short':hl, - 'f_path':f_path}) - #del d['content'] - c.formated_results.append(d) - + log.debug(query) + log.debug(highlight_items) + results = searcher.search(query) + res_ln = len(results) + c.runtime = '%s results (%.3f seconds)' \ + % (res_ln, results.runtime) + + def url_generator(**kw): + return update_params("?q=%s" % c.cur_query, **kw) + + c.formated_results = Page( + ResultWrapper(searcher, matcher, highlight_items), + page=p, item_count=res_ln, + items_per_page=10, url=url_generator) + except QueryParserError: c.runtime = _('Invalid search query. Try quoting it.') - + searcher.close() except (EmptyIndexError, IOError): log.error(traceback.format_exc()) log.error('Empty Index data') c.runtime = _('There is no index to search in. Please run whoosh indexer') - - - + # Return a rendered template return render('/search/search.html') diff --git a/pylons_app/controllers/summary.py b/pylons_app/controllers/summary.py --- a/pylons_app/controllers/summary.py +++ b/pylons_app/controllers/summary.py @@ -22,15 +22,17 @@ Created on April 18, 2010 summary controller for pylons @author: marcink """ -from datetime import datetime, timedelta -from pylons import tmpl_context as c, request +from pylons import tmpl_context as c, request, url from pylons_app.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator from pylons_app.lib.base import BaseController, render -from pylons_app.lib.helpers import person from pylons_app.lib.utils import OrderedDict from pylons_app.model.hg_model import HgModel +from pylons_app.model.db import Statistics +from webhelpers.paginate import Page +from pylons_app.lib.celerylib import run_task +from pylons_app.lib.celerylib.tasks import get_commits_stats +from datetime import datetime, timedelta from time import mktime -from webhelpers.paginate import Page import calendar import logging @@ -62,78 +64,33 @@ class SummaryController(BaseController): c.repo_branches = OrderedDict() for name, hash in c.repo_info.branches.items()[:10]: c.repo_branches[name] = c.repo_info.get_changeset(hash) + + td = datetime.today() + timedelta(days=1) + y, m, d = td.year, td.month, td.day + + ts_min_y = mktime((y - 1, (td - timedelta(days=calendar.mdays[m])).month, + d, 0, 0, 0, 0, 0, 0,)) + ts_min_m = mktime((y, (td - timedelta(days=calendar.mdays[m])).month, + d, 0, 0, 0, 0, 0, 0,)) + + ts_max_y = mktime((y, m, d, 0, 0, 0, 0, 0, 0,)) + + run_task(get_commits_stats, c.repo_info.name, ts_min_y, ts_max_y) + c.ts_min = ts_min_m + c.ts_max = ts_max_y + + + stats = self.sa.query(Statistics)\ + .filter(Statistics.repository == c.repo_info.dbrepo)\ + .scalar() - c.commit_data = self.__get_commit_stats(c.repo_info) + if stats: + c.commit_data = stats.commit_activity + c.overview_data = stats.commit_activity_combined + else: + import json + c.commit_data = json.dumps({}) + c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 0] ]) return render('summary/summary.html') - - - def __get_commit_stats(self, repo): - aggregate = OrderedDict() - - #graph range - td = datetime.today() + timedelta(days=1) - y, m, d = td.year, td.month, td.day - c.ts_min = mktime((y, (td - timedelta(days=calendar.mdays[m])).month, - d, 0, 0, 0, 0, 0, 0,)) - c.ts_max = mktime((y, m, d, 0, 0, 0, 0, 0, 0,)) - - def author_key_cleaner(k): - k = person(k) - k = k.replace('"', "'") #for js data compatibilty - return k - - for cs in repo[:200]:#added limit 200 until fix #29 is made - k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1], - cs.date.timetuple()[2]) - timetupple = [int(x) for x in k.split('-')] - timetupple.extend([0 for _ in xrange(6)]) - k = mktime(timetupple) - if aggregate.has_key(author_key_cleaner(cs.author)): - if aggregate[author_key_cleaner(cs.author)].has_key(k): - aggregate[author_key_cleaner(cs.author)][k]["commits"] += 1 - aggregate[author_key_cleaner(cs.author)][k]["added"] += len(cs.added) - aggregate[author_key_cleaner(cs.author)][k]["changed"] += len(cs.changed) - aggregate[author_key_cleaner(cs.author)][k]["removed"] += len(cs.removed) - - else: - #aggregate[author_key_cleaner(cs.author)].update(dates_range) - if k >= c.ts_min and k <= c.ts_max: - aggregate[author_key_cleaner(cs.author)][k] = {} - aggregate[author_key_cleaner(cs.author)][k]["commits"] = 1 - aggregate[author_key_cleaner(cs.author)][k]["added"] = len(cs.added) - aggregate[author_key_cleaner(cs.author)][k]["changed"] = len(cs.changed) - aggregate[author_key_cleaner(cs.author)][k]["removed"] = len(cs.removed) - - else: - if k >= c.ts_min and k <= c.ts_max: - aggregate[author_key_cleaner(cs.author)] = OrderedDict() - #aggregate[author_key_cleaner(cs.author)].update(dates_range) - aggregate[author_key_cleaner(cs.author)][k] = {} - aggregate[author_key_cleaner(cs.author)][k]["commits"] = 1 - aggregate[author_key_cleaner(cs.author)][k]["added"] = len(cs.added) - aggregate[author_key_cleaner(cs.author)][k]["changed"] = len(cs.changed) - aggregate[author_key_cleaner(cs.author)][k]["removed"] = len(cs.removed) - - d = '' - tmpl0 = u""""%s":%s""" - tmpl1 = u"""{label:"%s",data:%s,schema:["commits"]},""" - for author in aggregate: - - d += tmpl0 % (author, - tmpl1 \ - % (author, - [{"time":x, - "commits":aggregate[author][x]['commits'], - "added":aggregate[author][x]['added'], - "changed":aggregate[author][x]['changed'], - "removed":aggregate[author][x]['removed'], - } for x in aggregate[author]])) - if d == '': - d = '"%s":{label:"%s",data:[[0,1],]}' \ - % (author_key_cleaner(repo.contact), - author_key_cleaner(repo.contact)) - return d - - diff --git a/pylons_app/lib/auth.py b/pylons_app/lib/auth.py --- a/pylons_app/lib/auth.py +++ b/pylons_app/lib/auth.py @@ -34,9 +34,36 @@ from sqlalchemy.orm.exc import NoResultF import bcrypt from decorator import decorator import logging +import random log = logging.getLogger(__name__) +class PasswordGenerator(object): + """This is a simple class for generating password from + different sets of characters + usage: + passwd_gen = PasswordGenerator() + #print 8-letter password containing only big and small letters of alphabet + print passwd_gen.gen_password(8, passwd_gen.ALPHABETS_BIG_SMALL) + """ + ALPHABETS_NUM = r'''1234567890'''#[0] + ALPHABETS_SMALL = r'''qwertyuiopasdfghjklzxcvbnm'''#[1] + ALPHABETS_BIG = r'''QWERTYUIOPASDFGHJKLZXCVBNM'''#[2] + ALPHABETS_SPECIAL = r'''`-=[]\;',./~!@#$%^&*()_+{}|:"<>?''' #[3] + ALPHABETS_FULL = ALPHABETS_BIG + ALPHABETS_SMALL + ALPHABETS_NUM + ALPHABETS_SPECIAL#[4] + ALPHABETS_ALPHANUM = ALPHABETS_BIG + ALPHABETS_SMALL + ALPHABETS_NUM#[5] + ALPHABETS_BIG_SMALL = ALPHABETS_BIG + ALPHABETS_SMALL + ALPHABETS_ALPHANUM_BIG = ALPHABETS_BIG + ALPHABETS_NUM#[6] + ALPHABETS_ALPHANUM_SMALL = ALPHABETS_SMALL + ALPHABETS_NUM#[7] + + def __init__(self, passwd=''): + self.passwd = passwd + + def gen_password(self, len, type): + self.passwd = ''.join([random.choice(type) for _ in xrange(len)]) + return self.passwd + + def get_crypt_password(password): """Cryptographic function used for password hashing based on sha1 @param password: password to hash @@ -231,9 +258,9 @@ class LoginRequired(object): p = request.environ.get('PATH_INFO') if request.environ.get('QUERY_STRING'): - p+='?'+request.environ.get('QUERY_STRING') - log.debug('redirecting to login page with %s',p) - return redirect(url('login_home',came_from=p)) + p += '?' + request.environ.get('QUERY_STRING') + log.debug('redirecting to login page with %s', p) + return redirect(url('login_home', came_from=p)) class PermsDecorator(object): """Base class for decorators""" diff --git a/pylons_app/lib/celerylib/__init__.py b/pylons_app/lib/celerylib/__init__.py new file mode 100644 --- /dev/null +++ b/pylons_app/lib/celerylib/__init__.py @@ -0,0 +1,66 @@ +from pylons_app.lib.pidlock import DaemonLock, LockHeld +from vcs.utils.lazy import LazyProperty +from decorator import decorator +import logging +import os +import sys +import traceback +from hashlib import md5 +log = logging.getLogger(__name__) + +class ResultWrapper(object): + def __init__(self, task): + self.task = task + + @LazyProperty + def result(self): + return self.task + +def run_task(task, *args, **kwargs): + try: + t = task.delay(*args, **kwargs) + log.info('running task %s', t.task_id) + return t + except Exception, e: + print e + if e.errno == 111: + log.debug('Unnable to connect. Sync execution') + else: + log.error(traceback.format_exc()) + #pure sync version + return ResultWrapper(task(*args, **kwargs)) + + +class LockTask(object): + """LockTask decorator""" + + def __init__(self, func): + self.func = func + + def __call__(self, func): + return decorator(self.__wrapper, func) + + def __wrapper(self, func, *fargs, **fkwargs): + params = [] + params.extend(fargs) + params.extend(fkwargs.values()) + lockkey = 'task_%s' % \ + md5(str(self.func) + '-' + '-'.join(map(str, params))).hexdigest() + log.info('running task with lockkey %s', lockkey) + try: + l = DaemonLock(lockkey) + return func(*fargs, **fkwargs) + l.release() + except LockHeld: + log.info('LockHeld') + return 'Task with key %s already running' % lockkey + + + + + + + + + + diff --git a/pylons_app/lib/celerylib/tasks.py b/pylons_app/lib/celerylib/tasks.py new file mode 100644 --- /dev/null +++ b/pylons_app/lib/celerylib/tasks.py @@ -0,0 +1,270 @@ +from celery.decorators import task +from celery.task.sets import subtask +from celeryconfig import PYLONS_CONFIG as config +from pylons.i18n.translation import _ +from pylons_app.lib.celerylib import run_task, LockTask +from pylons_app.lib.helpers import person +from pylons_app.lib.smtp_mailer import SmtpMailer +from pylons_app.lib.utils import OrderedDict +from operator import itemgetter +from vcs.backends.hg import MercurialRepository +from time import mktime +import traceback +import json + +__all__ = ['whoosh_index', 'get_commits_stats', + 'reset_user_password', 'send_email'] + +def get_session(): + from sqlalchemy import engine_from_config + from sqlalchemy.orm import sessionmaker, scoped_session + engine = engine_from_config(dict(config.items('app:main')), 'sqlalchemy.db1.') + sa = scoped_session(sessionmaker(bind=engine)) + return sa + +def get_hg_settings(): + from pylons_app.model.db import HgAppSettings + try: + sa = get_session() + ret = sa.query(HgAppSettings).all() + finally: + sa.remove() + + if not ret: + raise Exception('Could not get application settings !') + settings = {} + for each in ret: + settings['hg_app_' + each.app_settings_name] = each.app_settings_value + + return settings + +def get_hg_ui_settings(): + from pylons_app.model.db import HgAppUi + try: + sa = get_session() + ret = sa.query(HgAppUi).all() + finally: + sa.remove() + + if not ret: + raise Exception('Could not get application ui settings !') + settings = {} + for each in ret: + k = each.ui_key + v = each.ui_value + if k == '/': + k = 'root_path' + + if k.find('.') != -1: + k = k.replace('.', '_') + + if each.ui_section == 'hooks': + v = each.ui_active + + settings[each.ui_section + '_' + k] = v + + return settings + +@task +def whoosh_index(repo_location, full_index): + log = whoosh_index.get_logger() + from pylons_app.lib.pidlock import DaemonLock + from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon, LockHeld + try: + l = DaemonLock() + WhooshIndexingDaemon(repo_location=repo_location)\ + .run(full_index=full_index) + l.release() + return 'Done' + except LockHeld: + log.info('LockHeld') + return 'LockHeld' + + +@task +@LockTask('get_commits_stats') +def get_commits_stats(repo_name, ts_min_y, ts_max_y): + author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty + + from pylons_app.model.db import Statistics, Repository + log = get_commits_stats.get_logger() + commits_by_day_author_aggregate = {} + commits_by_day_aggregate = {} + repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '') + repo = MercurialRepository(repos_path + repo_name) + + skip_date_limit = True + parse_limit = 350 #limit for single task changeset parsing + last_rev = 0 + last_cs = None + timegetter = itemgetter('time') + + sa = get_session() + + dbrepo = sa.query(Repository)\ + .filter(Repository.repo_name == repo_name).scalar() + cur_stats = sa.query(Statistics)\ + .filter(Statistics.repository == dbrepo).scalar() + if cur_stats: + last_rev = cur_stats.stat_on_revision + + if last_rev == repo.revisions[-1]: + #pass silently without any work + return True + + if cur_stats: + commits_by_day_aggregate = OrderedDict( + json.loads( + cur_stats.commit_activity_combined)) + commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity) + + for cnt, rev in enumerate(repo.revisions[last_rev:]): + last_cs = cs = repo.get_changeset(rev) + k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1], + cs.date.timetuple()[2]) + timetupple = [int(x) for x in k.split('-')] + timetupple.extend([0 for _ in xrange(6)]) + k = mktime(timetupple) + if commits_by_day_author_aggregate.has_key(author_key_cleaner(cs.author)): + try: + l = [timegetter(x) for x in commits_by_day_author_aggregate\ + [author_key_cleaner(cs.author)]['data']] + time_pos = l.index(k) + except ValueError: + time_pos = False + + if time_pos >= 0 and time_pos is not False: + + datadict = commits_by_day_author_aggregate\ + [author_key_cleaner(cs.author)]['data'][time_pos] + + datadict["commits"] += 1 + datadict["added"] += len(cs.added) + datadict["changed"] += len(cs.changed) + datadict["removed"] += len(cs.removed) + #print datadict + + else: + #print 'ELSE !!!!' + if k >= ts_min_y and k <= ts_max_y or skip_date_limit: + + datadict = {"time":k, + "commits":1, + "added":len(cs.added), + "changed":len(cs.changed), + "removed":len(cs.removed), + } + commits_by_day_author_aggregate\ + [author_key_cleaner(cs.author)]['data'].append(datadict) + + else: + #print k, 'nokey ADDING' + if k >= ts_min_y and k <= ts_max_y or skip_date_limit: + commits_by_day_author_aggregate[author_key_cleaner(cs.author)] = { + "label":author_key_cleaner(cs.author), + "data":[{"time":k, + "commits":1, + "added":len(cs.added), + "changed":len(cs.changed), + "removed":len(cs.removed), + }], + "schema":["commits"], + } + +# #gather all data by day + if commits_by_day_aggregate.has_key(k): + commits_by_day_aggregate[k] += 1 + else: + commits_by_day_aggregate[k] = 1 + + if cnt >= parse_limit: + #don't fetch to much data since we can freeze application + break + + overview_data = [] + for k, v in commits_by_day_aggregate.items(): + overview_data.append([k, v]) + overview_data = sorted(overview_data, key=itemgetter(0)) + + if not commits_by_day_author_aggregate: + commits_by_day_author_aggregate[author_key_cleaner(repo.contact)] = { + "label":author_key_cleaner(repo.contact), + "data":[0, 1], + "schema":["commits"], + } + + stats = cur_stats if cur_stats else Statistics() + stats.commit_activity = json.dumps(commits_by_day_author_aggregate) + stats.commit_activity_combined = json.dumps(overview_data) + stats.repository = dbrepo + stats.stat_on_revision = last_cs.revision + stats.languages = json.dumps({'_TOTAL_':0, '':0}) + + try: + sa.add(stats) + sa.commit() + except: + log.error(traceback.format_exc()) + sa.rollback() + return False + + run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) + + return True + +@task +def reset_user_password(user_email): + log = reset_user_password.get_logger() + from pylons_app.lib import auth + from pylons_app.model.db import User + + try: + try: + sa = get_session() + user = sa.query(User).filter(User.email == user_email).scalar() + new_passwd = auth.PasswordGenerator().gen_password(8, + auth.PasswordGenerator.ALPHABETS_BIG_SMALL) + if user: + user.password = auth.get_crypt_password(new_passwd) + sa.add(user) + sa.commit() + log.info('change password for %s', user_email) + if new_passwd is None: + raise Exception('unable to generate new password') + + except: + log.error(traceback.format_exc()) + sa.rollback() + + run_task(send_email, user_email, + "Your new hg-app password", + 'Your new hg-app password:%s' % (new_passwd)) + log.info('send new password mail to %s', user_email) + + + except: + log.error('Failed to update user password') + log.error(traceback.format_exc()) + return True + +@task +def send_email(recipients, subject, body): + log = send_email.get_logger() + email_config = dict(config.items('DEFAULT')) + mail_from = email_config.get('app_email_from') + user = email_config.get('smtp_username') + passwd = email_config.get('smtp_password') + mail_server = email_config.get('smtp_server') + mail_port = email_config.get('smtp_port') + tls = email_config.get('smtp_use_tls') + ssl = False + + try: + m = SmtpMailer(mail_from, user, passwd, mail_server, + mail_port, ssl, tls) + m.send(recipients, subject, body) + except: + log.error('Mail sending failed') + log.error(traceback.format_exc()) + return False + return True diff --git a/pylons_app/lib/db_manage.py b/pylons_app/lib/db_manage.py --- a/pylons_app/lib/db_manage.py +++ b/pylons_app/lib/db_manage.py @@ -43,7 +43,7 @@ import logging log = logging.getLogger(__name__) class DbManage(object): - def __init__(self, log_sql, dbname,tests=False): + def __init__(self, log_sql, dbname, tests=False): self.dbname = dbname self.tests = tests dburi = 'sqlite:////%s' % jn(ROOT, self.dbname) @@ -68,7 +68,7 @@ class DbManage(object): if override: log.info("database exisist and it's going to be destroyed") if self.tests: - destroy=True + destroy = True else: destroy = ask_ok('Are you sure to destroy old database ? [y/n]') if not destroy: @@ -84,15 +84,17 @@ class DbManage(object): import getpass username = raw_input('Specify admin username:') password = getpass.getpass('Specify admin password:') - self.create_user(username, password, True) + email = raw_input('Specify admin email:') + self.create_user(username, password, email, True) else: log.info('creating admin and regular test users') - self.create_user('test_admin', 'test', True) - self.create_user('test_regular', 'test', False) + self.create_user('test_admin', 'test', 'test_admin@mail.com', True) + self.create_user('test_regular', 'test', 'test_regular@mail.com', False) + self.create_user('test_regular2', 'test', 'test_regular2@mail.com', False) - def config_prompt(self,test_repo_path=''): + def config_prompt(self, test_repo_path=''): log.info('Setting up repositories config') if not self.tests and not test_repo_path: @@ -102,7 +104,7 @@ class DbManage(object): path = test_repo_path if not os.path.isdir(path): - log.error('You entered wrong path: %s',path) + log.error('You entered wrong path: %s', path) sys.exit() hooks1 = HgAppUi() @@ -166,14 +168,14 @@ class DbManage(object): raise log.info('created ui config') - def create_user(self, username, password, admin=False): + def create_user(self, username, password, email='', admin=False): log.info('creating administrator user %s', username) new_user = User() new_user.username = username new_user.password = get_crypt_password(password) new_user.name = 'Hg' new_user.lastname = 'Admin' - new_user.email = 'admin@localhost' + new_user.email = email new_user.admin = admin new_user.active = True diff --git a/pylons_app/lib/helpers.py b/pylons_app/lib/helpers.py --- a/pylons_app/lib/helpers.py +++ b/pylons_app/lib/helpers.py @@ -277,13 +277,17 @@ def pygmentize_annotation(filenode, **kw return literal(annotate_highlight(filenode, url_func, **kwargs)) def repo_name_slug(value): - """ - Return slug of name of repository + """Return slug of name of repository + This function is called on each creation/modification + of repository to prevent bad names in repo """ - slug = urlify(value) - for c in """=[]\;'"<>,/~!@#$%^&*()+{}|:""": + slug = remove_formatting(value) + slug = strip_tags(slug) + + for c in """=[]\;'"<>,/~!@#$%^&*()+{}|: """: slug = slug.replace(c, '-') slug = recursive_replace(slug, '-') + slug = collapse(slug, '-') return slug def get_changeset_safe(repo, rev): @@ -321,6 +325,7 @@ isodate = lambda x: util.datestr(x, '%Y isodatesec = lambda x: util.datestr(x, '%Y-%m-%d %H:%M:%S %1%2') localdate = lambda x: (x[0], util.makedate()[1]) rfc822date = lambda x: util.datestr(x, "%a, %d %b %Y %H:%M:%S %1%2") +rfc822date_notz = lambda x: util.datestr(x, "%a, %d %b %Y %H:%M:%S") rfc3339date = lambda x: util.datestr(x, "%Y-%m-%dT%H:%M:%S%1:%2") time_ago = lambda x: util.datestr(_age(x), "%a, %d %b %Y %H:%M:%S %1%2") diff --git a/pylons_app/lib/indexers/__init__.py b/pylons_app/lib/indexers/__init__.py --- a/pylons_app/lib/indexers/__init__.py +++ b/pylons_app/lib/indexers/__init__.py @@ -1,41 +1,139 @@ -import sys +from os.path import dirname as dn, join as jn +from pylons_app.config.environment import load_environment +from pylons_app.model.hg_model import HgModel +from shutil import rmtree +from webhelpers.html.builder import escape +from vcs.utils.lazy import LazyProperty + +from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter +from whoosh.fields import TEXT, ID, STORED, Schema, FieldType +from whoosh.index import create_in, open_dir +from whoosh.formats import Characters +from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter + import os -from pidlock import LockHeld, DaemonLock +import sys import traceback -from os.path import dirname as dn -from os.path import join as jn - #to get the pylons_app import sys.path.append(dn(dn(dn(os.path.realpath(__file__))))) -from pylons_app.config.environment import load_environment -from pylons_app.model.hg_model import HgModel -from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter -from whoosh.fields import TEXT, ID, STORED, Schema -from whoosh.index import create_in, open_dir -from shutil import rmtree #LOCATION WE KEEP THE INDEX IDX_LOCATION = jn(dn(dn(dn(dn(os.path.abspath(__file__))))), 'data', 'index') #EXTENSIONS WE WANT TO INDEX CONTENT OFF -INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c', - 'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl', 'h', - 'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp', - 'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3', - 'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql', - 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml','xsl','xslt', +INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c', + 'cfg', 'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl', + 'h', 'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp', + 'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3', + 'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql', + 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', 'yaws'] #CUSTOM ANALYZER wordsplit + lowercase filter ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter() + #INDEX SCHEMA DEFINITION SCHEMA = Schema(owner=TEXT(), repository=TEXT(stored=True), path=ID(stored=True, unique=True), - content=TEXT(stored=True, analyzer=ANALYZER), - modtime=STORED(),extension=TEXT(stored=True)) + content=FieldType(format=Characters(ANALYZER), + scorable=True, stored=True), + modtime=STORED(), extension=TEXT(stored=True)) + + +IDX_NAME = 'HG_INDEX' +FORMATTER = HtmlFormatter('span', between='\n...\n') +FRAGMENTER = SimpleFragmenter(200) + +class ResultWrapper(object): + def __init__(self, searcher, matcher, highlight_items): + self.searcher = searcher + self.matcher = matcher + self.highlight_items = highlight_items + self.fragment_size = 200 / 2 + + @LazyProperty + def doc_ids(self): + docs_id = [] + while self.matcher.is_active(): + docnum = self.matcher.id() + chunks = [offsets for offsets in self.get_chunks()] + docs_id.append([docnum, chunks]) + self.matcher.next() + return docs_id + + def __str__(self): + return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids)) + + def __repr__(self): + return self.__str__() + + def __len__(self): + return len(self.doc_ids) + + def __iter__(self): + """ + Allows Iteration over results,and lazy generate content + + *Requires* implementation of ``__getitem__`` method. + """ + for docid in self.doc_ids: + yield self.get_full_content(docid) -IDX_NAME = 'HG_INDEX' \ No newline at end of file + def __getslice__(self, i, j): + """ + Slicing of resultWrapper + """ + slice = [] + for docid in self.doc_ids[i:j]: + slice.append(self.get_full_content(docid)) + return slice + + + def get_full_content(self, docid): + res = self.searcher.stored_fields(docid[0]) + f_path = res['path'][res['path'].find(res['repository']) \ + + len(res['repository']):].lstrip('/') + + content_short = self.get_short_content(res, docid[1]) + res.update({'content_short':content_short, + 'content_short_hl':self.highlight(content_short), + 'f_path':f_path}) + + return res + + def get_short_content(self, res, chunks): + + return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks]) + + def get_chunks(self): + """ + Smart function that implements chunking the content + but not overlap chunks so it doesn't highlight the same + close occurences twice. + @param matcher: + @param size: + """ + memory = [(0, 0)] + for span in self.matcher.spans(): + start = span.startchar or 0 + end = span.endchar or 0 + start_offseted = max(0, start - self.fragment_size) + end_offseted = end + self.fragment_size + + if start_offseted < memory[-1][1]: + start_offseted = memory[-1][1] + memory.append((start_offseted, end_offseted,)) + yield (start_offseted, end_offseted,) + + def highlight(self, content, top=5): + hl = highlight(escape(content), + self.highlight_items, + analyzer=ANALYZER, + fragmenter=FRAGMENTER, + formatter=FORMATTER, + top=top) + return hl diff --git a/pylons_app/lib/indexers/daemon.py b/pylons_app/lib/indexers/daemon.py --- a/pylons_app/lib/indexers/daemon.py +++ b/pylons_app/lib/indexers/daemon.py @@ -32,20 +32,31 @@ from os.path import join as jn project_path = dn(dn(dn(dn(os.path.realpath(__file__))))) sys.path.append(project_path) -from pidlock import LockHeld, DaemonLock -import traceback -from pylons_app.config.environment import load_environment +from pylons_app.lib.pidlock import LockHeld, DaemonLock from pylons_app.model.hg_model import HgModel from pylons_app.lib.helpers import safe_unicode from whoosh.index import create_in, open_dir from shutil import rmtree -from pylons_app.lib.indexers import ANALYZER, INDEX_EXTENSIONS, IDX_LOCATION, \ -SCHEMA, IDX_NAME +from pylons_app.lib.indexers import INDEX_EXTENSIONS, IDX_LOCATION, SCHEMA, IDX_NAME import logging -import logging.config -logging.config.fileConfig(jn(project_path, 'development.ini')) + log = logging.getLogger('whooshIndexer') +# create logger +log.setLevel(logging.DEBUG) +log.propagate = False +# create console handler and set level to debug +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) + +# create formatter +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + +# add formatter to ch +ch.setFormatter(formatter) + +# add ch to logger +log.addHandler(ch) def scan_paths(root_location): return HgModel.repo_scan('/', root_location, None, True) @@ -221,6 +232,7 @@ if __name__ == "__main__": WhooshIndexingDaemon(repo_location=repo_location)\ .run(full_index=full_index) l.release() + reload(logging) except LockHeld: sys.exit(1) diff --git a/pylons_app/lib/indexers/pidlock.py b/pylons_app/lib/pidlock.py rename from pylons_app/lib/indexers/pidlock.py rename to pylons_app/lib/pidlock.py --- a/pylons_app/lib/indexers/pidlock.py +++ b/pylons_app/lib/pidlock.py @@ -6,7 +6,7 @@ class LockHeld(Exception):pass class DaemonLock(object): - '''daemon locking + """daemon locking USAGE: try: l = lock() @@ -14,7 +14,7 @@ class DaemonLock(object): l.release() except LockHeld: sys.exit(1) - ''' + """ def __init__(self, file=None, callbackfn=None, desc='daemon lock', debug=False): @@ -40,9 +40,9 @@ class DaemonLock(object): def lock(self): - ''' + """ locking function, if lock is present it will raise LockHeld exception - ''' + """ lockname = '%s' % (os.getpid()) self.trylock() @@ -75,9 +75,9 @@ class DaemonLock(object): def release(self): - ''' + """ releases the pid by removing the pidfile - ''' + """ if self.callbackfn: #execute callback function on release if self.debug: @@ -94,11 +94,11 @@ class DaemonLock(object): pass def makelock(self, lockname, pidfile): - ''' + """ this function will make an actual lock @param lockname: acctual pid of file @param pidfile: the file to write the pid in - ''' + """ if self.debug: print 'creating a file %s and pid: %s' % (pidfile, lockname) pidfile = open(self.pidfile, "wb") diff --git a/pylons_app/lib/smtp_mailer.py b/pylons_app/lib/smtp_mailer.py new file mode 100644 --- /dev/null +++ b/pylons_app/lib/smtp_mailer.py @@ -0,0 +1,118 @@ +import logging +import smtplib +import mimetypes +from email.mime.multipart import MIMEMultipart +from email.mime.image import MIMEImage +from email.mime.audio import MIMEAudio +from email.mime.base import MIMEBase +from email.mime.text import MIMEText +from email.utils import formatdate +from email import encoders + +class SmtpMailer(object): + """simple smtp mailer class + + mailer = SmtpMailer(mail_from, user, passwd, mail_server, mail_port, ssl, tls) + mailer.send(recipients, subject, body, attachment_files) + + :param recipients might be a list of string or single string + :param attachment_files is a dict of {filename:location} + it tries to guess the mimetype and attach the file + """ + + def __init__(self, mail_from, user, passwd, mail_server, + mail_port=None, ssl=False, tls=False): + + self.mail_from = mail_from + self.mail_server = mail_server + self.mail_port = mail_port + self.user = user + self.passwd = passwd + self.ssl = ssl + self.tls = tls + self.debug = False + + def send(self, recipients=[], subject='', body='', attachment_files={}): + + if isinstance(recipients, basestring): + recipients = [recipients] + if self.ssl: + smtp_serv = smtplib.SMTP_SSL(self.mail_server, self.mail_port) + else: + smtp_serv = smtplib.SMTP(self.mail_server, self.mail_port) + + if self.tls: + smtp_serv.starttls() + + if self.debug: + smtp_serv.set_debuglevel(1) + + smtp_serv.ehlo("mailer") + + #if server requires authorization you must provide login and password + smtp_serv.login(self.user, self.passwd) + + date_ = formatdate(localtime=True) + msg = MIMEMultipart() + msg['From'] = self.mail_from + msg['To'] = ','.join(recipients) + msg['Date'] = date_ + msg['Subject'] = subject + msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' + + msg.attach(MIMEText(body)) + + if attachment_files: + self.__atach_files(msg, attachment_files) + + smtp_serv.sendmail(self.mail_from, recipients, msg.as_string()) + logging.info('MAIL SEND TO: %s' % recipients) + smtp_serv.quit() + + + def __atach_files(self, msg, attachment_files): + if isinstance(attachment_files, dict): + for f_name, msg_file in attachment_files.items(): + ctype, encoding = mimetypes.guess_type(f_name) + logging.info("guessing file %s type based on %s" , ctype, f_name) + if ctype is None or encoding is not None: + # No guess could be made, or the file is encoded (compressed), so + # use a generic bag-of-bits type. + ctype = 'application/octet-stream' + maintype, subtype = ctype.split('/', 1) + if maintype == 'text': + # Note: we should handle calculating the charset + file_part = MIMEText(self.get_content(msg_file), + _subtype=subtype) + elif maintype == 'image': + file_part = MIMEImage(self.get_content(msg_file), + _subtype=subtype) + elif maintype == 'audio': + file_part = MIMEAudio(self.get_content(msg_file), + _subtype=subtype) + else: + file_part = MIMEBase(maintype, subtype) + file_part.set_payload(self.get_content(msg_file)) + # Encode the payload using Base64 + encoders.encode_base64(msg) + # Set the filename parameter + file_part.add_header('Content-Disposition', 'attachment', + filename=f_name) + file_part.add_header('Content-Type', ctype, name=f_name) + msg.attach(file_part) + else: + raise Exception('Attachment files should be' + 'a dict in format {"filename":"filepath"}') + + def get_content(self, msg_file): + ''' + Get content based on type, if content is a string do open first + else just read because it's a probably open file object + @param msg_file: + ''' + if isinstance(msg_file, str): + return open(msg_file, "rb").read() + else: + #just for safe seek to 0 + msg_file.seek(0) + return msg_file.read() diff --git a/pylons_app/lib/timerproxy.py b/pylons_app/lib/timerproxy.py --- a/pylons_app/lib/timerproxy.py +++ b/pylons_app/lib/timerproxy.py @@ -1,7 +1,6 @@ from sqlalchemy.interfaces import ConnectionProxy import time -import logging -log = logging.getLogger('timerproxy') +from sqlalchemy import log BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38) def color_sql(sql): @@ -22,7 +21,7 @@ def format_sql(sql): sql = sql.replace('\n', '') sql = one_space_trim(sql) sql = sql\ - .replace(',',',\n\t')\ + .replace(',', ',\n\t')\ .replace('SELECT', '\n\tSELECT \n\t')\ .replace('UPDATE', '\n\tUPDATE \n\t')\ .replace('DELETE', '\n\tDELETE \n\t')\ @@ -39,19 +38,22 @@ def format_sql(sql): class TimerProxy(ConnectionProxy): + + def __init__(self): + super(TimerProxy, self).__init__() + self.logging_name = 'timerProxy' + self.log = log.instance_logger(self, True) + def cursor_execute(self, execute, cursor, statement, parameters, context, executemany): + now = time.time() try: - log.info(">>>>> STARTING QUERY >>>>>") + self.log.info(">>>>> STARTING QUERY >>>>>") return execute(cursor, statement, parameters, context) finally: total = time.time() - now try: - log.info(format_sql("Query: %s" % statement % parameters)) + self.log.info(format_sql("Query: %s" % statement % parameters)) except TypeError: - log.info(format_sql("Query: %s %s" % (statement, parameters))) - log.info("<<<<< TOTAL TIME: %f <<<<<" % total) - - - - + self.log.info(format_sql("Query: %s %s" % (statement, parameters))) + self.log.info("<<<<< TOTAL TIME: %f <<<<<" % total) diff --git a/pylons_app/lib/utils.py b/pylons_app/lib/utils.py --- a/pylons_app/lib/utils.py +++ b/pylons_app/lib/utils.py @@ -31,6 +31,7 @@ from vcs.backends.base import BaseChange from vcs.utils.lazy import LazyProperty import logging import os + log = logging.getLogger(__name__) @@ -218,6 +219,7 @@ class EmptyChangeset(BaseChangeset): revision = -1 message = '' + author = '' @LazyProperty def raw_id(self): @@ -362,3 +364,75 @@ class OrderedDict(dict, DictMixin): def __ne__(self, other): return not self == other + + +#=============================================================================== +# TEST FUNCTIONS +#=============================================================================== +def create_test_index(repo_location, full_index): + """Makes default test index + @param repo_location: + @param full_index: + """ + from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon + from pylons_app.lib.pidlock import DaemonLock, LockHeld + from pylons_app.lib.indexers import IDX_LOCATION + import shutil + + if os.path.exists(IDX_LOCATION): + shutil.rmtree(IDX_LOCATION) + + try: + l = DaemonLock() + WhooshIndexingDaemon(repo_location=repo_location)\ + .run(full_index=full_index) + l.release() + except LockHeld: + pass + +def create_test_env(repos_test_path, config): + """Makes a fresh database and + install test repository into tmp dir + """ + from pylons_app.lib.db_manage import DbManage + import tarfile + import shutil + from os.path import dirname as dn, join as jn, abspath + + log = logging.getLogger('TestEnvCreator') + # create logger + log.setLevel(logging.DEBUG) + log.propagate = True + # create console handler and set level to debug + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + + # create formatter + formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + + # add formatter to ch + ch.setFormatter(formatter) + + # add ch to logger + log.addHandler(ch) + + #PART ONE create db + log.debug('making test db') + dbname = config['sqlalchemy.db1.url'].split('/')[-1] + dbmanage = DbManage(log_sql=True, dbname=dbname, tests=True) + dbmanage.create_tables(override=True) + dbmanage.config_prompt(repos_test_path) + dbmanage.create_default_user() + dbmanage.admin_prompt() + dbmanage.create_permissions() + dbmanage.populate_default_permissions() + + #PART TWO make test repo + log.debug('making test vcs repo') + if os.path.isdir('/tmp/vcs_test'): + shutil.rmtree('/tmp/vcs_test') + + cur_dir = dn(dn(abspath(__file__))) + tar = tarfile.open(jn(cur_dir, 'tests', "vcs_test.tar.gz")) + tar.extractall('/tmp') + tar.close() diff --git a/pylons_app/model/__init__.py b/pylons_app/model/__init__.py --- a/pylons_app/model/__init__.py +++ b/pylons_app/model/__init__.py @@ -1,15 +1,8 @@ """The application's model objects""" import logging -import sqlalchemy as sa -from sqlalchemy import orm from pylons_app.model import meta -from pylons_app.model.meta import Session log = logging.getLogger(__name__) -# Add these two imports: -import datetime -from sqlalchemy import schema, types - def init_model(engine): """Call me before using any of the tables or classes in the model""" log.info("INITIALIZING DB MODELS") diff --git a/pylons_app/model/caching_query.py b/pylons_app/model/caching_query.py new file mode 100644 --- /dev/null +++ b/pylons_app/model/caching_query.py @@ -0,0 +1,267 @@ +"""caching_query.py + +Represent persistence structures which allow the usage of +Beaker caching with SQLAlchemy. + +The three new concepts introduced here are: + + * CachingQuery - a Query subclass that caches and + retrieves results in/from Beaker. + * FromCache - a query option that establishes caching + parameters on a Query + * RelationshipCache - a variant of FromCache which is specific + to a query invoked during a lazy load. + * _params_from_query - extracts value parameters from + a Query. + +The rest of what's here are standard SQLAlchemy and +Beaker constructs. + +""" +from sqlalchemy.orm.interfaces import MapperOption +from sqlalchemy.orm.query import Query +from sqlalchemy.sql import visitors + +class CachingQuery(Query): + """A Query subclass which optionally loads full results from a Beaker + cache region. + + The CachingQuery stores additional state that allows it to consult + a Beaker cache before accessing the database: + + * A "region", which is a cache region argument passed to a + Beaker CacheManager, specifies a particular cache configuration + (including backend implementation, expiration times, etc.) + * A "namespace", which is a qualifying name that identifies a + group of keys within the cache. A query that filters on a name + might use the name "by_name", a query that filters on a date range + to a joined table might use the name "related_date_range". + + When the above state is present, a Beaker cache is retrieved. + + The "namespace" name is first concatenated with + a string composed of the individual entities and columns the Query + requests, i.e. such as ``Query(User.id, User.name)``. + + The Beaker cache is then loaded from the cache manager based + on the region and composed namespace. The key within the cache + itself is then constructed against the bind parameters specified + by this query, which are usually literals defined in the + WHERE clause. + + The FromCache and RelationshipCache mapper options below represent + the "public" method of configuring this state upon the CachingQuery. + + """ + + def __init__(self, manager, *args, **kw): + self.cache_manager = manager + Query.__init__(self, *args, **kw) + + def __iter__(self): + """override __iter__ to pull results from Beaker + if particular attributes have been configured. + + Note that this approach does *not* detach the loaded objects from + the current session. If the cache backend is an in-process cache + (like "memory") and lives beyond the scope of the current session's + transaction, those objects may be expired. The method here can be + modified to first expunge() each loaded item from the current + session before returning the list of items, so that the items + in the cache are not the same ones in the current Session. + + """ + if hasattr(self, '_cache_parameters'): + return self.get_value(createfunc=lambda: list(Query.__iter__(self))) + else: + return Query.__iter__(self) + + def invalidate(self): + """Invalidate the value represented by this Query.""" + + cache, cache_key = _get_cache_parameters(self) + cache.remove(cache_key) + + def get_value(self, merge=True, createfunc=None): + """Return the value from the cache for this query. + + Raise KeyError if no value present and no + createfunc specified. + + """ + cache, cache_key = _get_cache_parameters(self) + ret = cache.get_value(cache_key, createfunc=createfunc) + if merge: + ret = self.merge_result(ret, load=False) + return ret + + def set_value(self, value): + """Set the value in the cache for this query.""" + + cache, cache_key = _get_cache_parameters(self) + cache.put(cache_key, value) + +def query_callable(manager): + def query(*arg, **kw): + return CachingQuery(manager, *arg, **kw) + return query + +def _get_cache_parameters(query): + """For a query with cache_region and cache_namespace configured, + return the correspoinding Cache instance and cache key, based + on this query's current criterion and parameter values. + + """ + if not hasattr(query, '_cache_parameters'): + raise ValueError("This Query does not have caching parameters configured.") + + region, namespace, cache_key = query._cache_parameters + + namespace = _namespace_from_query(namespace, query) + + if cache_key is None: + # cache key - the value arguments from this query's parameters. + args = _params_from_query(query) + cache_key = " ".join([str(x) for x in args]) + + # get cache + cache = query.cache_manager.get_cache_region(namespace, region) + + # optional - hash the cache_key too for consistent length + # import uuid + # cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key)) + + return cache, cache_key + +def _namespace_from_query(namespace, query): + # cache namespace - the token handed in by the + # option + class we're querying against + namespace = " ".join([namespace] + [str(x) for x in query._entities]) + + # memcached wants this + namespace = namespace.replace(' ', '_') + + return namespace + +def _set_cache_parameters(query, region, namespace, cache_key): + + if hasattr(query, '_cache_parameters'): + region, namespace, cache_key = query._cache_parameters + raise ValueError("This query is already configured " + "for region %r namespace %r" % + (region, namespace) + ) + query._cache_parameters = region, namespace, cache_key + +class FromCache(MapperOption): + """Specifies that a Query should load results from a cache.""" + + propagate_to_loaders = False + + def __init__(self, region, namespace, cache_key=None): + """Construct a new FromCache. + + :param region: the cache region. Should be a + region configured in the Beaker CacheManager. + + :param namespace: the cache namespace. Should + be a name uniquely describing the target Query's + lexical structure. + + :param cache_key: optional. A string cache key + that will serve as the key to the query. Use this + if your query has a huge amount of parameters (such + as when using in_()) which correspond more simply to + some other identifier. + + """ + self.region = region + self.namespace = namespace + self.cache_key = cache_key + + def process_query(self, query): + """Process a Query during normal loading operation.""" + + _set_cache_parameters(query, self.region, self.namespace, self.cache_key) + +class RelationshipCache(MapperOption): + """Specifies that a Query as called within a "lazy load" + should load results from a cache.""" + + propagate_to_loaders = True + + def __init__(self, region, namespace, attribute): + """Construct a new RelationshipCache. + + :param region: the cache region. Should be a + region configured in the Beaker CacheManager. + + :param namespace: the cache namespace. Should + be a name uniquely describing the target Query's + lexical structure. + + :param attribute: A Class.attribute which + indicates a particular class relationship() whose + lazy loader should be pulled from the cache. + + """ + self.region = region + self.namespace = namespace + self._relationship_options = { + (attribute.property.parent.class_, attribute.property.key) : self + } + + def process_query_conditionally(self, query): + """Process a Query that is used within a lazy loader. + + (the process_query_conditionally() method is a SQLAlchemy + hook invoked only within lazyload.) + + """ + if query._current_path: + mapper, key = query._current_path[-2:] + + for cls in mapper.class_.__mro__: + if (cls, key) in self._relationship_options: + relationship_option = self._relationship_options[(cls, key)] + _set_cache_parameters( + query, + relationship_option.region, + relationship_option.namespace, + None) + + def and_(self, option): + """Chain another RelationshipCache option to this one. + + While many RelationshipCache objects can be specified on a single + Query separately, chaining them together allows for a more efficient + lookup during load. + + """ + self._relationship_options.update(option._relationship_options) + return self + + +def _params_from_query(query): + """Pull the bind parameter values from a query. + + This takes into account any scalar attribute bindparam set up. + + E.g. params_from_query(query.filter(Cls.foo==5).filter(Cls.bar==7))) + would return [5, 7]. + + """ + v = [] + def visit_bindparam(bind): + value = query._params.get(bind.key, bind.value) + + # lazyloader may dig a callable in here, intended + # to late-evaluate params after autoflush is called. + # convert to a scalar value. + if callable(value): + value = value() + + v.append(value) + if query._criterion is not None: + visitors.traverse(query._criterion, {}, {'bindparam':visit_bindparam}) + return v diff --git a/pylons_app/model/db.py b/pylons_app/model/db.py --- a/pylons_app/model/db.py +++ b/pylons_app/model/db.py @@ -26,7 +26,7 @@ class HgAppUi(Base): class User(Base): __tablename__ = 'users' - __table_args__ = (UniqueConstraint('username'), {'useexisting':True}) + __table_args__ = (UniqueConstraint('username'), UniqueConstraint('email'), {'useexisting':True}) user_id = Column("user_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True) username = Column("username", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None) password = Column("password", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None) @@ -56,7 +56,7 @@ class User(Base): self.last_login = datetime.datetime.now() session.add(self) session.commit() - log.debug('updated user %s lastlogin',self.username) + log.debug('updated user %s lastlogin', self.username) except Exception: session.rollback() @@ -120,6 +120,15 @@ class UserToPerm(Base): user = relation('User') permission = relation('Permission') - +class Statistics(Base): + __tablename__ = 'statistics' + __table_args__ = (UniqueConstraint('repository_id'), {'useexisting':True}) + stat_id = Column("stat_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True) + repository_id = Column("repository_id", INTEGER(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=True, default=None) + stat_on_revision = Column("stat_on_revision", INTEGER(), nullable=False) + commit_activity = Column("commit_activity", BLOB(), nullable=False)#JSON data + commit_activity_combined = Column("commit_activity_combined", BLOB(), nullable=False)#JSON data + languages = Column("languages", BLOB(), nullable=False)#JSON data + + repository = relation('Repository') - diff --git a/pylons_app/model/forms.py b/pylons_app/model/forms.py --- a/pylons_app/model/forms.py +++ b/pylons_app/model/forms.py @@ -102,7 +102,7 @@ class ValidAuth(formencode.validators.Fa error_dict=self.e_dict) if user: if user.active: - if user.username == username and check_password(password, + if user.username == username and check_password(password, user.password): return value else: @@ -208,7 +208,37 @@ class ValidPath(formencode.validators.Fa raise formencode.Invalid(msg, value, state, error_dict={'paths_root_path':msg}) - + +def UniqSystemEmail(old_data): + class _UniqSystemEmail(formencode.validators.FancyValidator): + def to_python(self, value, state): + if old_data.get('email') != value: + sa = meta.Session + try: + user = sa.query(User).filter(User.email == value).scalar() + if user: + raise formencode.Invalid(_("That e-mail address is already taken") , + value, state) + finally: + meta.Session.remove() + + return value + + return _UniqSystemEmail + +class ValidSystemEmail(formencode.validators.FancyValidator): + def to_python(self, value, state): + sa = meta.Session + try: + user = sa.query(User).filter(User.email == value).scalar() + if user is None: + raise formencode.Invalid(_("That e-mail address doesn't exist.") , + value, state) + finally: + meta.Session.remove() + + return value + #=============================================================================== # FORMS #=============================================================================== @@ -250,13 +280,19 @@ def UserForm(edit=False, old_data={}): active = StringBoolean(if_missing=False) name = UnicodeString(strip=True, min=3, not_empty=True) lastname = UnicodeString(strip=True, min=3, not_empty=True) - email = Email(not_empty=True) + email = All(Email(not_empty=True), UniqSystemEmail(old_data)) return _UserForm RegisterForm = UserForm - - + +def PasswordResetForm(): + class _PasswordResetForm(formencode.Schema): + allow_extra_fields = True + filter_extra_fields = True + email = All(ValidSystemEmail(), Email(not_empty=True)) + return _PasswordResetForm + def RepoForm(edit=False, old_data={}): class _RepoForm(formencode.Schema): allow_extra_fields = True diff --git a/pylons_app/model/hg_model.py b/pylons_app/model/hg_model.py --- a/pylons_app/model/hg_model.py +++ b/pylons_app/model/hg_model.py @@ -43,16 +43,14 @@ except ImportError: raise Exception('Unable to import vcs') def _get_repos_cached_initial(app_globals, initial): - """ - return cached dict with repos + """return cached dict with repos """ g = app_globals return HgModel.repo_scan(g.paths[0][0], g.paths[0][1], g.baseui, initial) @cache_region('long_term', 'cached_repo_list') def _get_repos_cached(): - """ - return cached dict with repos + """return cached dict with repos """ log.info('getting all repositories list') from pylons import app_globals as g @@ -61,11 +59,12 @@ def _get_repos_cached(): @cache_region('super_short_term', 'cached_repos_switcher_list') def _get_repos_switcher_cached(cached_repo_list): repos_lst = [] - for repo in sorted(x.name.lower() for x in cached_repo_list.values()): - if HasRepoPermissionAny('repository.write', 'repository.read', 'repository.admin')(repo, 'main page check'): - repos_lst.append(repo) + for repo in [x for x in cached_repo_list.values()]: + if HasRepoPermissionAny('repository.write', 'repository.read', + 'repository.admin')(repo.name.lower(), 'main page check'): + repos_lst.append((repo.name, repo.dbrepo.private,)) - return repos_lst + return sorted(repos_lst, key=lambda k:k[0]) @cache_region('long_term', 'full_changelog') def _full_changelog_cached(repo_name): @@ -73,14 +72,11 @@ def _full_changelog_cached(repo_name): return list(reversed(list(HgModel().get_repo(repo_name)))) class HgModel(object): - """ - Mercurial Model + """Mercurial Model """ def __init__(self): - """ - Constructor - """ + pass @staticmethod def repo_scan(repos_prefix, repos_path, baseui, initial=False): @@ -92,8 +88,7 @@ class HgModel(object): """ sa = meta.Session() def check_repo_dir(path): - """ - Checks the repository + """Checks the repository :param path: """ repos_path = path.split('/') @@ -102,7 +97,7 @@ class HgModel(object): if repos_path[0] != '/': repos_path[0] = '/' if not os.path.isdir(os.path.join(*repos_path)): - raise RepositoryError('Not a valid repository in %s' % path[0][1]) + raise RepositoryError('Not a valid repository in %s' % path) if not repos_path.endswith('*'): raise VCSError('You need to specify * or ** at the end of path ' 'for recursive scanning') diff --git a/pylons_app/model/meta.py b/pylons_app/model/meta.py --- a/pylons_app/model/meta.py +++ b/pylons_app/model/meta.py @@ -1,15 +1,58 @@ """SQLAlchemy Metadata and Session object""" from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import scoped_session, sessionmaker +from pylons_app.model import caching_query +from beaker import cache +import os +from os.path import join as jn, dirname as dn, abspath +import time + +# Beaker CacheManager. A home base for cache configurations. +cache_manager = cache.CacheManager() __all__ = ['Base', 'Session'] # # SQLAlchemy session manager. Updated by model.init_model() # -Session = scoped_session(sessionmaker()) -# +Session = scoped_session( + sessionmaker( + query_cls=caching_query.query_callable(cache_manager) + ) + ) # The declarative Base Base = declarative_base() #For another db... #Base2 = declarative_base() + +#=============================================================================== +# CACHE OPTIONS +#=============================================================================== +cache_dir = jn(dn(dn(dn(abspath(__file__)))), 'data', 'cache') +if not os.path.isdir(cache_dir): + os.mkdir(cache_dir) +# set start_time to current time +# to re-cache everything +# upon application startup +start_time = time.time() +# configure the "sqlalchemy" cache region. +cache_manager.regions['sql_cache_short'] = { + 'type':'memory', + 'data_dir':cache_dir, + 'expire':10, + 'start_time':start_time + } +cache_manager.regions['sql_cache_med'] = { + 'type':'memory', + 'data_dir':cache_dir, + 'expire':360, + 'start_time':start_time + } +cache_manager.regions['sql_cache_long'] = { + 'type':'file', + 'data_dir':cache_dir, + 'expire':3600, + 'start_time':start_time + } +#to use cache use this in query +#.options(FromCache("sqlalchemy_cache_type", "cachekey")) diff --git a/pylons_app/model/user_model.py b/pylons_app/model/user_model.py --- a/pylons_app/model/user_model.py +++ b/pylons_app/model/user_model.py @@ -2,7 +2,7 @@ # encoding: utf-8 # Model for users # Copyright (C) 2009-2010 Marcin Kuzminski - +# # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; version 2 @@ -23,10 +23,12 @@ Created on April 9, 2010 Model for users @author: marcink """ - +from pylons_app.lib import auth +from pylons.i18n.translation import _ +from pylons_app.lib.celerylib import tasks, run_task from pylons_app.model.db import User from pylons_app.model.meta import Session -from pylons.i18n.translation import _ +import traceback import logging log = logging.getLogger(__name__) @@ -43,7 +45,7 @@ class UserModel(object): def get_user(self, id): return self.sa.query(User).get(id) - def get_user_by_name(self,name): + def get_user_by_name(self, name): return self.sa.query(User).filter(User.username == name).scalar() def create(self, form_data): @@ -54,8 +56,8 @@ class UserModel(object): self.sa.add(new_user) self.sa.commit() - except Exception as e: - log.error(e) + except: + log.error(traceback.format_exc()) self.sa.rollback() raise @@ -68,8 +70,8 @@ class UserModel(object): self.sa.add(new_user) self.sa.commit() - except Exception as e: - log.error(e) + except: + log.error(traceback.format_exc()) self.sa.rollback() raise @@ -88,8 +90,8 @@ class UserModel(object): self.sa.add(new_user) self.sa.commit() - except Exception as e: - log.error(e) + except: + log.error(traceback.format_exc()) self.sa.rollback() raise @@ -109,13 +111,12 @@ class UserModel(object): self.sa.add(new_user) self.sa.commit() - except Exception as e: - log.error(e) + except: + log.error(traceback.format_exc()) self.sa.rollback() raise def delete(self, id): - try: user = self.sa.query(User).get(id) @@ -125,7 +126,10 @@ class UserModel(object): " crucial for entire application")) self.sa.delete(user) self.sa.commit() - except Exception as e: - log.error(e) + except: + log.error(traceback.format_exc()) self.sa.rollback() raise + + def reset_password(self, data): + run_task(tasks.reset_user_password, data['email']) diff --git a/pylons_app/public/css/style.css b/pylons_app/public/css/style.css --- a/pylons_app/public/css/style.css +++ b/pylons_app/public/css/style.css @@ -505,6 +505,33 @@ div.options a:hover /*ICONS*/ +#header #header-inner #quick li ul li a.journal, +#header #header-inner #quick li ul li a.journal:hover +{ + background:url("../images/icons/book.png") no-repeat scroll 4px 9px #FFFFFF; + margin:0; + padding:12px 9px 7px 24px; + width:167px; + +} +#header #header-inner #quick li ul li a.private_repo, +#header #header-inner #quick li ul li a.private_repo:hover +{ + background:url("../images/icons/lock.png") no-repeat scroll 4px 9px #FFFFFF; + margin:0; + padding:12px 9px 7px 24px; + width:167px; + +} +#header #header-inner #quick li ul li a.public_repo, +#header #header-inner #quick li ul li a.public_repo:hover +{ + background:url("../images/icons/lock_open.png") no-repeat scroll 4px 9px #FFFFFF; + margin:0; + padding:12px 9px 7px 24px; + width:167px; + +} #header #header-inner #quick li ul li a.repos, #header #header-inner #quick li ul li a.repos:hover @@ -2877,7 +2904,7 @@ div.form div.fields div.buttons input #register div.form div.fields div.buttons { margin: 0; - padding: 10px 0 0 97px; + padding: 10px 0 0 114px; clear: both; overflow: hidden; border-top: 1px solid #DDDDDD; diff --git a/pylons_app/templates/admin/admin_log.html b/pylons_app/templates/admin/admin_log.html --- a/pylons_app/templates/admin/admin_log.html +++ b/pylons_app/templates/admin/admin_log.html @@ -11,8 +11,8 @@ %for cnt,l in enumerate(c.users_log): - ${l.user.username} - ${l.repository} + ${h.link_to(l.user.username,h.url('edit_user', id=l.user.user_id))} + ${h.link_to(l.repository,h.url('summary_home',repo_name=l.repository))} ${l.action} ${l.action_date} ${l.user_ip} diff --git a/pylons_app/templates/admin/permissions/permissions.html b/pylons_app/templates/admin/permissions/permissions.html --- a/pylons_app/templates/admin/permissions/permissions.html +++ b/pylons_app/templates/admin/permissions/permissions.html @@ -29,7 +29,7 @@
- +
${h.select('default_perm','',c.perms_choices)} @@ -51,7 +51,7 @@
- +
${h.select('default_create','',c.create_choices)} diff --git a/pylons_app/templates/admin/settings/settings.html b/pylons_app/templates/admin/settings/settings.html --- a/pylons_app/templates/admin/settings/settings.html +++ b/pylons_app/templates/admin/settings/settings.html @@ -47,7 +47,32 @@
${h.end_form()} - + +

${_('Whoosh indexing')}

+ ${h.form(url('admin_setting', setting_id='whoosh'),method='put')} +
+ + +
+
+
+ +
+
+
+ ${h.checkbox('full_index',True)} + +
+
+
+ +
+ ${h.submit('reindex','reindex',class_="ui-button ui-widget ui-state-default ui-corner-all")} +
+
+
+ ${h.end_form()} +

${_('Global application settings')}

${h.form(url('admin_setting', setting_id='global'),method='put')}
diff --git a/pylons_app/templates/base/base.html b/pylons_app/templates/base/base.html --- a/pylons_app/templates/base/base.html +++ b/pylons_app/templates/base/base.html @@ -97,8 +97,12 @@ ↓
    - %for repo in c.repo_switcher_list: -
  • ${h.link_to(repo,h.url('summary_home',repo_name=repo))}
    • + %for repo,private in c.repo_switcher_list: + %if private: +
  • ${h.link_to(repo,h.url('summary_home',repo_name=repo),class_="private_repo")}
    • + %else: +
  • ${h.link_to(repo,h.url('summary_home',repo_name=repo),class_="public_repo")}
    • + %endif %endfor
@@ -203,6 +207,7 @@ ${_('Admin')}
    +
  • ${h.link_to(_('journal'),h.url('admin_home'),class_='journal')}
  • ${h.link_to(_('repositories'),h.url('repos'),class_='repos')}
  • ${h.link_to(_('users'),h.url('users'),class_='users')}
  • ${h.link_to(_('permissions'),h.url('edit_permission',id='default'),class_='permissions')}
    • diff --git a/pylons_app/templates/files/files_annotate.html b/pylons_app/templates/files/files_annotate.html --- a/pylons_app/templates/files/files_annotate.html +++ b/pylons_app/templates/files/files_annotate.html @@ -23,18 +23,22 @@
-

${_('Location')}: ${h.files_breadcrumbs(c.repo_name,c.cur_rev,c.file.path)}

+

${_('Location')}: ${h.files_breadcrumbs(c.repo_name,c.cur_rev,c.file.path)}

${_('Last revision')}
${h.link_to("r%s:%s" % (c.file.last_changeset.revision,c.file.last_changeset._short), h.url('files_annotate_home',repo_name=c.repo_name,revision=c.file.last_changeset._short,f_path=c.f_path))}
${_('Size')}
${h.format_byte_size(c.file.size,binary=True)}
+
${_('Mimetype')}
+
${c.file.mimetype}
${_('Options')}
${h.link_to(_('show source'), h.url('files_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.f_path))} + / ${h.link_to(_('show as raw'), + h.url('files_raw_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.f_path))} / ${h.link_to(_('download as raw'), - h.url('files_raw_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.f_path))} + h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.f_path))}
@@ -43,7 +47,12 @@
"${c.file_msg}"
- ${h.pygmentize_annotation(c.file,linenos=True,anchorlinenos=True,lineanchors='S',cssclass="code-highlight")} + % if c.file.size < c.file_size_limit: + ${h.pygmentize_annotation(c.file,linenos=True,anchorlinenos=True,lineanchors='S',cssclass="code-highlight")} + %else: + ${_('File is to big to display')} ${h.link_to(_('show as raw'), + h.url('files_raw_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.f_path))} + %endif
diff --git a/pylons_app/templates/files/files_browser.html b/pylons_app/templates/files/files_browser.html --- a/pylons_app/templates/files/files_browser.html +++ b/pylons_app/templates/files/files_browser.html @@ -23,31 +23,38 @@ ${_('Name')} ${_('Size')} + ${_('Mimetype')} ${_('Revision')} ${_('Last modified')} ${_('Last commiter')} - - - % if c.files_list.parent: - ${h.link_to('..',h.url('files_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.files_list.parent.path),class_="browser-dir")} - %endif - - - - - - + + % if c.files_list.parent: + + + ${h.link_to('..',h.url('files_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.files_list.parent.path),class_="browser-dir")} + + + + + + + + %endif + %for cnt,node in enumerate(c.files_list,1): ${h.link_to(node.name,h.url('files_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=node.path),class_=file_class(node))} - %if node.is_file(): - ${h.format_byte_size(node.size,binary=True)} - %endif + ${h.format_byte_size(node.size,binary=True)} + + + %if node.is_file(): + ${node.mimetype} + %endif %if node.is_file(): diff --git a/pylons_app/templates/files/files_source.html b/pylons_app/templates/files/files_source.html --- a/pylons_app/templates/files/files_source.html +++ b/pylons_app/templates/files/files_source.html @@ -6,11 +6,15 @@
${_('Size')}
${h.format_byte_size(c.files_list.size,binary=True)}
+
${_('Mimetype')}
+
${c.files_list.mimetype}
${_('Options')}
${h.link_to(_('show annotation'), - h.url('files_annotate_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.f_path))} + h.url('files_annotate_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.f_path))} + / ${h.link_to(_('show as raw'), + h.url('files_raw_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.f_path))} / ${h.link_to(_('download as raw'), - h.url('files_raw_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.f_path))} + h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.f_path))}
${_('History')}
@@ -32,7 +36,12 @@
"${c.files_list.last_changeset.message}"
- ${h.pygmentize(c.files_list,linenos=True,anchorlinenos=True,lineanchors='S',cssclass="code-highlight")} + % if c.files_list.size < c.file_size_limit: + ${h.pygmentize(c.files_list,linenos=True,anchorlinenos=True,lineanchors='S',cssclass="code-highlight")} + %else: + ${_('File is to big to display')} ${h.link_to(_('show as raw'), + h.url('files_raw_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.f_path))} + %endif
diff --git a/pylons_app/templates/login.html b/pylons_app/templates/login.html --- a/pylons_app/templates/login.html +++ b/pylons_app/templates/login.html @@ -60,7 +60,7 @@
- ${h.link_to(_('Forgot your password ?'),h.url('#'))} + ${h.link_to(_('Forgot your password ?'),h.url('reset_password'))} %if h.HasPermissionAny('hg.admin', 'hg.register.auto_activate', 'hg.register.manual_activate')(): / ${h.link_to(_("Don't have an account ?"),h.url('register'))} diff --git a/pylons_app/templates/password_reset.html b/pylons_app/templates/password_reset.html new file mode 100644 --- /dev/null +++ b/pylons_app/templates/password_reset.html @@ -0,0 +1,54 @@ +## -*- coding: utf-8 -*- + + + + ${_('Reset You password to hg-app')} + + + + + + + + + + + + + +
+ +
+
${_('Reset You password to hg-app')}
+
+
+
+
+ ${h.form(url('password_reset'))} +
+ +
+ +
+
+ +
+
+ ${h.text('email')} +
+
+ +
+
+ ${h.submit('send','Reset my password',class_="ui-button ui-widget ui-state-default ui-corner-all")} +
${_('Your new password will be send to matching email address')}
+
+
+
+
+ ${h.end_form()} +
+
+ + + diff --git a/pylons_app/templates/search/search.html b/pylons_app/templates/search/search.html --- a/pylons_app/templates/search/search.html +++ b/pylons_app/templates/search/search.html @@ -46,7 +46,7 @@ h.url('files_home',repo_name=sr['repository'],revision='tip',f_path=sr['f_path']))}
- 
${h.literal(sr['content_short'])}
+ 
${h.literal(sr['content_short_hl'])}
@@ -59,11 +59,13 @@ %endif - %endif + %endif %endfor - - - + %if c.cur_query: +
+ ${c.formated_results.pager('$link_previous ~2~ $link_next')} +
+ %endif diff --git a/pylons_app/templates/shortlog/shortlog_data.html b/pylons_app/templates/shortlog/shortlog_data.html --- a/pylons_app/templates/shortlog/shortlog_data.html +++ b/pylons_app/templates/shortlog/shortlog_data.html @@ -13,7 +13,7 @@ %for cnt,cs in enumerate(c.repo_changesets): - ${h.age(cs._ctx.date())} + ${h.age(cs._ctx.date())} - ${h.rfc822date_notz(cs._ctx.date())} ${h.person(cs.author)} r${cs.revision}:${cs.raw_id} diff --git a/pylons_app/templates/summary/summary.html b/pylons_app/templates/summary/summary.html --- a/pylons_app/templates/summary/summary.html +++ b/pylons_app/templates/summary/summary.html @@ -76,7 +76,9 @@ E.onDOMReady(function(e){
- ${h.age(c.repo_info.last_change)} - ${h.rfc822date(c.repo_info.last_change)} + ${h.age(c.repo_info.last_change)} - ${h.rfc822date(c.repo_info.last_change)} + ${_('by')} ${h.get_changeset_safe(c.repo_info,'tip').author} +
@@ -121,151 +123,356 @@ E.onDOMReady(function(e){
-
${_('Last month commit activity')}
+
${_('Commit activity by day / author')}
-
+
+
+ +
+ YAHOO.util.Event.on(choiceContainer.getElementsByTagName("input"), "click", plotchoiced, [data, initial_ranges]); + } + SummaryPlot(${c.ts_min},${c.ts_max},${c.commit_data|n},${c.overview_data|n}); +
diff --git a/pylons_app/tests/__init__.py b/pylons_app/tests/__init__.py --- a/pylons_app/tests/__init__.py +++ b/pylons_app/tests/__init__.py @@ -16,12 +16,18 @@ from routes.util import URLGenerator from webtest import TestApp import os from pylons_app.model import meta +import logging + + +log = logging.getLogger(__name__) + import pylons.test __all__ = ['environ', 'url', 'TestController'] # Invoke websetup with the current config file -SetupCommand('setup-app').run([pylons.test.pylonsapp.config['__file__']]) +#SetupCommand('setup-app').run([config_file]) + environ = {} @@ -33,13 +39,13 @@ class TestController(TestCase): self.app = TestApp(wsgiapp) url._push_object(URLGenerator(config['routes.map'], environ)) self.sa = meta.Session + TestCase.__init__(self, *args, **kwargs) - - def log_user(self): + def log_user(self, username='test_admin', password='test'): response = self.app.post(url(controller='login', action='index'), - {'username':'test_admin', - 'password':'test'}) + {'username':username, + 'password':password}) assert response.status == '302 Found', 'Wrong response code from login got %s' % response.status assert response.session['hg_app_user'].username == 'test_admin', 'wrong logged in user' - return response.follow() \ No newline at end of file + return response.follow() diff --git a/pylons_app/tests/functional/test_admin.py b/pylons_app/tests/functional/test_admin.py --- a/pylons_app/tests/functional/test_admin.py +++ b/pylons_app/tests/functional/test_admin.py @@ -3,5 +3,7 @@ from pylons_app.tests import * class TestAdminController(TestController): def test_index(self): + self.log_user() response = self.app.get(url(controller='admin/admin', action='index')) + assert 'Admin dashboard - journal' in response.body,'No proper title in dashboard' # Test response... diff --git a/pylons_app/tests/functional/test_admin_settings.py b/pylons_app/tests/functional/test_admin_settings.py --- a/pylons_app/tests/functional/test_admin_settings.py +++ b/pylons_app/tests/functional/test_admin_settings.py @@ -1,4 +1,5 @@ from pylons_app.tests import * +from pylons_app.model.db import User class TestSettingsController(TestController): @@ -41,3 +42,75 @@ class TestSettingsController(TestControl def test_edit_as_xml(self): response = self.app.get(url('formatted_admin_edit_setting', setting_id=1, format='xml')) + + def test_my_account(self): + self.log_user() + response = self.app.get(url('admin_settings_my_account')) + print response + assert 'value="test_admin' in response.body + + + + def test_my_account_update(self): + self.log_user() + new_email = 'new@mail.pl' + response = self.app.post(url('admin_settings_my_account_update'), params=dict( + _method='put', + username='test_admin', + new_password='test', + password='', + name='NewName', + lastname='NewLastname', + email=new_email,)) + response.follow() + print response + + print 'x' * 100 + print response.session + assert 'Your account was updated succesfully' in response.session['flash'][0][1], 'no flash message about success of change' + user = self.sa.query(User).filter(User.username == 'test_admin').one() + assert user.email == new_email , 'incorrect user email after update got %s vs %s' % (user.email, new_email) + + def test_my_account_update_own_email_ok(self): + self.log_user() + + new_email = 'new@mail.pl' + response = self.app.post(url('admin_settings_my_account_update'), params=dict( + _method='put', + username='test_admin', + new_password='test', + name='NewName', + lastname='NewLastname', + email=new_email,)) + print response + + def test_my_account_update_err_email_exists(self): + self.log_user() + + new_email = 'test_regular@mail.com'#already exisitn email + response = self.app.post(url('admin_settings_my_account_update'), params=dict( + _method='put', + username='test_admin', + new_password='test', + name='NewName', + lastname='NewLastname', + email=new_email,)) + print response + + assert 'That e-mail address is already taken' in response.body, 'Missing error message about existing email' + + + def test_my_account_update_err(self): + self.log_user() + + new_email = 'newmail.pl' + response = self.app.post(url('admin_settings_my_account_update'), params=dict( + _method='put', + username='test_regular2', + new_password='test', + name='NewName', + lastname='NewLastname', + email=new_email,)) + print response + assert 'An email address must contain a single @' in response.body, 'Missing error message about wrong email' + assert 'This username already exists' in response.body, 'Missing error message about existing user' diff --git a/pylons_app/tests/functional/test_branches.py b/pylons_app/tests/functional/test_branches.py --- a/pylons_app/tests/functional/test_branches.py +++ b/pylons_app/tests/functional/test_branches.py @@ -3,5 +3,6 @@ from pylons_app.tests import * class TestBranchesController(TestController): def test_index(self): + self.log_user() response = self.app.get(url(controller='branches', action='index',repo_name='vcs_test')) # Test response... diff --git a/pylons_app/tests/functional/test_changelog.py b/pylons_app/tests/functional/test_changelog.py --- a/pylons_app/tests/functional/test_changelog.py +++ b/pylons_app/tests/functional/test_changelog.py @@ -3,5 +3,6 @@ from pylons_app.tests import * class TestChangelogController(TestController): def test_index(self): + self.log_user() response = self.app.get(url(controller='changelog', action='index',repo_name='vcs_test')) # Test response... diff --git a/pylons_app/tests/functional/test_feed.py b/pylons_app/tests/functional/test_feed.py --- a/pylons_app/tests/functional/test_feed.py +++ b/pylons_app/tests/functional/test_feed.py @@ -3,11 +3,13 @@ from pylons_app.tests import * class TestFeedController(TestController): def test_rss(self): + self.log_user() response = self.app.get(url(controller='feed', action='rss', repo_name='vcs_test')) # Test response... def test_atom(self): + self.log_user() response = self.app.get(url(controller='feed', action='atom', repo_name='vcs_test')) # Test response... \ No newline at end of file diff --git a/pylons_app/tests/functional/test_files.py b/pylons_app/tests/functional/test_files.py --- a/pylons_app/tests/functional/test_files.py +++ b/pylons_app/tests/functional/test_files.py @@ -3,6 +3,7 @@ from pylons_app.tests import * class TestFilesController(TestController): def test_index(self): + self.log_user() response = self.app.get(url(controller='files', action='index', repo_name='vcs_test', revision='tip', diff --git a/pylons_app/tests/functional/test_login.py b/pylons_app/tests/functional/test_login.py --- a/pylons_app/tests/functional/test_login.py +++ b/pylons_app/tests/functional/test_login.py @@ -82,9 +82,9 @@ class TestLoginController(TestController def test_register_ok(self): - username = 'test_regular2' + username = 'test_regular4' password = 'qweqwe' - email = 'goodmail@mail.com' + email = 'marcin@test.com' name = 'testname' lastname = 'testlastname' @@ -94,18 +94,46 @@ class TestLoginController(TestController 'email':email, 'name':name, 'lastname':lastname}) - + print response.body assert response.status == '302 Found', 'Wrong response from register page got %s' % response.status + assert 'You have successfully registered into hg-app' in response.session['flash'][0], 'No flash message about user registration' - ret = self.sa.query(User).filter(User.username == 'test_regular2').one() + ret = self.sa.query(User).filter(User.username == 'test_regular4').one() assert ret.username == username , 'field mismatch %s %s' % (ret.username, username) - assert check_password(password,ret.password) == True , 'password mismatch' + assert check_password(password, ret.password) == True , 'password mismatch' assert ret.email == email , 'field mismatch %s %s' % (ret.email, email) assert ret.name == name , 'field mismatch %s %s' % (ret.name, name) assert ret.lastname == lastname , 'field mismatch %s %s' % (ret.lastname, lastname) + def test_forgot_password_wrong_mail(self): + response = self.app.post(url(controller='login', action='password_reset'), + {'email':'marcin@wrongmail.org', }) + + assert "That e-mail address doesn't exist" in response.body, 'Missing error message about wrong email' + + def test_forgot_password(self): + response = self.app.get(url(controller='login', action='password_reset')) + assert response.status == '200 OK', 'Wrong response from login page got %s' % response.status + + username = 'test_password_reset_1' + password = 'qweqwe' + email = 'marcin@python-works.com' + name = 'passwd' + lastname = 'reset' + + response = self.app.post(url(controller='login', action='register'), + {'username':username, + 'password':password, + 'email':email, + 'name':name, + 'lastname':lastname}) + #register new user for email test + response = self.app.post(url(controller='login', action='password_reset'), + {'email':email, }) + print response.session['flash'] + assert 'You have successfully registered into hg-app' in response.session['flash'][0], 'No flash message about user registration' + assert 'Your new password was sent' in response.session['flash'][1], 'No flash message about password reset' - diff --git a/pylons_app/tests/functional/test_search.py b/pylons_app/tests/functional/test_search.py --- a/pylons_app/tests/functional/test_search.py +++ b/pylons_app/tests/functional/test_search.py @@ -9,7 +9,7 @@ class TestSearchController(TestControlle self.log_user() response = self.app.get(url(controller='search', action='index')) print response.body - assert 'class="small" id="q" name="q" type="text"' in response.body,'Search box content error' + assert 'class="small" id="q" name="q" type="text"' in response.body, 'Search box content error' # Test response... def test_empty_search(self): @@ -18,12 +18,21 @@ class TestSearchController(TestControlle raise SkipTest('skipped due to existing index') else: self.log_user() - response = self.app.get(url(controller='search', action='index'),{'q':'vcs_test'}) - assert 'There is no index to search in. Please run whoosh indexer' in response.body,'No error message about empty index' + response = self.app.get(url(controller='search', action='index'), {'q':'vcs_test'}) + assert 'There is no index to search in. Please run whoosh indexer' in response.body, 'No error message about empty index' def test_normal_search(self): self.log_user() - response = self.app.get(url(controller='search', action='index'),{'q':'def+repo'}) + response = self.app.get(url(controller='search', action='index'), {'q':'def repo'}) print response.body - assert '9 results' in response.body,'no message about proper search results' + assert '10 results' in response.body, 'no message about proper search results' + assert 'Permission denied' not in response.body, 'Wrong permissions settings for that repo and user' + + def test_repo_search(self): + self.log_user() + response = self.app.get(url(controller='search', action='index'), {'q':'repository:vcs_test def test'}) + print response.body + assert '4 results' in response.body, 'no message about proper search results' + assert 'Permission denied' not in response.body, 'Wrong permissions settings for that repo and user' + diff --git a/pylons_app/tests/functional/test_settings.py b/pylons_app/tests/functional/test_settings.py --- a/pylons_app/tests/functional/test_settings.py +++ b/pylons_app/tests/functional/test_settings.py @@ -3,6 +3,7 @@ from pylons_app.tests import * class TestSettingsController(TestController): def test_index(self): + self.log_user() response = self.app.get(url(controller='settings', action='index', repo_name='vcs_test')) # Test response... diff --git a/pylons_app/tests/functional/test_shortlog.py b/pylons_app/tests/functional/test_shortlog.py --- a/pylons_app/tests/functional/test_shortlog.py +++ b/pylons_app/tests/functional/test_shortlog.py @@ -3,5 +3,6 @@ from pylons_app.tests import * class TestShortlogController(TestController): def test_index(self): + self.log_user() response = self.app.get(url(controller='shortlog', action='index',repo_name='vcs_test')) # Test response... diff --git a/pylons_app/tests/functional/test_summary.py b/pylons_app/tests/functional/test_summary.py --- a/pylons_app/tests/functional/test_summary.py +++ b/pylons_app/tests/functional/test_summary.py @@ -3,5 +3,6 @@ from pylons_app.tests import * class TestSummaryController(TestController): def test_index(self): + self.log_user() response = self.app.get(url(controller='summary', action='index',repo_name='vcs_test')) # Test response... diff --git a/pylons_app/tests/functional/test_tags.py b/pylons_app/tests/functional/test_tags.py --- a/pylons_app/tests/functional/test_tags.py +++ b/pylons_app/tests/functional/test_tags.py @@ -3,5 +3,6 @@ from pylons_app.tests import * class TestTagsController(TestController): def test_index(self): + self.log_user() response = self.app.get(url(controller='tags', action='index',repo_name='vcs_test')) # Test response... diff --git a/pylons_app/websetup.py b/pylons_app/websetup.py --- a/pylons_app/websetup.py +++ b/pylons_app/websetup.py @@ -1,40 +1,25 @@ """Setup the pylons_app application""" -from os.path import dirname as dn, join as jn +from os.path import dirname as dn from pylons_app.config.environment import load_environment from pylons_app.lib.db_manage import DbManage -import datetime -from time import mktime import logging import os import sys -import tarfile log = logging.getLogger(__name__) ROOT = dn(dn(os.path.realpath(__file__))) sys.path.append(ROOT) + def setup_app(command, conf, vars): """Place any commands to setup pylons_app here""" log_sql = True tests = False - - dbname = os.path.split(conf['sqlalchemy.db1.url'])[-1] - filename = os.path.split(conf.filename)[-1] + REPO_TEST_PATH = None - if filename == 'tests.ini': - uniq_suffix = str(int(mktime(datetime.datetime.now().timetuple()))) - REPO_TEST_PATH = '/tmp/hg_app_test_%s' % uniq_suffix - - if not os.path.isdir(REPO_TEST_PATH): - os.mkdir(REPO_TEST_PATH) - cur_dir = dn(os.path.abspath(__file__)) - tar = tarfile.open(jn(cur_dir,'tests',"vcs_test.tar.gz")) - tar.extractall(REPO_TEST_PATH) - tar.close() - - tests = True + dbname = os.path.split(conf['sqlalchemy.db1.url'])[-1] dbmanage = DbManage(log_sql, dbname, tests) dbmanage.create_tables(override=True) diff --git a/setup.cfg b/setup.cfg --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,7 @@ find_links = http://www.pylonshq.com/dow [nosetests] verbose=True verbosity=2 -with-pylons=tests.ini +with-pylons=test.ini detailed-errors=1 # Babel configuration diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ except ImportError: from setuptools import setup, find_packages setup( - name='HgApp-%s'%get_version(), + name='HgApp-%s' % get_version(), version=get_version(), description='Mercurial repository serving and browsing app', keywords='mercurial web hgwebdir replacement serving hgweb', @@ -20,12 +20,13 @@ setup( "SQLAlchemy>=0.6", "babel", "Mako>=0.3.2", - "vcs>=0.1.4", + "vcs>=0.1.5", "pygments>=1.3.0", "mercurial>=1.6", "pysqlite", - "whoosh==1.0.0b10", + "whoosh==1.0.0b17", "py-bcrypt", + "celery", ], setup_requires=["PasteScript>=1.6.3"], packages=find_packages(exclude=['ez_setup']), diff --git a/tests.ini b/test.ini rename from tests.ini rename to test.ini --- a/tests.ini +++ b/test.ini @@ -1,28 +1,33 @@ ################################################################################ ################################################################################ -# pylons_app - Pylons environment configuration # +# hg-app - Pylons environment configuration # # # # The %(here)s variable will be replaced with the parent directory of this file# ################################################################################ [DEFAULT] debug = true -############################################ -## Uncomment and replace with the address ## -## which should receive any error reports ## -############################################ +################################################################################ +## Uncomment and replace with the address which should receive ## +## any error reports after application crash ## +## Additionally those settings will be used by hg-app mailing system ## +################################################################################ #email_to = admin@localhost +#error_email_from = paste_error@localhost +#app_email_from = hg-app-noreply@localhost +#error_message = + #smtp_server = mail.server.com -#error_email_from = paste_error@localhost #smtp_username = #smtp_password = -#error_message = 'mercurial crash !' +#smtp_port = +#smtp_use_tls = false [server:main] ##nr of threads to spawn threadpool_workers = 5 -##max request before +##max request before thread respawn threadpool_max_requests = 2 ##option to use threads of process @@ -56,7 +61,7 @@ beaker.cache.super_short_term.expire=10 ### BEAKER SESSION #### #################################### ## Type of storage used for the session, current types are -## “dbm”, “file”, “memcached”, “database”, and “memory”. +## "dbm", "file", "memcached", "database", and "memory". ## The storage uses the Container API ##that is also used by the cache system. beaker.session.type = file