# HG changeset patch # User Marcin Kuzminski # Date 2010-09-22 23:08:33 # Node ID fb0c3af6031b71a7861e064c0d0f06f3232b3610 # Parent 47f4c7ff245be5c985c55a40a43236b16aacf31b Implemented locking for task, to prevent for running the same tasks, moved out pidlock library. Added dirsize display diff --git a/pylons_app/lib/celerylib/__init__.py b/pylons_app/lib/celerylib/__init__.py --- a/pylons_app/lib/celerylib/__init__.py +++ b/pylons_app/lib/celerylib/__init__.py @@ -1,9 +1,11 @@ +from pylons_app.lib.pidlock import DaemonLock, LockHeld from vcs.utils.lazy import LazyProperty +from decorator import decorator import logging import os import sys import traceback - +from hashlib import md5 log = logging.getLogger(__name__) class ResultWrapper(object): @@ -20,10 +22,45 @@ def run_task(task, *args, **kwargs): log.info('running task %s', t.task_id) return t except Exception, e: + print e if e.errno == 111: log.debug('Unnable to connect. Sync execution') else: log.error(traceback.format_exc()) #pure sync version return ResultWrapper(task(*args, **kwargs)) + + +class LockTask(object): + """LockTask decorator""" + def __init__(self, func): + self.func = func + + def __call__(self, func): + return decorator(self.__wrapper, func) + + def __wrapper(self, func, *fargs, **fkwargs): + params = [] + params.extend(fargs) + params.extend(fkwargs.values()) + lockkey = 'task_%s' % \ + md5(str(self.func) + '-' + '-'.join(map(str, params))).hexdigest() + log.info('running task with lockkey %s', lockkey) + try: + l = DaemonLock(lockkey) + return func(*fargs, **fkwargs) + l.release() + except LockHeld: + log.info('LockHeld') + return 'Task with key %s already running' % lockkey + + + + + + + + + + diff --git a/pylons_app/lib/celerylib/tasks.py b/pylons_app/lib/celerylib/tasks.py --- a/pylons_app/lib/celerylib/tasks.py +++ b/pylons_app/lib/celerylib/tasks.py @@ -2,7 +2,7 @@ from celery.decorators import task from celery.task.sets import subtask from celeryconfig import PYLONS_CONFIG as config from pylons.i18n.translation import _ -from pylons_app.lib.celerylib import run_task +from pylons_app.lib.celerylib import run_task, LockTask from pylons_app.lib.helpers import person from pylons_app.lib.smtp_mailer import SmtpMailer from pylons_app.lib.utils import OrderedDict @@ -68,7 +68,7 @@ def get_hg_ui_settings(): @task def whoosh_index(repo_location, full_index): log = whoosh_index.get_logger() - from pylons_app.lib.indexers import DaemonLock + from pylons_app.lib.pidlock import DaemonLock from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon, LockHeld try: l = DaemonLock() @@ -80,7 +80,9 @@ def whoosh_index(repo_location, full_ind log.info('LockHeld') return 'LockHeld' + @task +@LockTask('get_commits_stats') def get_commits_stats(repo_name, ts_min_y, ts_max_y): author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty @@ -92,7 +94,7 @@ def get_commits_stats(repo_name, ts_min_ repo = MercurialRepository(repos_path + repo_name) skip_date_limit = True - parse_limit = 500 #limit for single task changeset parsing + parse_limit = 350 #limit for single task changeset parsing last_rev = 0 last_cs = None timegetter = itemgetter('time') @@ -205,7 +207,9 @@ def get_commits_stats(repo_name, ts_min_ log.error(traceback.format_exc()) sa.rollback() return False - + + run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) + return True @task diff --git a/pylons_app/lib/indexers/__init__.py b/pylons_app/lib/indexers/__init__.py --- a/pylons_app/lib/indexers/__init__.py +++ b/pylons_app/lib/indexers/__init__.py @@ -1,5 +1,4 @@ from os.path import dirname as dn, join as jn -from pidlock import LockHeld, DaemonLock from pylons_app.config.environment import load_environment from pylons_app.model.hg_model import HgModel from shutil import rmtree diff --git a/pylons_app/lib/indexers/daemon.py b/pylons_app/lib/indexers/daemon.py --- a/pylons_app/lib/indexers/daemon.py +++ b/pylons_app/lib/indexers/daemon.py @@ -32,7 +32,7 @@ from os.path import join as jn project_path = dn(dn(dn(dn(os.path.realpath(__file__))))) sys.path.append(project_path) -from pidlock import LockHeld, DaemonLock +from pylons_app.lib.pidlock import LockHeld, DaemonLock from pylons_app.model.hg_model import HgModel from pylons_app.lib.helpers import safe_unicode from whoosh.index import create_in, open_dir diff --git a/pylons_app/lib/indexers/pidlock.py b/pylons_app/lib/pidlock.py rename from pylons_app/lib/indexers/pidlock.py rename to pylons_app/lib/pidlock.py --- a/pylons_app/lib/indexers/pidlock.py +++ b/pylons_app/lib/pidlock.py @@ -6,7 +6,7 @@ class LockHeld(Exception):pass class DaemonLock(object): - '''daemon locking + """daemon locking USAGE: try: l = lock() @@ -14,7 +14,7 @@ class DaemonLock(object): l.release() except LockHeld: sys.exit(1) - ''' + """ def __init__(self, file=None, callbackfn=None, desc='daemon lock', debug=False): @@ -40,9 +40,9 @@ class DaemonLock(object): def lock(self): - ''' + """ locking function, if lock is present it will raise LockHeld exception - ''' + """ lockname = '%s' % (os.getpid()) self.trylock() @@ -75,9 +75,9 @@ class DaemonLock(object): def release(self): - ''' + """ releases the pid by removing the pidfile - ''' + """ if self.callbackfn: #execute callback function on release if self.debug: @@ -94,11 +94,11 @@ class DaemonLock(object): pass def makelock(self, lockname, pidfile): - ''' + """ this function will make an actual lock @param lockname: acctual pid of file @param pidfile: the file to write the pid in - ''' + """ if self.debug: print 'creating a file %s and pid: %s' % (pidfile, lockname) pidfile = open(self.pidfile, "wb") diff --git a/pylons_app/lib/utils.py b/pylons_app/lib/utils.py --- a/pylons_app/lib/utils.py +++ b/pylons_app/lib/utils.py @@ -374,9 +374,8 @@ def create_test_index(repo_location, ful @param repo_location: @param full_index: """ - from pylons_app.lib.indexers import daemon from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon - from pylons_app.lib.indexers.pidlock import DaemonLock, LockHeld + from pylons_app.lib.pidlock import DaemonLock, LockHeld from pylons_app.lib.indexers import IDX_LOCATION import shutil diff --git a/pylons_app/templates/files/files_browser.html b/pylons_app/templates/files/files_browser.html --- a/pylons_app/templates/files/files_browser.html +++ b/pylons_app/templates/files/files_browser.html @@ -29,26 +29,27 @@ ${_('Last commiter')} - - - % if c.files_list.parent: - ${h.link_to('..',h.url('files_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.files_list.parent.path),class_="browser-dir")} - %endif - - - - - - + + % if c.files_list.parent: + + + ${h.link_to('..',h.url('files_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.files_list.parent.path),class_="browser-dir")} + + + + + + + + %endif + %for cnt,node in enumerate(c.files_list,1): ${h.link_to(node.name,h.url('files_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=node.path),class_=file_class(node))} - %if node.is_file(): - ${h.format_byte_size(node.size,binary=True)} - %endif + ${h.format_byte_size(node.size,binary=True)} %if node.is_file():