Implemented locking for task, to prevent for running the same tasks,

moved out pidlock library. Added dirsize display

pylons_app/lib/celerylib/__init__.py

@@ -1,9 +1,11 b''
+from pylons_app.lib.pidlock import DaemonLock, LockHeld
 from vcs.utils.lazy import LazyProperty
+from decorator import decorator
 import logging
 import os
 import sys
 import traceback
-
+from hashlib import md5
 log = logging.getLogger(__name__)
 
 class ResultWrapper(object):
@@ -20,10 +22,45 b' def run_task(task, *args, **kwargs):'
         log.info('running task %s', t.task_id)
         return t
     except Exception, e:
+        print e
         if e.errno == 111:
             log.debug('Unnable to connect. Sync execution')
         else:
             log.error(traceback.format_exc())
         #pure sync version
         return ResultWrapper(task(*args, **kwargs))
+
+
+class LockTask(object):
+    """LockTask decorator"""
     
+    def __init__(self, func):
+        self.func = func
+        
+    def __call__(self, func):
+        return decorator(self.__wrapper, func)
+    
+    def __wrapper(self, func, *fargs, **fkwargs):
+        params = []
+        params.extend(fargs)
+        params.extend(fkwargs.values())
+        lockkey = 'task_%s' % \
+           md5(str(self.func) + '-' + '-'.join(map(str, params))).hexdigest()
+        log.info('running task with lockkey %s', lockkey)
+        try:
+            l = DaemonLock(lockkey)
+            return func(*fargs, **fkwargs)
+            l.release()
+        except LockHeld:
+            log.info('LockHeld')
+            return 'Task with key %s already running' % lockkey   
+
+            
+            
+
+        
+        
+    
+    
+    
+  

pylons_app/lib/celerylib/tasks.py

@@ -2,7 +2,7 b' from celery.decorators import task'
 from celery.task.sets import subtask
 from celeryconfig import PYLONS_CONFIG as config
 from pylons.i18n.translation import _
-from pylons_app.lib.celerylib import run_task
+from pylons_app.lib.celerylib import run_task, LockTask
 from pylons_app.lib.helpers import person
 from pylons_app.lib.smtp_mailer import SmtpMailer
 from pylons_app.lib.utils import OrderedDict
@@ -68,7 +68,7 b' def get_hg_ui_settings():'
 @task
 def whoosh_index(repo_location, full_index):
     log = whoosh_index.get_logger()
-    from pylons_app.lib.indexers import DaemonLock
+    from pylons_app.lib.pidlock import DaemonLock
     from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon, LockHeld
     try:
         l = DaemonLock()
@@ -80,7 +80,9 b' def whoosh_index(repo_location, full_ind'
         log.info('LockHeld')
         return 'LockHeld'    
 
+
 @task
+@LockTask('get_commits_stats')
 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
     author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
         
@@ -92,7 +94,7 b' def get_commits_stats(repo_name, ts_min_'
     repo = MercurialRepository(repos_path + repo_name)
 
     skip_date_limit = True
-    parse_limit = 500 #limit for single task changeset parsing
+    parse_limit = 350 #limit for single task changeset parsing
     last_rev = 0
     last_cs = None
     timegetter = itemgetter('time')
@@ -205,7 +207,9 b' def get_commits_stats(repo_name, ts_min_'
         log.error(traceback.format_exc())
         sa.rollback()
         return False
-                        
+    
+    run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
+                            
     return True
 
 @task

pylons_app/lib/indexers/__init__.py

@@ -1,5 +1,4 b''
 from os.path import dirname as dn, join as jn
-from pidlock import LockHeld, DaemonLock
 from pylons_app.config.environment import load_environment
 from pylons_app.model.hg_model import HgModel
 from shutil import rmtree

pylons_app/lib/indexers/daemon.py

@@ -32,7 +32,7 b' from os.path import join as jn'
 project_path = dn(dn(dn(dn(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 
-from pidlock import LockHeld, DaemonLock
+from pylons_app.lib.pidlock import LockHeld, DaemonLock
 from pylons_app.model.hg_model import HgModel
 from pylons_app.lib.helpers import safe_unicode
 from whoosh.index import create_in, open_dir

pylons_app/lib/pidlock.py

@@ -6,7 +6,7 b' class LockHeld(Exception):pass'
 
 
 class DaemonLock(object):
-    '''daemon locking
+    """daemon locking
     USAGE:
     try:
         l = lock()
@@ -14,7 +14,7 b' class DaemonLock(object):'
         l.release()
     except LockHeld:
         sys.exit(1)
-    '''
+    """
 
     def __init__(self, file=None, callbackfn=None,
                  desc='daemon lock', debug=False):
@@ -40,9 +40,9 b' class DaemonLock(object):'
 
 
     def lock(self):
-        '''
+        """
         locking function, if lock is present it will raise LockHeld exception
-        '''
+        """
         lockname = '%s' % (os.getpid())
 
         self.trylock()
@@ -75,9 +75,9 b' class DaemonLock(object):'
 
 
     def release(self):
-        '''
+        """
         releases the pid by removing the pidfile
-        '''
+        """
         if self.callbackfn:
             #execute callback function on release
             if self.debug:
@@ -94,11 +94,11 b' class DaemonLock(object):'
             pass
 
     def makelock(self, lockname, pidfile):
-        '''
+        """
         this function will make an actual lock
         @param lockname: acctual pid of file
         @param pidfile: the file to write the pid in
-        '''
+        """
         if self.debug:
             print 'creating a file %s and pid: %s' % (pidfile, lockname)
         pidfile = open(self.pidfile, "wb")

pylons_app/lib/utils.py

@@ -374,9 +374,8 b' def create_test_index(repo_location, ful'
     @param repo_location:
     @param full_index:
     """
-    from pylons_app.lib.indexers import daemon
     from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon
-    from pylons_app.lib.indexers.pidlock import DaemonLock, LockHeld
+    from pylons_app.lib.pidlock import DaemonLock, LockHeld
     from pylons_app.lib.indexers import IDX_LOCATION
     import shutil
     

pylons_app/templates/files/files_browser.html

@@ -29,26 +29,27 b''
 		                 <th>${_('Last commiter')}</th>
 		             </tr>
 		         </thead>
-		         	<tr class="parity0">
-		          		<td>
-		          		% if c.files_list.parent:
-		          			${h.link_to('..',h.url('files_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.files_list.parent.path),class_="browser-dir")}
-		          		%endif
-		          		</td>
-		          		<td></td>
-		          		<td></td>
-		          		<td></td>
-		          		<td></td>
-		         	</tr>
+
+          		% if c.files_list.parent:
+         		<tr class="parity0">
+	          		<td>		          		
+	          			${h.link_to('..',h.url('files_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=c.files_list.parent.path),class_="browser-dir")}
+	          		</td>
+	          		<td></td>
+	          		<td></td>
+	          		<td></td>
+	          		<td></td>
+	          		<td></td>
+				</tr>	          		
+          		%endif
+		         	
 		    %for cnt,node in enumerate(c.files_list,1):
 				<tr class="parity${cnt%2}">
 		             <td>
 						${h.link_to(node.name,h.url('files_home',repo_name=c.repo_name,revision=c.cur_rev,f_path=node.path),class_=file_class(node))}
 		             </td>
 		             <td>
-		                %if node.is_file():
-		             		${h.format_byte_size(node.size,binary=True)}
-		             	%endif
+		             	${h.format_byte_size(node.size,binary=True)}
 		             </td>
 		             <td>
 		              %if node.is_file():