fixed lock decorator bug which didn't release the lock after func execution and rewrote the pidlock a little with Ask Solem suggestions!

added tredning languages stats

celeryconfig.py

@@ -36,7 +36,7 b' BROKER_PASSWORD = "qweqwe"'
 CELERYD_CONCURRENCY = 2
 # CELERYD_LOG_FILE = "celeryd.log"
 CELERYD_LOG_LEVEL = "DEBUG"
-CELERYD_MAX_TASKS_PER_CHILD = 1
+CELERYD_MAX_TASKS_PER_CHILD = 3
 
 #Tasks will never be sent to the queue, but executed locally instead.
 CELERY_ALWAYS_EAGER = False

pylons_app/controllers/summary.py

@@ -35,7 +35,7 b' from datetime import datetime, timedelta'
 from time import mktime
 import calendar
 import logging
-
+import json
 log = logging.getLogger(__name__)
 
 class SummaryController(BaseController):
@@ -79,18 +79,25 b' class SummaryController(BaseController):'
         c.ts_min = ts_min_m
         c.ts_max = ts_max_y
         
-        
         stats = self.sa.query(Statistics)\
             .filter(Statistics.repository == c.repo_info.dbrepo)\
             .scalar()
-
-        if stats:
+        
+        
+        if stats and stats.languages:
+            lang_stats = json.loads(stats.languages)
             c.commit_data = stats.commit_activity
             c.overview_data = stats.commit_activity_combined
+            c.trending_languages = json.dumps(OrderedDict(
+                                       sorted(lang_stats.items(), reverse=True,
+                                            key=lambda k: k[1])[:2]
+                                        )
+                                    )
+            print c.trending_languages
         else:
-            import json
             c.commit_data = json.dumps({})
             c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 0] ])
+            c.trending_languages = json.dumps({})
         
         return render('summary/summary.html')
 

pylons_app/lib/celerylib/__init__.py

@@ -42,7 +42,7 b' def locked_task(func):'
         log.info('running task with lockkey %s', lockkey)
         try:
             l = DaemonLock(lockkey)
-            return func(*fargs, **fkwargs)
+            func(*fargs, **fkwargs)
             l.release()
         except LockHeld:
             log.info('LockHeld')

pylons_app/lib/celerylib/tasks.py

@@ -1,16 +1,16 b''
 from celery.decorators import task
 from celery.task.sets import subtask
 from celeryconfig import PYLONS_CONFIG as config
+from operator import itemgetter
 from pylons.i18n.translation import _
 from pylons_app.lib.celerylib import run_task, locked_task
 from pylons_app.lib.helpers import person
 from pylons_app.lib.smtp_mailer import SmtpMailer
 from pylons_app.lib.utils import OrderedDict
-from operator import itemgetter
+from time import mktime
 from vcs.backends.hg import MercurialRepository
-from time import mktime
+import json
 import traceback
-import json
 
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
@@ -75,10 +75,10 b' def whoosh_index(repo_location, full_ind'
 @task
 @locked_task
 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
-    author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
-        
     from pylons_app.model.db import Statistics, Repository
     log = get_commits_stats.get_logger()
+    author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
+    
     commits_by_day_author_aggregate = {}
     commits_by_day_aggregate = {}
     repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
@@ -99,7 +99,7 b' def get_commits_stats(repo_name, ts_min_'
     if cur_stats:
         last_rev = cur_stats.stat_on_revision
     
-    if last_rev == repo.revisions[-1]:
+    if last_rev == repo.revisions[-1] and len(repo.revisions) > 1:
         #pass silently without any work
         return True
     
@@ -109,6 +109,7 b' def get_commits_stats(repo_name, ts_min_'
                                         cur_stats.commit_activity_combined))
         commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity)
     
+    log.debug('starting parsing %s', parse_limit)
     for cnt, rev in enumerate(repo.revisions[last_rev:]):
         last_cs = cs = repo.get_changeset(rev)
         k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1],
@@ -187,9 +188,16 b' def get_commits_stats(repo_name, ts_min_'
     stats = cur_stats if cur_stats else Statistics()
     stats.commit_activity = json.dumps(commits_by_day_author_aggregate)
     stats.commit_activity_combined = json.dumps(overview_data)
+
+    log.debug('last revison %s', last_rev)
+    leftovers = len(repo.revisions[last_rev:])
+    log.debug('revisions to parse %s', leftovers)
+    
+    if last_rev == 0 or leftovers < parse_limit:    
+        stats.languages = json.dumps(__get_codes_stats(repo_name))
+        
     stats.repository = dbrepo
     stats.stat_on_revision = last_cs.revision
-    stats.languages = json.dumps({'_TOTAL_':0, '':0})
     
     try:
         sa.add(stats)
@@ -198,8 +206,8 b' def get_commits_stats(repo_name, ts_min_'
         log.error(traceback.format_exc())
         sa.rollback()
         return False
-    
-    run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
+    if len(repo.revisions) > 1:
+        run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
                             
     return True
 
@@ -259,3 +267,31 b' def send_email(recipients, subject, body'
         log.error(traceback.format_exc())
         return False
     return True
+
+def __get_codes_stats(repo_name):
+    LANGUAGES_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',
+                    'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el', 'erl',
+                    'h', 'java', 'js', 'jsp', 'jspx', 'lisp',
+                    'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3',
+                    'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh',
+                    'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt',
+                    'yaws']
+    repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
+    repo = MercurialRepository(repos_path + repo_name)
+
+    code_stats = {}
+    for topnode, dirs, files in repo.walk('/', 'tip'):
+        for f in files:
+            k = f.mimetype
+            if f.extension in LANGUAGES_EXTENSIONS:
+                if code_stats.has_key(k):
+                    code_stats[k] += 1
+                else:
+                    code_stats[k] = 1
+                    
+    return code_stats or {}
+
+
+            
+
+

pylons_app/lib/pidlock.py

@@ -1,6 +1,8 b''
 import os, time
 import sys
 from warnings import warn
+from multiprocessing.util import Finalize
+import errno
 
 class LockHeld(Exception):pass
 
@@ -27,55 +29,67 b' class DaemonLock(object):'
         self.held = False
         #run the lock automatically !
         self.lock()
-
-    def __del__(self):
-        if self.held:
+        self._finalize = Finalize(self, DaemonLock._on_finalize,
+                                    args=(self, debug), exitpriority=10)
 
-#            warn("use lock.release instead of del lock",
-#                    category = DeprecationWarning,
-#                    stacklevel = 2)
-
-            # ensure the lock will be removed
-            self.release()
+    @staticmethod
+    def _on_finalize(lock, debug):
+        if lock.held:
+            if debug:
+                print 'leck held finilazing and running lock.release()'
+            lock.release()
 
 
     def lock(self):
         """locking function, if lock is present it will raise LockHeld exception
         """
         lockname = '%s' % (os.getpid())
-
+        if self.debug:
+            print 'running lock'
         self.trylock()
         self.makelock(lockname, self.pidfile)
         return True
 
     def trylock(self):
         running_pid = False
+        if self.debug:
+            print 'checking for already running process'
         try:
             pidfile = open(self.pidfile, "r")
             pidfile.seek(0)
-            running_pid = pidfile.readline()
+            running_pid = int(pidfile.readline())
+            
+            pidfile.close()
+            
             if self.debug:
                 print 'lock file present running_pid: %s, checking for execution'\
                 % running_pid
             # Now we check the PID from lock file matches to the current
             # process PID
             if running_pid:
-                if os.path.exists("/proc/%s" % running_pid):
-                        print "You already have an instance of the program running"
-                        print "It is running as process %s" % running_pid
-                        raise LockHeld
+                try:
+                    os.kill(running_pid, 0)
+                except OSError, exc:
+                    if exc.errno in (errno.ESRCH, errno.EPERM):
+                        print "Lock File is there but the program is not running"
+                        print "Removing lock file for the: %s" % running_pid                        
+                        self.release()
+                    raise
                 else:
-                        print "Lock File is there but the program is not running"
-                        print "Removing lock file for the: %s" % running_pid
-                        self.release()
+                    print "You already have an instance of the program running"
+                    print "It is running as process %s" % running_pid                    
+                    raise LockHeld()
+                         
         except IOError, e:
             if e.errno != 2:
                 raise
 
-
     def release(self):
         """releases the pid by removing the pidfile
         """
+        if self.debug:
+            print 'trying to release the pidlock'
+            
         if self.callbackfn:
             #execute callback function on release
             if self.debug:

pylons_app/public/css/style.css

@@ -2963,6 +2963,28 b' div.form div.fields div.buttons input'
 /* -----------------------------------------------------------
 	SUMMARY
 ----------------------------------------------------------- */
+.trending_language_tbl, .trending_language_tbl td {
+	margin:  0px !important;
+	padding: 0px !important;
+	border: 0 !important;
+
+}
+.trending_language{
+	-moz-border-radius-bottomright:4px;
+	-moz-border-radius-topright:4px;
+	border-bottom-right-radius: 4px 4px;
+	border-top-right-radius: 4px 4px;
+	background-color:#336699;
+	color:#FFFFFF;
+	display:block;
+	min-width:20px;
+	max-width:400px;
+	padding:3px;
+	text-decoration:none;
+	height: 10px;
+	margin-bottom: 4px;
+	margin-left: 5px;
+}
 
 #clone_url{
 	border: none;

pylons_app/templates/summary/summary.html

@@ -90,7 +90,62 b' E.onDOMReady(function(e){'
 			      <input type="text" id="clone_url" readonly="readonly" value="hg clone ${c.clone_repo_url}" size="70"/>
 			  </div>
 			 </div>
-			
+			 
+			 <div class="field">
+			  <div class="label">
+			      <label>${_('Trending languages')}:</label>
+			  </div>
+			  <div class="input-short">
+			    <div id="lang_stats">
+			    
+			    </div> 			  
+			  	<script type="text/javascript">
+			  		var data = ${c.trending_languages|n};
+			  		var total = 0;
+			  		var no_data = true;
+			  		for (k in data){
+			  		    total += data[k];
+			  		    no_data = false;
+			  		} 
+					var tbl = document.createElement('table');
+					tbl.setAttribute('class','trending_language_tbl');
+			  		for (k in data){
+				  		var tr = document.createElement('tr');
+				  		var percentage = Math.round((data[k]/total*100),2);
+						var value = data[k];
+				  		var td1 = document.createElement('td');
+				  		td1.width=150;
+				  		var trending_language_label = document.createElement('div');
+				  		trending_language_label.innerHTML = k;
+				  		td1.appendChild(trending_language_label);
+
+				  		var td2 = document.createElement('td');
+			  		    var trending_language = document.createElement('div');
+			  		    trending_language.title = k;
+			  		    trending_language.innerHTML = "<b>"+value+" ${_('files')} - "+percentage+" %</b>";
+			  		    trending_language.setAttribute("class", 'trending_language');
+			  		    trending_language.style.width=percentage+"%";
+						td2.appendChild(trending_language);
+						
+						tr.appendChild(td1);
+						tr.appendChild(td2);
+			  		    tbl.appendChild(tr);
+ 				  		//YAHOO.util.Dom.get('lang_stats').appendChild(trending_language_label);
+			  		    
+			  		}
+			  		if(no_data){
+			  			var tr = document.createElement('tr');
+			  			var td1 = document.createElement('td');
+			  			td1.innerHTML = "${_('No data loaded yet...')}";
+			  			tr.appendChild(td1);
+			  			tbl.appendChild(tr);
+					}
+			  		YAHOO.util.Dom.get('lang_stats').appendChild(tbl);
+			  	</script>
+ 
+			  </div>
+			 </div>
+			 			
 			 <div class="field">
 			  <div class="label">
 			      <label>${_('Download')}:</label>