implemented basic autoupdating statistics fetched from database

celeryconfig.py

@@ -1,42 +1,74
 # List of modules to import when celery starts.
 import sys
 import os
 import ConfigParser
 root = os.getcwd()
 
 PYLONS_CONFIG_NAME = 'development.ini'
 
 sys.path.append(root)
 config = ConfigParser.ConfigParser({'here':root})
 config.read('%s/%s' % (root, PYLONS_CONFIG_NAME))
 PYLONS_CONFIG = config
 
 CELERY_IMPORTS = ("pylons_app.lib.celerylib.tasks",)
 
 ## Result store settings.
 CELERY_RESULT_BACKEND = "database"
 CELERY_RESULT_DBURI = dict(config.items('app:main'))['sqlalchemy.db1.url']
+CELERY_RESULT_SERIALIZER = 'json'
+
 
 BROKER_CONNECTION_MAX_RETRIES = 30
 
 ## Broker settings.
 BROKER_HOST = "localhost"
 BROKER_PORT = 5672
 BROKER_VHOST = "rabbitmqhost"
 BROKER_USER = "rabbitmq"
 BROKER_PASSWORD = "qweqwe"
 
 ## Worker settings
 ## If you're doing mostly I/O you can have more processes,
 ## but if mostly spending CPU, try to keep it close to the
 ## number of CPUs on your machine. If not set, the number of CPUs/cores
 ## available will be used.
 CELERYD_CONCURRENCY = 2
 # CELERYD_LOG_FILE = "celeryd.log"
 CELERYD_LOG_LEVEL = "DEBUG"
 CELERYD_MAX_TASKS_PER_CHILD = 1
 
-#CELERY_ALWAYS_EAGER = True
-#rabbitmqctl add_user rabbitmq qweqwe
-#rabbitmqctl add_vhost rabbitmqhost
-#rabbitmqctl set_permissions -p rabbitmqhost rabbitmq ".*" ".*" ".*"
+#Tasks will never be sent to the queue, but executed locally instead.
+CELERY_ALWAYS_EAGER = False
+
+#===============================================================================
+# EMAIL SETTINGS
+#===============================================================================
+pylons_email_config = dict(config.items('DEFAULT'))
+
+CELERY_SEND_TASK_ERROR_EMAILS = True
+
+#List of (name, email_address) tuples for the admins that should receive error e-mails.
+ADMINS = [('Administrator', pylons_email_config.get('email_to'))]
+
+#The e-mail address this worker sends e-mails from. Default is "celery@localhost".
+SERVER_EMAIL = pylons_email_config.get('error_email_from')
+
+#The mail server to use. Default is "localhost".
+MAIL_HOST = pylons_email_config.get('smtp_server')
+
+#Username (if required) to log on to the mail server with.
+MAIL_HOST_USER = pylons_email_config.get('smtp_username')
+
+#Password (if required) to log on to the mail server with.
+MAIL_HOST_PASSWORD = pylons_email_config.get('smtp_password')
+
+MAIL_PORT = pylons_email_config.get('smtp_port')
+
+
+#===============================================================================
+# INSTRUCTIONS FOR RABBITMQ
+#===============================================================================
+# rabbitmqctl add_user rabbitmq qweqwe
+# rabbitmqctl add_vhost rabbitmqhost
+# rabbitmqctl set_permissions -p rabbitmqhost rabbitmq ".*" ".*" ".*"

pylons_app/controllers/summary.py

@@ -1,71 +1,96
 #!/usr/bin/env python
 # encoding: utf-8
 # summary controller for pylons
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 # 
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on April 18, 2010
 summary controller for pylons
 @author: marcink
 """
 from pylons import tmpl_context as c, request, url
 from pylons_app.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from pylons_app.lib.base import BaseController, render
 from pylons_app.lib.utils import OrderedDict
 from pylons_app.model.hg_model import HgModel
+from pylons_app.model.db import Statistics
 from webhelpers.paginate import Page
 from pylons_app.lib.celerylib import run_task
 from pylons_app.lib.celerylib.tasks import get_commits_stats
+from datetime import datetime, timedelta
+from time import mktime
+import calendar
 import logging
 
 log = logging.getLogger(__name__)
 
 class SummaryController(BaseController):
     
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')           
     def __before__(self):
         super(SummaryController, self).__before__()
                 
     def index(self):
         hg_model = HgModel()
         c.repo_info = hg_model.get_repo(c.repo_name)
         c.repo_changesets = Page(list(c.repo_info[:10]), page=1, items_per_page=20)
         e = request.environ
         uri = u'%(protocol)s://%(user)s@%(host)s/%(repo_name)s' % {
                                         'protocol': e.get('wsgi.url_scheme'),
                                         'user':str(c.hg_app_user.username),
                                         'host':e.get('HTTP_HOST'),
                                         'repo_name':c.repo_name, }
         c.clone_repo_url = uri
         c.repo_tags = OrderedDict()
         for name, hash in c.repo_info.tags.items()[:10]:
             c.repo_tags[name] = c.repo_info.get_changeset(hash)
         
         c.repo_branches = OrderedDict()
         for name, hash in c.repo_info.branches.items()[:10]:
             c.repo_branches[name] = c.repo_info.get_changeset(hash)
         
-        task = run_task(get_commits_stats, c.repo_info.name)
-        c.ts_min = task.result[0]
-        c.ts_max = task.result[1]
-        c.commit_data = task.result[2]
-        c.overview_data = task.result[3]
+        td = datetime.today() + timedelta(days=1) 
+        y, m, d = td.year, td.month, td.day
+        
+        ts_min_y = mktime((y - 1, (td - timedelta(days=calendar.mdays[m])).month,
+                            d, 0, 0, 0, 0, 0, 0,))
+        ts_min_m = mktime((y, (td - timedelta(days=calendar.mdays[m])).month,
+                            d, 0, 0, 0, 0, 0, 0,))
+        
+        ts_max_y = mktime((y, m, d, 0, 0, 0, 0, 0, 0,))
+            
+        run_task(get_commits_stats, c.repo_info.name, ts_min_y, ts_max_y)
+        c.ts_min = ts_min_m
+        c.ts_max = ts_max_y
+        
+        
+        stats = self.sa.query(Statistics)\
+            .filter(Statistics.repository == c.repo_info.dbrepo)\
+            .scalar()
+
+        if stats:
+            c.commit_data = stats.commit_activity
+            c.overview_data = stats.commit_activity_combined
+        else:
+            import json
+            c.commit_data = json.dumps({})
+            c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 0] ])
         
         return render('summary/summary.html')
 

pylons_app/lib/celerylib/tasks.py

@@ -1,229 +1,266
 from celery.decorators import task
 from celery.task.sets import subtask
 from celeryconfig import PYLONS_CONFIG as config
-from datetime import datetime, timedelta
 from pylons.i18n.translation import _
 from pylons_app.lib.celerylib import run_task
 from pylons_app.lib.helpers import person
 from pylons_app.lib.smtp_mailer import SmtpMailer
 from pylons_app.lib.utils import OrderedDict
 from operator import itemgetter
 from vcs.backends.hg import MercurialRepository
 from time import mktime
-import calendar
 import traceback
 import json
 
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
 
 def get_session():
     from sqlalchemy import engine_from_config
     from sqlalchemy.orm import sessionmaker, scoped_session
     engine = engine_from_config(dict(config.items('app:main')), 'sqlalchemy.db1.')
     sa = scoped_session(sessionmaker(bind=engine))
     return sa
 
 def get_hg_settings():
     from pylons_app.model.db import HgAppSettings
     try:
         sa = get_session()
         ret = sa.query(HgAppSettings).all()
     finally:
         sa.remove()
         
     if not ret:
         raise Exception('Could not get application settings !')
     settings = {}
     for each in ret:
         settings['hg_app_' + each.app_settings_name] = each.app_settings_value    
     
     return settings
 
 def get_hg_ui_settings():
     from pylons_app.model.db import HgAppUi
     try:
         sa = get_session()
         ret = sa.query(HgAppUi).all()
     finally:
         sa.remove()
         
     if not ret:
         raise Exception('Could not get application ui settings !')
     settings = {}
     for each in ret:
         k = each.ui_key
         v = each.ui_value
         if k == '/':
             k = 'root_path'
         
         if k.find('.') != -1:
             k = k.replace('.', '_')
         
         if each.ui_section == 'hooks':
             v = each.ui_active
         
         settings[each.ui_section + '_' + k] = v  
     
     return settings   
 
 @task
 def whoosh_index(repo_location, full_index):
     log = whoosh_index.get_logger()
     from pylons_app.lib.indexers import DaemonLock
     from pylons_app.lib.indexers.daemon import WhooshIndexingDaemon, LockHeld
     try:
         l = DaemonLock()
         WhooshIndexingDaemon(repo_location=repo_location)\
             .run(full_index=full_index)
         l.release()
         return 'Done'
     except LockHeld:
         log.info('LockHeld')
         return 'LockHeld'    
 
 @task
-def get_commits_stats(repo):
+def get_commits_stats(repo_name, ts_min_y, ts_max_y):
+    author_key_cleaner = lambda k: person(k).replace('"', "") #for js data compatibilty
+        
+    from pylons_app.model.db import Statistics, Repository
     log = get_commits_stats.get_logger()
-    aggregate = OrderedDict()
-    overview_aggregate = OrderedDict()
+    commits_by_day_author_aggregate = {}
+    commits_by_day_aggregate = {}
     repos_path = get_hg_ui_settings()['paths_root_path'].replace('*', '')
-    repo = MercurialRepository(repos_path + repo)
-    #graph range
-    td = datetime.today() + timedelta(days=1) 
-    y, m, d = td.year, td.month, td.day
+    repo = MercurialRepository(repos_path + repo_name)
+
+    skip_date_limit = True
+    parse_limit = 500 #limit for single task changeset parsing
+    last_rev = 0
+    last_cs = None
+    timegetter = itemgetter('time')
+    
+    sa = get_session()
     
-    ts_min_y = mktime((y - 1, (td - timedelta(days=calendar.mdays[m])).month,
-                        d, 0, 0, 0, 0, 0, 0,))
-    ts_min_m = mktime((y, (td - timedelta(days=calendar.mdays[m])).month,
-                        d, 0, 0, 0, 0, 0, 0,))
+    dbrepo = sa.query(Repository)\
+        .filter(Repository.repo_name == repo_name).scalar()
+    cur_stats = sa.query(Statistics)\
+        .filter(Statistics.repository == dbrepo).scalar()
+    if cur_stats:
+        last_rev = cur_stats.stat_on_revision
     
-    ts_max_y = mktime((y, m, d, 0, 0, 0, 0, 0, 0,))
-    skip_date_limit = True
+    if last_rev == repo.revisions[-1]:
+        #pass silently without any work
+        return True
     
-    def author_key_cleaner(k):
-        k = person(k)
-        k = k.replace('"', "") #for js data compatibilty
-        return k
-            
-    for cs in repo[:200]:#added limit 200 until fix #29 is made
+    if cur_stats:
+        commits_by_day_aggregate = OrderedDict(
+                                       json.loads(
+                                        cur_stats.commit_activity_combined))
+        commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity)
+    
+    for cnt, rev in enumerate(repo.revisions[last_rev:]):
+        last_cs = cs = repo.get_changeset(rev)
         k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1],
                           cs.date.timetuple()[2])
         timetupple = [int(x) for x in k.split('-')]
         timetupple.extend([0 for _ in xrange(6)])
         k = mktime(timetupple)
-        if aggregate.has_key(author_key_cleaner(cs.author)):
-            if aggregate[author_key_cleaner(cs.author)].has_key(k):
-                aggregate[author_key_cleaner(cs.author)][k]["commits"] += 1
-                aggregate[author_key_cleaner(cs.author)][k]["added"] += len(cs.added)
-                aggregate[author_key_cleaner(cs.author)][k]["changed"] += len(cs.changed)
-                aggregate[author_key_cleaner(cs.author)][k]["removed"] += len(cs.removed)
+        if commits_by_day_author_aggregate.has_key(author_key_cleaner(cs.author)):
+            try:
+                l = [timegetter(x) for x in commits_by_day_author_aggregate\
+                        [author_key_cleaner(cs.author)]['data']]
+                time_pos = l.index(k)
+            except ValueError:
+                time_pos = False
+                
+            if time_pos >= 0 and time_pos is not False:
+                
+                datadict = commits_by_day_author_aggregate\
+                    [author_key_cleaner(cs.author)]['data'][time_pos]
+                
+                datadict["commits"] += 1
+                datadict["added"] += len(cs.added)
+                datadict["changed"] += len(cs.changed)
+                datadict["removed"] += len(cs.removed)
+                #print datadict
                 
             else:
-                #aggregate[author_key_cleaner(cs.author)].update(dates_range)
+                #print 'ELSE !!!!'
                 if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
-                    aggregate[author_key_cleaner(cs.author)][k] = {}
-                    aggregate[author_key_cleaner(cs.author)][k]["commits"] = 1
-                    aggregate[author_key_cleaner(cs.author)][k]["added"] = len(cs.added)
-                    aggregate[author_key_cleaner(cs.author)][k]["changed"] = len(cs.changed)
-                    aggregate[author_key_cleaner(cs.author)][k]["removed"] = len(cs.removed) 
+                    
+                    datadict = {"time":k,
+                                "commits":1,
+                                "added":len(cs.added),
+                                "changed":len(cs.changed),
+                                "removed":len(cs.removed),
+                               }
+                    commits_by_day_author_aggregate\
+                        [author_key_cleaner(cs.author)]['data'].append(datadict)
                                         
         else:
+            #print k, 'nokey ADDING'
             if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
-                aggregate[author_key_cleaner(cs.author)] = OrderedDict()
-                #aggregate[author_key_cleaner(cs.author)].update(dates_range)
-                aggregate[author_key_cleaner(cs.author)][k] = {}
-                aggregate[author_key_cleaner(cs.author)][k]["commits"] = 1
-                aggregate[author_key_cleaner(cs.author)][k]["added"] = len(cs.added)
-                aggregate[author_key_cleaner(cs.author)][k]["changed"] = len(cs.changed)
-                aggregate[author_key_cleaner(cs.author)][k]["removed"] = len(cs.removed)                 
+                commits_by_day_author_aggregate[author_key_cleaner(cs.author)] = {
+                                    "label":author_key_cleaner(cs.author),
+                                    "data":[{"time":k,
+                                             "commits":1,
+                                             "added":len(cs.added),
+                                             "changed":len(cs.changed),
+                                             "removed":len(cs.removed),
+                                             }],
+                                    "schema":["commits"],
+                                    }               
     
-        
-        if overview_aggregate.has_key(k):
-            overview_aggregate[k] += 1
+#        #gather all data by day
+        if commits_by_day_aggregate.has_key(k):
+            commits_by_day_aggregate[k] += 1
         else:
-            overview_aggregate[k] = 1
-    
+            commits_by_day_aggregate[k] = 1
+        
+        if cnt >= parse_limit:
+            #don't fetch to much data since we can freeze application
+            break
+
     overview_data = []
-    for k, v in overview_aggregate.items():
+    for k, v in commits_by_day_aggregate.items():
         overview_data.append([k, v])
     overview_data = sorted(overview_data, key=itemgetter(0))
-    data = {}
-    for author in aggregate:
-        commit_data = sorted([{"time":x,
-                               "commits":aggregate[author][x]['commits'],
-                               "added":aggregate[author][x]['added'],
-                               "changed":aggregate[author][x]['changed'],
-                               "removed":aggregate[author][x]['removed'],
-                              } for x in aggregate[author]],
-                              key=itemgetter('time'))
         
-        data[author] = {"label":author,
-                      "data":commit_data,
-                      "schema":["commits"]
-                      }
-        
-    if not data:
-        data[author_key_cleaner(repo.contact)] = {
+    if not commits_by_day_author_aggregate:
+        commits_by_day_author_aggregate[author_key_cleaner(repo.contact)] = {
             "label":author_key_cleaner(repo.contact),
             "data":[0, 1],
             "schema":["commits"],
         }
-                
-    return (ts_min_m, ts_max_y, json.dumps(data), json.dumps(overview_data))    
+
+    stats = cur_stats if cur_stats else Statistics()
+    stats.commit_activity = json.dumps(commits_by_day_author_aggregate)
+    stats.commit_activity_combined = json.dumps(overview_data)
+    stats.repository = dbrepo
+    stats.stat_on_revision = last_cs.revision
+    stats.languages = json.dumps({'_TOTAL_':0, '':0})
+    
+    try:
+        sa.add(stats)
+        sa.commit()    
+    except:
+        log.error(traceback.format_exc())
+        sa.rollback()
+        return False
+                        
+    return True
 
 @task
 def reset_user_password(user_email):
     log = reset_user_password.get_logger()
     from pylons_app.lib import auth
     from pylons_app.model.db import User
     
     try:
         try:
             sa = get_session()
             user = sa.query(User).filter(User.email == user_email).scalar()
             new_passwd = auth.PasswordGenerator().gen_password(8,
                              auth.PasswordGenerator.ALPHABETS_BIG_SMALL)
-            user.password = auth.get_crypt_password(new_passwd)
-            sa.add(user)
-            sa.commit()
-            log.info('change password for %s', user_email)
+            if user:
+                user.password = auth.get_crypt_password(new_passwd)
+                sa.add(user)
+                sa.commit()
+                log.info('change password for %s', user_email)
             if new_passwd is None:
                 raise Exception('unable to generate new password')
             
         except:
             log.error(traceback.format_exc())
             sa.rollback()
         
         run_task(send_email, user_email,
                  "Your new hg-app password",
                  'Your new hg-app password:%s' % (new_passwd))
         log.info('send new password mail to %s', user_email)
         
         
     except:
         log.error('Failed to update user password')
         log.error(traceback.format_exc())
     return True
 
 @task    
 def send_email(recipients, subject, body):
     log = send_email.get_logger()
     email_config = dict(config.items('DEFAULT')) 
     mail_from = email_config.get('app_email_from')
     user = email_config.get('smtp_username')
     passwd = email_config.get('smtp_password')
     mail_server = email_config.get('smtp_server')
     mail_port = email_config.get('smtp_port')
     tls = email_config.get('smtp_use_tls')
     ssl = False
     
     try:
         m = SmtpMailer(mail_from, user, passwd, mail_server,
                        mail_port, ssl, tls)
         m.send(recipients, subject, body)  
     except:
         log.error('Mail sending failed')
         log.error(traceback.format_exc())
         return False
     return True

pylons_app/model/db.py

@@ -1,125 +1,134
 from pylons_app.model.meta import Base
 from sqlalchemy import *
 from sqlalchemy.orm import relation, backref
 from sqlalchemy.orm.session import Session
 from vcs.utils.lazy import LazyProperty
 import logging
 
 log = logging.getLogger(__name__)
 
 class HgAppSettings(Base):
     __tablename__ = 'hg_app_settings'
     __table_args__ = (UniqueConstraint('app_settings_name'), {'useexisting':True})
     app_settings_id = Column("app_settings_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     app_settings_name = Column("app_settings_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     app_settings_value = Column("app_settings_value", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
 class HgAppUi(Base):
     __tablename__ = 'hg_app_ui'
     __table_args__ = {'useexisting':True}
     ui_id = Column("ui_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     ui_section = Column("ui_section", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_key = Column("ui_key", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_value = Column("ui_value", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ui_active = Column("ui_active", BOOLEAN(), nullable=True, unique=None, default=True)
     
     
 class User(Base): 
     __tablename__ = 'users'
     __table_args__ = (UniqueConstraint('username'), UniqueConstraint('email'), {'useexisting':True})
     user_id = Column("user_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     username = Column("username", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     password = Column("password", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     active = Column("active", BOOLEAN(), nullable=True, unique=None, default=None)
     admin = Column("admin", BOOLEAN(), nullable=True, unique=None, default=False)
     name = Column("name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     lastname = Column("lastname", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     email = Column("email", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     last_login = Column("last_login", DATETIME(timezone=False), nullable=True, unique=None, default=None)
     
     user_log = relation('UserLog')
     user_perms = relation('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id")
     
     @LazyProperty
     def full_contact(self):
         return '%s %s <%s>' % (self.name, self.lastname, self.email)
         
     def __repr__(self):
         return "<User('id:%s:%s')>" % (self.user_id, self.username)
     
     def update_lastlogin(self):
         """Update user lastlogin"""
         import datetime
         
         try:
             session = Session.object_session(self)
             self.last_login = datetime.datetime.now()
             session.add(self)
             session.commit()
             log.debug('updated user %s lastlogin', self.username)
         except Exception:
             session.rollback()        
     
       
 class UserLog(Base): 
     __tablename__ = 'user_logs'
     __table_args__ = {'useexisting':True}
     user_log_id = Column("user_log_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     user_ip = Column("user_ip", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None) 
     repository = Column("repository", TEXT(length=None, convert_unicode=False, assert_unicode=None), ForeignKey(u'repositories.repo_name'), nullable=False, unique=None, default=None)
     action = Column("action", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     action_date = Column("action_date", DATETIME(timezone=False), nullable=True, unique=None, default=None)
     
     user = relation('User')
     
 class Repository(Base):
     __tablename__ = 'repositories'
     __table_args__ = (UniqueConstraint('repo_name'), {'useexisting':True},)
     repo_id = Column("repo_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     repo_name = Column("repo_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
     user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=False, default=None)
     private = Column("private", BOOLEAN(), nullable=True, unique=None, default=None)
     description = Column("description", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     
     user = relation('User')
     repo_to_perm = relation('RepoToPerm', cascade='all')
     
     def __repr__(self):
         return "<Repository('id:%s:%s')>" % (self.repo_id, self.repo_name)
         
 class Permission(Base):
     __tablename__ = 'permissions'
     __table_args__ = {'useexisting':True}
     permission_id = Column("permission_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     permission_name = Column("permission_name", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     permission_longname = Column("permission_longname", TEXT(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     
     def __repr__(self):
         return "<Permission('%s:%s')>" % (self.permission_id, self.permission_name)
 
 class RepoToPerm(Base):
     __tablename__ = 'repo_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'repository_id'), {'useexisting':True})
     repo_to_perm_id = Column("repo_to_perm_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", INTEGER(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", INTEGER(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=None, default=None) 
     
     user = relation('User')
     permission = relation('Permission')
     repository = relation('Repository')
 
 class UserToPerm(Base):
     __tablename__ = 'user_to_perm'
     __table_args__ = (UniqueConstraint('user_id', 'permission_id'), {'useexisting':True})
     user_to_perm_id = Column("user_to_perm_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", INTEGER(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", INTEGER(), ForeignKey(u'permissions.permission_id'), nullable=False, unique=None, default=None)
     
     user = relation('User')
     permission = relation('Permission')
 
-
+class Statistics(Base):
+    __tablename__ = 'statistics'
+    __table_args__ = (UniqueConstraint('repository_id'), {'useexisting':True})
+    stat_id = Column("stat_id", INTEGER(), nullable=False, unique=True, default=None, primary_key=True)
+    repository_id = Column("repository_id", INTEGER(), ForeignKey(u'repositories.repo_id'), nullable=False, unique=True, default=None)
+    stat_on_revision = Column("stat_on_revision", INTEGER(), nullable=False)
+    commit_activity = Column("commit_activity", BLOB(), nullable=False)#JSON data
+    commit_activity_combined = Column("commit_activity_combined", BLOB(), nullable=False)#JSON data
+    languages = Column("languages", BLOB(), nullable=False)#JSON data
+    
+    repository = relation('Repository')
 
-

pylons_app/templates/summary/summary.html

@@ -1,508 +1,508
 <%inherit file="/base/base.html"/>
 
 <%def name="title()">
     ${_('Mercurial Repository Overview')}
 </%def>
 
 <%def name="breadcrumbs_links()">
     ${h.link_to(u'Home',h.url('/'))}
     &raquo; 
     ${h.link_to(c.repo_name,h.url('summary_home',repo_name=c.repo_name))}
     &raquo;
     ${_('summary')}
 </%def>
 
 <%def name="page_nav()">
 	${self.menu('summary')}    
 </%def>
 
 <%def name="main()">
 <script type="text/javascript">
 var E = YAHOO.util.Event;
 var D = YAHOO.util.Dom;
 
 E.onDOMReady(function(e){
     id = 'clone_url';
     E.addListener(id,'click',function(e){
         D.get('clone_url').select();
     })
 })
 </script>
 <div class="box box-left">
     <!-- box / title -->
     <div class="title">
         ${self.breadcrumbs()}
     </div>
     <!-- end box / title -->
 	<div class="form">
 	  <div class="fields">
 		 
 			 <div class="field">
 			  <div class="label">
 			      <label>${_('Name')}:</label>
 			  </div>
 			  <div class="input-short">
 			      <span style="font-size: 1.6em;font-weight: bold">${c.repo_info.name}</span>
 			  </div>
 			 </div>
 			
 			
 			 <div class="field">
 			  <div class="label">
 			      <label>${_('Description')}:</label>
 			  </div>
 			  <div class="input-short">
 			      ${c.repo_info.description}
 			  </div>
 			 </div>
 			
 			
 			 <div class="field">
 			  <div class="label">
 			      <label>${_('Contact')}:</label>
 			  </div>
 			  <div class="input-short">
 			  	<div class="gravatar">
 			  		<img alt="gravatar" src="${h.gravatar_url(c.repo_info.dbrepo.user.email)}"/>
 			  	</div>
 			  		${_('Username')}: ${c.repo_info.dbrepo.user.username}<br/>
 			  		${_('Name')}: ${c.repo_info.dbrepo.user.name} ${c.repo_info.dbrepo.user.lastname}<br/>
 			  		${_('Email')}: <a href="mailto:${c.repo_info.dbrepo.user.email}">${c.repo_info.dbrepo.user.email}</a>
 			  </div>
 			 </div>
 			
 			 <div class="field">
 			  <div class="label">
 			      <label>${_('Last change')}:</label>
 			  </div>
 			  <div class="input-short">
 			      ${h.age(c.repo_info.last_change)} - ${h.rfc822date(c.repo_info.last_change)} 
 			      ${_('by')} ${h.get_changeset_safe(c.repo_info,'tip').author} 
 			      
 			  </div>
 			 </div>
 			
 			 <div class="field">
 			  <div class="label">
 			      <label>${_('Clone url')}:</label>
 			  </div>
 			  <div class="input-short">
 			      <input type="text" id="clone_url" readonly="readonly" value="hg clone ${c.clone_repo_url}" size="70"/>
 			  </div>
 			 </div>
 			
 			 <div class="field">
 			  <div class="label">
 			      <label>${_('Download')}:</label>
 			  </div>
 			  <div class="input-short">
 		        %for cnt,archive in enumerate(c.repo_info._get_archives()):
 		             %if cnt >=1:
 		             |
 		             %endif
 		             ${h.link_to(c.repo_info.name+'.'+archive['type'],
 		                h.url('files_archive_home',repo_name=c.repo_info.name,
 		                revision='tip',fileformat=archive['extension']),class_="archive_icon")}
 		        %endfor
 			  </div>
 			 </div>
 			 
 			 <div class="field">
 			  <div class="label">
 			      <label>${_('Feeds')}:</label>
 			  </div>
 			  <div class="input-short">
 	            ${h.link_to(_('RSS'),h.url('rss_feed_home',repo_name=c.repo_info.name),class_='rss_icon')}
 	            ${h.link_to(_('Atom'),h.url('atom_feed_home',repo_name=c.repo_info.name),class_='atom_icon')}
 			  </div>
 			 </div>				 			 			 
 	  </div>		 
 	</div>				
 </div>
         
 <div class="box box-right"  style="min-height:455px">
     <!-- box / title -->
     <div class="title">
-        <h5>${_('Commit activity')}</h5>
+        <h5>${_('Commit activity by day / author')}</h5>
     </div>
     
     <div class="table">
         <div id="commit_history" style="width:560px;height:300px;float:left"></div>
         <div style="clear: both;height: 10px"></div>
         <div id="overview" style="width:560px;height:100px;float:left"></div>
         
     	<div id="legend_data" style="clear:both;margin-top:10px;">
 	    	<div id="legend_container"></div>
 	    	<div id="legend_choices">
 				<table id="legend_choices_tables" style="font-size:smaller;color:#545454"></table>
 	    	</div>
     	</div>
 		<script type="text/javascript">
 		/**
 		 * Plots summary graph
 		 *
 		 * @class SummaryPlot
 		 * @param {from} initial from for detailed graph
 		 * @param {to} initial to for detailed graph
 		 * @param {dataset}
 		 * @param {overview_dataset}
 		 */
 		function SummaryPlot(from,to,dataset,overview_dataset) {
 			var initial_ranges = {
 			    "xaxis":{
 				    "from":from,
 				   	"to":to,
 				},
 			};
 		    var dataset = dataset;
 		    var overview_dataset = [overview_dataset];
 		    var choiceContainer = YAHOO.util.Dom.get("legend_choices");
 		    var choiceContainerTable = YAHOO.util.Dom.get("legend_choices_tables");
 		    var plotContainer = YAHOO.util.Dom.get('commit_history');
 		    var overviewContainer = YAHOO.util.Dom.get('overview');
 		    
 		    var plot_options = {
 				bars: {show:true,align:'center',lineWidth:4},
 				legend: {show:true, container:"legend_container"},
 				points: {show:true,radius:0,fill:false},
 				yaxis: {tickDecimals:0,},
 				xaxis: {
 					mode: "time", 
 					timeformat: "%d/%m",
 				    min:from,
 				    max:to,	
 				}, 
 				grid: {
 					hoverable: true, 
 				    clickable: true,
 				    autoHighlight:true,
 				    color: "#999"
 				},
 				//selection: {mode: "x"}
 		    };
 		    var overview_options = {
 				legend:{show:false},
 			    bars: {show:true,barWidth: 2,},
 			    shadowSize: 0,
 			    xaxis: {mode: "time", timeformat: "%d/%m/%y",},
 			    yaxis: {ticks: 3, min: 0,},
 			    grid: {color: "#999",},
 			    selection: {mode: "x"}
 			};
 
 			/**
 			*get dummy data needed in few places
 			*/
 		    function getDummyData(label){
 		    	return {"label":label,
                	 "data":[{"time":0,
                		 "commits":0,
 	                     "added":0,
 	                     "changed":0,
 	                     "removed":0,
                     }],
                     "schema":["commits"],
                     "color":'#ffffff',
            		}
 			}
 			
 		    /**
 		     * generate checkboxes accordindly to data
 		     * @param keys
 		     * @returns
 		     */
 		    function generateCheckboxes(data) {
 			    //append checkboxes
 			    var i = 0;
 			    choiceContainerTable.innerHTML = '';
 			    for(var pos in data) {
 			    	
 			    	data[pos].color = i;
 			        i++;
 			        if(data[pos].label != ''){
 				        choiceContainerTable.innerHTML += '<tr><td>'+
 				        '<input type="checkbox" name="' + data[pos].label +'" checked="checked" />'
 				        +data[pos].label+
 				        '</td></tr>';
 			        }
 			    }	
 		    }
 		    
 		    /**
 		     * ToolTip show
 		     */
 		    function showTooltip(x, y, contents) {
 		        var div=document.getElementById('tooltip');
 		        if(!div) {
 		            div = document.createElement('div');
 		            div.id="tooltip";
 		            div.style.position="absolute";
 		            div.style.border='1px solid #fdd';
 		            div.style.padding='2px';
 		            div.style.backgroundColor='#fee';
 		            document.body.appendChild(div);
 		        }
 		        YAHOO.util.Dom.setStyle(div, 'opacity', 0);
 		        div.innerHTML = contents;
 		        div.style.top=(y + 5) + "px";
 		        div.style.left=(x + 5) + "px";
 
 		        var anim = new YAHOO.util.Anim(div, {opacity: {to: 0.8}}, 0.2);
 		        anim.animate();
 		    }
 		    
 			/**
 			 * This function will detect if selected period has some changesets for this user
 			if it does this data is then pushed for displaying
 			Additionally it will only display users that are selected by the checkbox
 			*/
 		    function getDataAccordingToRanges(ranges) {
 		    	
 		        var data = [];
 		        var keys = [];
 				for(var key in dataset){
 					var push = false;
 					//method1 slow !!
 		            ///*
 		            for(var ds in dataset[key].data){
 			            commit_data = dataset[key].data[ds];
 			            //console.log(key);
 			            //console.log(new Date(commit_data.time*1000));
 			            //console.log(new Date(ranges.xaxis.from*1000));
 			            //console.log(new Date(ranges.xaxis.to*1000));
 			            if (commit_data.time >= ranges.xaxis.from && commit_data.time <= ranges.xaxis.to){
 			            	push = true;
 			            	break;
 					    }
 				    }
 				    //*/
 				    /*//method2 sorted commit data !!!
 				    var first_commit = dataset[key].data[0].time;
 				    var last_commit = dataset[key].data[dataset[key].data.length-1].time;
 				    
 				    console.log(first_commit);
 				    console.log(last_commit);
 				    
 				    if (first_commit >= ranges.xaxis.from && last_commit <= ranges.xaxis.to){
 						push = true;
 					}
 				    */
 				    if(push){			
 				    	data.push(dataset[key]);
 				    }
 				}
 				if(data.length >= 1){
 					return data;
 				} 
 				else{
 					//just return dummy data for graph to plot itself
 					return [getDummyData('')];	
 				}
 				
 		    }
 		    
 			/**
 			* redraw using new checkbox data
 			*/
 		    function plotchoiced(e,args){
 			    var cur_data = args[0];
 			    var cur_ranges = args[1];
 		    	
 				var new_data = [];
 		    	var inputs = choiceContainer.getElementsByTagName("input");
 
 		    	//show only checked labels
 		        for(var i=0; i<inputs.length; i++) {
 		            var checkbox_key = inputs[i].name;
 		            
 	                if(inputs[i].checked){
 						for(var d in cur_data){
 							if(cur_data[d].label == checkbox_key){
 								new_data.push(cur_data[d]);
 							}
 						}			                
 	    	        }
 	                else{
 		                //push dummy data to not hide the label
 						new_data.push(getDummyData(checkbox_key));
 			        }
 		        }
 						        
 		    	var new_options = YAHOO.lang.merge(plot_options, {
 		            xaxis: { 
 		  	      		min: cur_ranges.xaxis.from, 
 		  	      		max: cur_ranges.xaxis.to,
 		  	      		mode:"time",
 		  	      		timeformat: "%d/%m",
 		        	}
 		    	});
 		    	if (!new_data){
 					new_data = [[0,1]];
 				}
 		    	// do the zooming
 		       plot = YAHOO.widget.Flot(plotContainer, new_data, new_options);
 		       
 		       plot.subscribe("plotselected", plotselected);
 	
 		       //resubscribe plothover
 		       plot.subscribe("plothover", plothover);
 		        
 		       // don't fire event on the overview to prevent eternal loop
 		       overview.setSelection(cur_ranges, true);
 	
 		    }
 		    
 			/**
 		     * plot only selected items from overview
 		     * @param ranges
 		     * @returns
 		     */
 		    function plotselected(ranges,cur_data) {
 			    //updates the data for new plot
 	    		data = getDataAccordingToRanges(ranges);
 	    		generateCheckboxes(data);
 	    		
 		    	var new_options = YAHOO.lang.merge(plot_options, {
 		            xaxis: { 
 		  	      		min: ranges.xaxis.from, 
 		  	      		max: ranges.xaxis.to,
 		  	      		mode:"time",
 		  	      		timeformat: "%d/%m",
 		        	}
 		    	});
 		    	// do the zooming
 		        plot = YAHOO.widget.Flot(plotContainer, data, new_options);
 
 		        plot.subscribe("plotselected", plotselected);
 
 		        //resubscribe plothover
 		        plot.subscribe("plothover", plothover);
 		        
 		        // don't fire event on the overview to prevent eternal loop
 		        overview.setSelection(ranges, true);
 
 		        //resubscribe choiced
 		        YAHOO.util.Event.on(choiceContainer.getElementsByTagName("input"), "click", plotchoiced, [data, ranges]);
 		    }
 		    
 		    var previousPoint = null;
 
 			function plothover(o) {
 		        var pos = o.pos;
 		        var item = o.item;
 		        
 		        //YAHOO.util.Dom.get("x").innerHTML = pos.x.toFixed(2);
 		        //YAHOO.util.Dom.get("y").innerHTML = pos.y.toFixed(2);
 		        if (item) {
 		            if (previousPoint != item.datapoint) {
 		                previousPoint = item.datapoint;
 		                
 		                var tooltip = YAHOO.util.Dom.get("tooltip");
 		                if(tooltip) {
 		                	  tooltip.parentNode.removeChild(tooltip);
 		                }
 		                var x = item.datapoint.x.toFixed(2);
 		                var y = item.datapoint.y.toFixed(2);
 						
 		                if (!item.series.label){
 		                    item.series.label = 'commits';
 		                }
 		                var d = new Date(x*1000);
 		                var fd = d.toDateString()
 		                var nr_commits = parseInt(y);
 		                
 		                var cur_data = dataset[item.series.label].data[item.dataIndex];
 		                var added = cur_data.added;
 		                var changed = cur_data.changed;
 		                var removed = cur_data.removed;
 		                
 		                var nr_commits_suffix = " ${_('commits')} ";
 		                var added_suffix = " ${_('files added')} ";
 		                var changed_suffix = " ${_('files changed')} ";
 		                var removed_suffix = " ${_('files removed')} ";
 
 		                
 		                if(nr_commits == 1){nr_commits_suffix = " ${_('commit')} ";}
 						if(added==1){added_suffix=" ${_('file added')} ";}
 						if(changed==1){changed_suffix=" ${_('file changed')} ";}
 						if(removed==1){removed_suffix=" ${_('file removed')} ";}
 										                
 		                showTooltip(item.pageX, item.pageY, item.series.label + " on " + fd
 								 +'<br/>'+
 		                         nr_commits + nr_commits_suffix+'<br/>'+
 		                         added + added_suffix +'<br/>'+
 		                         changed + changed_suffix + '<br/>'+
 		                         removed + removed_suffix + '<br/>');
 		            }
 		        }
 		        else {
 		        	  var tooltip = YAHOO.util.Dom.get("tooltip");
 		        	  
 			          if(tooltip) {
 			                tooltip.parentNode.removeChild(tooltip);
 			          }
 		            previousPoint = null;
 		        }
 		    }
 			
 		    /**
 		     * MAIN EXECUTION
 		     */
 			
 			var data = getDataAccordingToRanges(initial_ranges);
 			generateCheckboxes(data);
 			
 		    //main plot
 		    var plot = YAHOO.widget.Flot(plotContainer,data,plot_options);
 		    
 			//overview
 			var overview = YAHOO.widget.Flot(overviewContainer, overview_dataset, overview_options);
 			
 			//show initial selection on overview
 			overview.setSelection(initial_ranges);    
 			
 		    plot.subscribe("plotselected", plotselected);
 		    
 		    overview.subscribe("plotselected", function (ranges) {
 		        plot.setSelection(ranges);
 		    });		
 				
 		    plot.subscribe("plothover", plothover);
 
 		    YAHOO.util.Event.on(choiceContainer.getElementsByTagName("input"), "click", plotchoiced, [data, initial_ranges]);
 		}
 			SummaryPlot(${c.ts_min},${c.ts_max},${c.commit_data|n},${c.overview_data|n});		
 		</script>
 
     </div>
 </div>    
 
 <div class="box">    
     <div class="title">
         <div class="breadcrumbs">${h.link_to(_('Last ten changes'),h.url('changelog_home',repo_name=c.repo_name))}</div>
     </div>    
     <div class="table">
         <%include file='../shortlog/shortlog_data.html'/>
         ${h.link_to(_('show more'),h.url('changelog_home',repo_name=c.repo_name))}
     </div>
 </div>
 <div class="box">    
     <div class="title">
         <div class="breadcrumbs">${h.link_to(_('Last ten tags'),h.url('tags_home',repo_name=c.repo_name))}</div>
     </div>    
     <div class="table">
         <%include file='../tags/tags_data.html'/>
         ${h.link_to(_('show more'),h.url('tags_home',repo_name=c.repo_name))}
     </div>
 </div>
 <div class="box">
     <div class="title">
         <div class="breadcrumbs">${h.link_to(_('Last ten branches'),h.url('branches_home',repo_name=c.repo_name))}</div>
     </div>    
     <div class="table">
         <%include file='../branches/branches_data.html'/>
         ${h.link_to(_('show more'),h.url('branches_home',repo_name=c.repo_name))}
     </div>      
 </div> 
 
 </%def>    
  No newline at end of file