fixed Example celery config to ampq,

fixed session problems on models and celery,
implemented one base model as a parent for other

celeryconfig.py

@@ -1,77 +1,79
 # List of modules to import when celery starts.
 import sys
 import os
 import ConfigParser
 root = os.getcwd()
 
 PYLONS_CONFIG_NAME = 'production.ini'
 
 sys.path.append(root)
 config = ConfigParser.ConfigParser({'here':root})
 config.read('%s/%s' % (root, PYLONS_CONFIG_NAME))
 PYLONS_CONFIG = config
 
 CELERY_IMPORTS = ("rhodecode.lib.celerylib.tasks",)
 
 ## Result store settings.
-CELERY_RESULT_BACKEND = "database"
-CELERY_RESULT_DBURI = dict(config.items('app:main'))['sqlalchemy.db1.url']
+CELERY_RESULT_BACKEND = "amqp"
+CELERY_AMQP_TASK_RESULT_EXPIRES = 18000  # 5 hours.
+
+#CELERY_RESULT_DBURI = dict(config.items('app:main'))['sqlalchemy.db1.url']
 CELERY_RESULT_SERIALIZER = 'json'
 
 
 BROKER_CONNECTION_MAX_RETRIES = 30
 
 ## Broker settings.
 BROKER_HOST = "localhost"
 BROKER_PORT = 5672
 BROKER_VHOST = "rabbitmqhost"
 BROKER_USER = "rabbitmq"
 BROKER_PASSWORD = "qweqwe"
 
 ## Worker settings
 ## If you're doing mostly I/O you can have more processes,
 ## but if mostly spending CPU, try to keep it close to the
 ## number of CPUs on your machine. If not set, the number of CPUs/cores
 ## available will be used.
 CELERYD_CONCURRENCY = 2
 # CELERYD_LOG_FILE = "celeryd.log"
 CELERYD_LOG_LEVEL = "DEBUG"
 CELERYD_MAX_TASKS_PER_CHILD = 3
 
 #Tasks will never be sent to the queue, but executed locally instead.
 CELERY_ALWAYS_EAGER = False
 if PYLONS_CONFIG_NAME == 'test.ini':
     #auto eager for tests
     CELERY_ALWAYS_EAGER = True
 
 #===============================================================================
 # EMAIL SETTINGS
 #===============================================================================
 pylons_email_config = dict(config.items('DEFAULT'))
 
 CELERY_SEND_TASK_ERROR_EMAILS = True
 
 #List of (name, email_address) tuples for the admins that should receive error e-mails.
 ADMINS = [('Administrator', pylons_email_config.get('email_to'))]
 
 #The e-mail address this worker sends e-mails from. Default is "celery@localhost".
 SERVER_EMAIL = pylons_email_config.get('error_email_from')
 
 #The mail server to use. Default is "localhost".
 MAIL_HOST = pylons_email_config.get('smtp_server')
 
 #Username (if required) to log on to the mail server with.
 MAIL_HOST_USER = pylons_email_config.get('smtp_username')
 
 #Password (if required) to log on to the mail server with.
 MAIL_HOST_PASSWORD = pylons_email_config.get('smtp_password')
 
 MAIL_PORT = pylons_email_config.get('smtp_port')
 
 
 #===============================================================================
 # INSTRUCTIONS FOR RABBITMQ
 #===============================================================================
 # rabbitmqctl add_user rabbitmq qweqwe
 # rabbitmqctl add_vhost rabbitmqhost
 # rabbitmqctl set_permissions -p rabbitmqhost rabbitmq ".*" ".*" ".*"

rhodecode/lib/celerylib/tasks.py

@@ -1,324 +1,324
 from celery.decorators import task
 
 import os
 import traceback
 from time import mktime
 
 from operator import itemgetter
 from pylons.i18n.translation import _
 from rhodecode.lib.celerylib import run_task, locked_task
 from rhodecode.lib.helpers import person
 from rhodecode.lib.smtp_mailer import SmtpMailer
 from rhodecode.lib.utils import OrderedDict
 from vcs.backends import get_repo
 from rhodecode.model.db import RhodeCodeUi
 
 try:
     import json
 except ImportError:
     #python 2.5 compatibility
     import simplejson as json
 
 try:
     from celeryconfig import PYLONS_CONFIG as config
     celery_on = True
 except ImportError:
     #if celeryconfig is not present let's just load our pylons
     #config instead
     from pylons import config
     celery_on = False
 
 
 __all__ = ['whoosh_index', 'get_commits_stats',
            'reset_user_password', 'send_email']
 
 def get_session():
     if celery_on:
         from sqlalchemy import engine_from_config
         from sqlalchemy.orm import sessionmaker, scoped_session
         engine = engine_from_config(dict(config.items('app:main')),
                                     'sqlalchemy.db1.')
         sa = scoped_session(sessionmaker(bind=engine))
     else:
         #If we don't use celery reuse our current application Session
         from rhodecode.model.meta import Session
         sa = Session()
 
     return sa
 
 def get_repos_path():
     sa = get_session()
     q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
     return q.ui_value
 
 @task
 @locked_task
 def whoosh_index(repo_location, full_index):
     log = whoosh_index.get_logger()
     from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
     index_location = dict(config.items('app:main'))['index_dir']
     WhooshIndexingDaemon(index_location=index_location,
                          repo_location=repo_location).run(full_index=full_index)
 
 @task
 @locked_task
 def get_commits_stats(repo_name, ts_min_y, ts_max_y):
     from rhodecode.model.db import Statistics, Repository
     log = get_commits_stats.get_logger()
 
     #for js data compatibilty
     author_key_cleaner = lambda k: person(k).replace('"', "")
 
     commits_by_day_author_aggregate = {}
     commits_by_day_aggregate = {}
     repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
 
     skip_date_limit = True
     parse_limit = 250 #limit for single task changeset parsing optimal for
     last_rev = 0
     last_cs = None
     timegetter = itemgetter('time')
 
     sa = get_session()
 
     dbrepo = sa.query(Repository)\
         .filter(Repository.repo_name == repo_name).scalar()
     cur_stats = sa.query(Statistics)\
         .filter(Statistics.repository == dbrepo).scalar()
     if cur_stats:
         last_rev = cur_stats.stat_on_revision
     if not repo.revisions:
         return True
 
     if last_rev == repo.revisions[-1] and len(repo.revisions) > 1:
         #pass silently without any work if we're not on first revision or 
         #current state of parsing revision(from db marker) is the last revision
         return True
 
     if cur_stats:
         commits_by_day_aggregate = OrderedDict(
                                        json.loads(
                                         cur_stats.commit_activity_combined))
         commits_by_day_author_aggregate = json.loads(cur_stats.commit_activity)
 
     log.debug('starting parsing %s', parse_limit)
     lmktime = mktime
 
     for cnt, rev in enumerate(repo.revisions[last_rev:]):
         last_cs = cs = repo.get_changeset(rev)
         k = '%s-%s-%s' % (cs.date.timetuple()[0], cs.date.timetuple()[1],
                           cs.date.timetuple()[2])
         timetupple = [int(x) for x in k.split('-')]
         timetupple.extend([0 for _ in xrange(6)])
         k = lmktime(timetupple)
         if commits_by_day_author_aggregate.has_key(author_key_cleaner(cs.author)):
             try:
                 l = [timegetter(x) for x in commits_by_day_author_aggregate\
                         [author_key_cleaner(cs.author)]['data']]
                 time_pos = l.index(k)
             except ValueError:
                 time_pos = False
 
             if time_pos >= 0 and time_pos is not False:
 
                 datadict = commits_by_day_author_aggregate\
                     [author_key_cleaner(cs.author)]['data'][time_pos]
 
                 datadict["commits"] += 1
                 datadict["added"] += len(cs.added)
                 datadict["changed"] += len(cs.changed)
                 datadict["removed"] += len(cs.removed)
 
             else:
                 if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
 
                     datadict = {"time":k,
                                 "commits":1,
                                 "added":len(cs.added),
                                 "changed":len(cs.changed),
                                 "removed":len(cs.removed),
                                }
                     commits_by_day_author_aggregate\
                         [author_key_cleaner(cs.author)]['data'].append(datadict)
 
         else:
             if k >= ts_min_y and k <= ts_max_y or skip_date_limit:
                 commits_by_day_author_aggregate[author_key_cleaner(cs.author)] = {
                                     "label":author_key_cleaner(cs.author),
                                     "data":[{"time":k,
                                              "commits":1,
                                              "added":len(cs.added),
                                              "changed":len(cs.changed),
                                              "removed":len(cs.removed),
                                              }],
                                     "schema":["commits"],
                                     }
 
         #gather all data by day
         if commits_by_day_aggregate.has_key(k):
             commits_by_day_aggregate[k] += 1
         else:
             commits_by_day_aggregate[k] = 1
 
         if cnt >= parse_limit:
             #don't fetch to much data since we can freeze application
             break
     overview_data = []
     for k, v in commits_by_day_aggregate.items():
         overview_data.append([k, v])
     overview_data = sorted(overview_data, key=itemgetter(0))
     if not commits_by_day_author_aggregate:
         commits_by_day_author_aggregate[author_key_cleaner(repo.contact)] = {
             "label":author_key_cleaner(repo.contact),
             "data":[0, 1],
             "schema":["commits"],
         }
 
     stats = cur_stats if cur_stats else Statistics()
     stats.commit_activity = json.dumps(commits_by_day_author_aggregate)
     stats.commit_activity_combined = json.dumps(overview_data)
 
     log.debug('last revison %s', last_rev)
     leftovers = len(repo.revisions[last_rev:])
     log.debug('revisions to parse %s', leftovers)
 
     if last_rev == 0 or leftovers < parse_limit:
         stats.languages = json.dumps(__get_codes_stats(repo_name))
 
     stats.repository = dbrepo
     stats.stat_on_revision = last_cs.revision
 
     try:
         sa.add(stats)
         sa.commit()
     except:
         log.error(traceback.format_exc())
         sa.rollback()
         return False
     if len(repo.revisions) > 1:
         run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y)
 
     return True
 
 @task
 def reset_user_password(user_email):
     log = reset_user_password.get_logger()
     from rhodecode.lib import auth
     from rhodecode.model.db import User
 
     try:
         try:
             sa = get_session()
             user = sa.query(User).filter(User.email == user_email).scalar()
             new_passwd = auth.PasswordGenerator().gen_password(8,
                              auth.PasswordGenerator.ALPHABETS_BIG_SMALL)
             if user:
                 user.password = auth.get_crypt_password(new_passwd)
                 sa.add(user)
                 sa.commit()
                 log.info('change password for %s', user_email)
             if new_passwd is None:
                 raise Exception('unable to generate new password')
 
         except:
             log.error(traceback.format_exc())
             sa.rollback()
 
         run_task(send_email, user_email,
                  "Your new rhodecode password",
                  'Your new rhodecode password:%s' % (new_passwd))
         log.info('send new password mail to %s', user_email)
 
 
     except:
         log.error('Failed to update user password')
         log.error(traceback.format_exc())
     return True
 
 @task
 def send_email(recipients, subject, body):
     """
     Sends an email with defined parameters from the .ini files.
     
     
     :param recipients: list of recipients, it this is empty the defined email
         address from field 'email_to' is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     """
     log = send_email.get_logger()
     email_config = dict(config.items('DEFAULT'))
 
     if not recipients:
         recipients = [email_config.get('email_to')]
 
     def str2bool(v):
         return v.lower() in ["yes", "true", "t", "1"] if v else None
 
     mail_from = email_config.get('app_email_from')
     user = email_config.get('smtp_username')
     passwd = email_config.get('smtp_password')
     mail_server = email_config.get('smtp_server')
     mail_port = email_config.get('smtp_port')
     tls = str2bool(email_config.get('smtp_use_tls'))
     ssl = str2bool(email_config.get('smtp_use_ssl'))
 
     try:
         m = SmtpMailer(mail_from, user, passwd, mail_server,
                        mail_port, ssl, tls)
         m.send(recipients, subject, body)
     except:
         log.error('Mail sending failed')
         log.error(traceback.format_exc())
         return False
     return True
 
 @task
 def create_repo_fork(form_data, cur_user):
     from rhodecode.model.repo import RepoModel
     from vcs import get_backend
     log = create_repo_fork.get_logger()
-    repo_model = RepoModel()
+    repo_model = RepoModel(get_session())
     repo_model.create(form_data, cur_user, just_db=True, fork=True)
     repo_name = form_data['repo_name']
     repos_path = get_repos_path()
     repo_path = os.path.join(repos_path, repo_name)
     repo_fork_path = os.path.join(repos_path, form_data['fork_name'])
     alias = form_data['repo_type']
 
     log.info('creating repo fork %s as %s', repo_name, repo_path)
     backend = get_backend(alias)
     backend(str(repo_fork_path), create=True, src_url=str(repo_path))
 
 def __get_codes_stats(repo_name):
     LANGUAGES_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx',
     'aspx', 'asx', 'axd', 'c', 'cfg', 'cfm', 'cpp', 'cs', 'diff', 'do', 'el',
     'erl', 'h', 'java', 'js', 'jsp', 'jspx', 'lisp', 'lua', 'm', 'mako', 'ml',
     'pas', 'patch', 'php', 'php3', 'php4', 'phtml', 'pm', 'py', 'rb', 'rst',
     's', 'sh', 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', 'yaws']
 
 
     repos_path = get_repos_path()
     p = os.path.join(repos_path, repo_name)
     repo = get_repo(p)
     tip = repo.get_changeset()
     code_stats = {}
 
     def aggregate(cs):
         for f in cs[2]:
             k = f.mimetype
             if f.extension in LANGUAGES_EXTENSIONS:
                 if code_stats.has_key(k):
                     code_stats[k] += 1
                 else:
                     code_stats[k] = 1
 
     map(aggregate, tip.walk('/'))
 
     return code_stats or {}
 
 
 
 

rhodecode/model/__init__.py

@@ -1,23 +1,32
 """The application's model objects"""
 import logging
 from rhodecode.model import meta
 log = logging.getLogger(__name__)
 
 def init_model(engine):
     """Call me before using any of the tables or classes in the model"""
     log.info("INITIALIZING DB MODELS")
     meta.Base.metadata.bind = engine
     #meta.Base2.metadata.bind = engine2
 
 #THIS IS A TEST FOR EXECUTING SCRIPT AND LOAD PYLONS APPLICATION GLOBALS
 #from paste.deploy import appconfig
 #from pylons import config
 #from sqlalchemy import engine_from_config
 #from rhodecode.config.environment import load_environment
 #
 #conf = appconfig('config:development.ini', relative_to = './../../')
 #load_environment(conf.global_conf, conf.local_conf)
 #
 #engine = engine_from_config(config, 'sqlalchemy.')
 #init_model(engine)
 # DO SOMETHING
+
+
+class BaseModel(object):
+
+    def __init__(self, sa=None):
+        if sa is not None:
+            self.sa = sa
+        else:
+            self.sa = meta.Session()

rhodecode/model/permission.py

@@ -1,99 +1,96
 #!/usr/bin/env python
 # encoding: utf-8
 # Model for permissions
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on Aug 20, 2010
 Model for permissions
-@author: marcink
+:author: marcink
 """
 
+from rhodecode.model import BaseModel
 from rhodecode.model.db import User, Permission, UserToPerm, RepoToPerm
 from rhodecode.model.caching_query import FromCache
-from rhodecode.model.meta import Session
 import logging
 import traceback
 log = logging.getLogger(__name__)
 
 
-class PermissionModel(object):
-
-    def __init__(self):
-        self.sa = Session()
+class PermissionModel(BaseModel):
 
     def get_permission(self, permission_id, cache=False):
         perm = self.sa.query(Permission)
         if cache:
             perm = perm.options(FromCache("sql_cache_short",
                                           "get_permission_%s" % permission_id))
         return perm.get(permission_id)
 
     def get_permission_by_name(self, name, cache=False):
         perm = self.sa.query(Permission)\
             .filter(Permission.permission_name == name)
         if cache:
             perm = perm.options(FromCache("sql_cache_short",
                                           "get_permission_%s" % name))
         return perm.scalar()
 
     def update(self, form_result):
         perm_user = self.sa.query(User)\
                 .filter(User.username == form_result['perm_user_name']).scalar()
         u2p = self.sa.query(UserToPerm).filter(UserToPerm.user == perm_user).all()
         if len(u2p) != 3:
             raise Exception('Defined: %s should be 3  permissions for default'
                             ' user. This should not happen please verify'
                             ' your database' % len(u2p))
 
         try:
             #stage 1 change defaults    
             for p in u2p:
                 if p.permission.permission_name.startswith('repository.'):
                     p.permission = self.get_permission_by_name(
                                        form_result['default_perm'])
                     self.sa.add(p)
 
                 if p.permission.permission_name.startswith('hg.register.'):
                     p.permission = self.get_permission_by_name(
                                        form_result['default_register'])
                     self.sa.add(p)
 
                 if p.permission.permission_name.startswith('hg.create.'):
                     p.permission = self.get_permission_by_name(
                                         form_result['default_create'])
                     self.sa.add(p)
 
             #stage 2 update all default permissions for repos if checked
             if form_result['overwrite_default'] == True:
                 for r2p in self.sa.query(RepoToPerm)\
                                .filter(RepoToPerm.user == perm_user).all():
                     r2p.permission = self.get_permission_by_name(
                                          form_result['default_perm'])
                     self.sa.add(r2p)
 
             #stage 3 set anonymous access
             if perm_user.username == 'default':
                 perm_user.active = bool(form_result['anonymous'])
                 self.sa.add(perm_user)
 
 
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise

rhodecode/model/repo.py

@@ -1,235 +1,233
 #!/usr/bin/env python
 # encoding: utf-8
 # model for handling repositories actions
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on Jun 5, 2010
 model for handling repositories actions
 :author: marcink
 """
-from vcs.backends import get_repo, get_backend
+from vcs.backends import get_backend
 from datetime import datetime
 from pylons import app_globals as g
 from rhodecode.model.db import Repository, RepoToPerm, User, Permission, \
     Statistics
-from rhodecode.model.meta import Session
+from rhodecode.model import BaseModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.caching_query import FromCache
 import logging
 import os
 import shutil
 import traceback
 log = logging.getLogger(__name__)
 
-class RepoModel(object):
-
-    def __init__(self):
-        self.sa = Session()
+class RepoModel(BaseModel):
 
     def get(self, repo_id, cache=False):
         repo = self.sa.query(Repository)\
             .filter(Repository.repo_id == repo_id)
 
         if cache:
             repo = repo.options(FromCache("sql_cache_short",
                                           "get_repo_%s" % repo_id))
         return repo.scalar()
 
 
     def get_by_repo_name(self, repo_name, cache=False):
         repo = self.sa.query(Repository)\
             .filter(Repository.repo_name == repo_name)
 
         if cache:
             repo = repo.options(FromCache("sql_cache_short",
                                           "get_repo_%s" % repo_name))
         return repo.scalar()        
 
     def get_users_js(self):
 
         users = self.sa.query(User).filter(User.active == True).all()
         u_tmpl = '''{id:%s, fname:"%s", lname:"%s", nname:"%s"},'''
         users_array = '[%s];' % '\n'.join([u_tmpl % (u.user_id, u.name,
                                                     u.lastname, u.username)
                                         for u in users])
         return users_array
 
 
     def update(self, repo_name, form_data):
         try:
 
             #update permissions
             for username, perm in form_data['perms_updates']:
                 r2p = self.sa.query(RepoToPerm)\
-                        .filter(RepoToPerm.user == UserModel()\
+                        .filter(RepoToPerm.user == UserModel(self.sa)\
                                 .get_by_username(username, cache=False))\
                         .filter(RepoToPerm.repository == \
                                 self.get_by_repo_name(repo_name))\
                         .one()
 
                 r2p.permission_id = self.sa.query(Permission).filter(
                                                 Permission.permission_name == 
                                                 perm).one().permission_id
                 self.sa.add(r2p)
 
             #set new permissions
             for username, perm in form_data['perms_new']:
                 r2p = RepoToPerm()
                 r2p.repository = self.get_by_repo_name(repo_name)
-                r2p.user = UserModel().get_by_username(username, cache=False)
+                r2p.user = UserModel(self.sa).get_by_username(username, cache=False)
 
                 r2p.permission_id = self.sa.query(Permission).filter(
                                         Permission.permission_name == perm)\
                                         .one().permission_id
                 self.sa.add(r2p)
 
             #update current repo
             cur_repo = self.get_by_repo_name(repo_name, cache=False)
 
             for k, v in form_data.items():
                 if k == 'user':
                     cur_repo.user_id = v
                 else:
                     setattr(cur_repo, k, v)
 
             self.sa.add(cur_repo)
 
             if repo_name != form_data['repo_name']:
                 #rename our data
                 self.__rename_repo(repo_name, form_data['repo_name'])
 
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def create(self, form_data, cur_user, just_db=False, fork=False):
         try:
             if fork:
                 #force str since hg doesn't go with unicode
                 repo_name = str(form_data['fork_name'])
                 org_name = str(form_data['repo_name'])
 
             else:
                 org_name = repo_name = str(form_data['repo_name'])
             new_repo = Repository()
             for k, v in form_data.items():
                 if k == 'repo_name':
                     v = repo_name
                 setattr(new_repo, k, v)
 
             if fork:
                 parent_repo = self.sa.query(Repository)\
                         .filter(Repository.repo_name == org_name).scalar()
                 new_repo.fork = parent_repo
 
             new_repo.user_id = cur_user.user_id
             self.sa.add(new_repo)
 
             #create default permission
             repo_to_perm = RepoToPerm()
             default = 'repository.read'
-            for p in UserModel().get_by_username('default', cache=False).user_perms:
+            for p in UserModel(self.sa).get_by_username('default', cache=False).user_perms:
                 if p.permission.permission_name.startswith('repository.'):
                     default = p.permission.permission_name
                     break
 
             default_perm = 'repository.none' if form_data['private'] else default
 
             repo_to_perm.permission_id = self.sa.query(Permission)\
                     .filter(Permission.permission_name == default_perm)\
                     .one().permission_id
 
             repo_to_perm.repository_id = new_repo.repo_id
-            repo_to_perm.user_id = UserModel().get_by_username('default', cache=False).user_id
+            repo_to_perm.user_id = UserModel(self.sa)\
+                .get_by_username('default', cache=False).user_id
 
             self.sa.add(repo_to_perm)
             self.sa.commit()
             if not just_db:
                 self.__create_repo(repo_name, form_data['repo_type'])
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def create_fork(self, form_data, cur_user):
         from rhodecode.lib.celerylib import tasks, run_task
         run_task(tasks.create_repo_fork, form_data, cur_user)
 
     def delete(self, repo):
         try:
             self.sa.delete(repo)
             self.__delete_repo(repo)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def delete_perm_user(self, form_data, repo_name):
         try:
             self.sa.query(RepoToPerm)\
                 .filter(RepoToPerm.repository \
                         == self.get_by_repo_name(repo_name))\
                 .filter(RepoToPerm.user_id == form_data['user_id']).delete()
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def delete_stats(self, repo_name):
         try:
             self.sa.query(Statistics)\
                 .filter(Statistics.repository == \
                         self.get_by_repo_name(repo_name)).delete()
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
 
     def __create_repo(self, repo_name, alias):
         from rhodecode.lib.utils import check_repo
         repo_path = os.path.join(g.base_path, repo_name)
         if check_repo(repo_name, g.base_path):
             log.info('creating repo %s in %s', repo_name, repo_path)
             backend = get_backend(alias)
             backend(repo_path, create=True)
 
     def __rename_repo(self, old, new):
         log.info('renaming repo from %s to %s', old, new)
 
         old_path = os.path.join(g.base_path, old)
         new_path = os.path.join(g.base_path, new)
         if os.path.isdir(new_path):
             raise Exception('Was trying to rename to already existing dir %s',
                             new_path)
         shutil.move(old_path, new_path)
 
     def __delete_repo(self, repo):
         rm_path = os.path.join(g.base_path, repo.repo_name)
         log.info("Removing %s", rm_path)
         #disable hg/git
         alias = repo.repo_type
         shutil.move(os.path.join(rm_path, '.%s' % alias),
                     os.path.join(rm_path, 'rm__.%s' % alias))
         #disable repo
         shutil.move(rm_path, os.path.join(g.base_path, 'rm__%s__%s' \
                                           % (datetime.today(), repo.repo_name)))

rhodecode/model/scm.py

@@ -1,350 +1,347
 #!/usr/bin/env python
 # encoding: utf-8
 # Model for RhodeCode
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 # 
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on April 9, 2010
 Model for RhodeCode
 @author: marcink
 """
 from beaker.cache import cache_region, region_invalidate
 from mercurial import ui
 from rhodecode import BACKENDS
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import HasRepoPermissionAny
 from rhodecode.lib.utils import get_repos, make_ui, action_logger
-from rhodecode.model import meta
+from rhodecode.model import BaseModel
 from rhodecode.model.db import Repository, User, RhodeCodeUi, CacheInvalidation, \
     UserFollowing
 from rhodecode.model.caching_query import FromCache
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.session import make_transient
 from vcs import get_backend
 from vcs.utils.helpers import get_scm
 from vcs.exceptions import RepositoryError, VCSError
 from vcs.utils.lazy import LazyProperty
 import traceback
 import logging
 import os
 import time
 
 log = logging.getLogger(__name__)
 
 class UserTemp(object):
     def __init__(self, user_id):
         self.user_id = user_id
 
 class RepoTemp(object):
     def __init__(self, repo_id):
         self.repo_id = repo_id
 
 
-class ScmModel(object):
+class ScmModel(BaseModel):
     """
     Mercurial Model
     """
 
-    def __init__(self):
-        self.sa = meta.Session()
-
     @LazyProperty
     def repos_path(self):
         """
         Get's the repositories root path from database
         """
         q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
 
         return q.ui_value
 
     def repo_scan(self, repos_path, baseui, initial=False):
         """
         Listing of repositories in given path. This path should not be a 
         repository itself. Return a dictionary of repository objects
         
         :param repos_path: path to directory containing repositories
         :param baseui
         :param initial: initial scan
         """
         log.info('scanning for repositories in %s', repos_path)
 
         if not isinstance(baseui, ui.ui):
             baseui = make_ui('db')
         repos_list = {}
 
         for name, path in get_repos(repos_path):
             try:
                 if repos_list.has_key(name):
                     raise RepositoryError('Duplicate repository name %s '
                                     'found in %s' % (name, path))
                 else:
 
                     klass = get_backend(path[0])
 
                     if path[0] == 'hg' and path[0] in BACKENDS.keys():
                         repos_list[name] = klass(path[1], baseui=baseui)
 
                     if path[0] == 'git' and path[0] in BACKENDS.keys():
                         repos_list[name] = klass(path[1])
             except OSError:
                 continue
 
         return repos_list
 
     def get_repos(self, all_repos=None):
         """
         Get all repos from db and for each repo create it's backend instance.
         and fill that backed with information from database
         
         :param all_repos: give specific repositories list, good for filtering
         """
         if not all_repos:
             all_repos = self.sa.query(Repository)\
                 .order_by(Repository.repo_name).all()
 
         invalidation_list = [str(x.cache_key) for x in \
                              self.sa.query(CacheInvalidation.cache_key)\
                              .filter(CacheInvalidation.cache_active == False)\
                              .all()]
 
         for r in all_repos:
 
             repo = self.get(r.repo_name, invalidation_list)
 
             if repo is not None:
                 last_change = repo.last_change
                 tip = h.get_changeset_safe(repo, 'tip')
 
                 tmp_d = {}
                 tmp_d['name'] = repo.name
                 tmp_d['name_sort'] = tmp_d['name'].lower()
                 tmp_d['description'] = repo.dbrepo.description
                 tmp_d['description_sort'] = tmp_d['description']
                 tmp_d['last_change'] = last_change
                 tmp_d['last_change_sort'] = time.mktime(last_change.timetuple())
                 tmp_d['tip'] = tip.raw_id
                 tmp_d['tip_sort'] = tip.revision
                 tmp_d['rev'] = tip.revision
                 tmp_d['contact'] = repo.dbrepo.user.full_contact
                 tmp_d['contact_sort'] = tmp_d['contact']
                 tmp_d['repo_archives'] = list(repo._get_archives())
                 tmp_d['last_msg'] = tip.message
                 tmp_d['repo'] = repo
                 yield tmp_d
 
     def get_repo(self, repo_name):
         return self.get(repo_name)
 
     def get(self, repo_name, invalidation_list=None):
         """
         Get's repository from given name, creates BackendInstance and
         propagates it's data from database with all additional information
         :param repo_name:
         """
         if not HasRepoPermissionAny('repository.read', 'repository.write',
                             'repository.admin')(repo_name, 'get repo check'):
             return
 
         @cache_region('long_term')
         def _get_repo(repo_name):
 
             repo_path = os.path.join(self.repos_path, repo_name)
             alias = get_scm(repo_path)[0]
 
             log.debug('Creating instance of %s repository', alias)
             backend = get_backend(alias)
 
             #TODO: get the baseui from somewhere for this
             if alias == 'hg':
                 from pylons import app_globals as g
                 repo = backend(repo_path, create=False, baseui=g.baseui)
                 #skip hidden web repository
                 if repo._get_hidden():
                     return
             else:
                 repo = backend(repo_path, create=False)
 
             dbrepo = self.sa.query(Repository)\
                 .options(joinedload(Repository.fork))\
                 .options(joinedload(Repository.user))\
                 .filter(Repository.repo_name == repo_name)\
                 .scalar()
             make_transient(dbrepo)
             repo.dbrepo = dbrepo
             return repo
 
         pre_invalidate = True
         if invalidation_list:
             pre_invalidate = repo_name in invalidation_list
 
         if pre_invalidate:
             invalidate = self._should_invalidate(repo_name)
 
             if invalidate:
                 log.info('invalidating cache for repository %s', repo_name)
                 region_invalidate(_get_repo, None, repo_name)
                 self._mark_invalidated(invalidate)
 
         return _get_repo(repo_name)
 
 
 
     def mark_for_invalidation(self, repo_name):
         """
         Puts cache invalidation task into db for 
         further global cache invalidation
         
         :param repo_name: this repo that should invalidation take place
         """
         log.debug('marking %s for invalidation', repo_name)
         cache = self.sa.query(CacheInvalidation)\
             .filter(CacheInvalidation.cache_key == repo_name).scalar()
 
         if cache:
             #mark this cache as inactive
             cache.cache_active = False
         else:
             log.debug('cache key not found in invalidation db -> creating one')
             cache = CacheInvalidation(repo_name)
 
         try:
             self.sa.add(cache)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
 
 
     def toggle_following_repo(self, follow_repo_id, user_id):
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repo_id == follow_repo_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
 
             try:
                 self.sa.delete(f)
                 self.sa.commit()
                 action_logger(UserTemp(user_id),
                               'stopped_following_repo',
                               RepoTemp(follow_repo_id))
                 return
             except:
                 log.error(traceback.format_exc())
                 self.sa.rollback()
                 raise
 
 
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_repo_id = follow_repo_id
             self.sa.add(f)
             self.sa.commit()
             action_logger(UserTemp(user_id),
                           'started_following_repo',
                           RepoTemp(follow_repo_id))
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def toggle_following_user(self, follow_user_id , user_id):
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user_id == follow_user_id)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         if f is not None:
             try:
                 self.sa.delete(f)
                 self.sa.commit()
                 return
             except:
                 log.error(traceback.format_exc())
                 self.sa.rollback()
                 raise
 
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_user_id = follow_user_id
             self.sa.add(f)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def is_following_repo(self, repo_name, user_id):
         r = self.sa.query(Repository)\
             .filter(Repository.repo_name == repo_name).scalar()
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_repository == r)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def is_following_user(self, username, user_id):
         u = self.sa.query(User)\
             .filter(User.username == username).scalar()
 
         f = self.sa.query(UserFollowing)\
             .filter(UserFollowing.follows_user == u)\
             .filter(UserFollowing.user_id == user_id).scalar()
 
         return f is not None
 
     def get_followers(self, repo_id):
         return self.sa.query(UserFollowing)\
                 .filter(UserFollowing.follows_repo_id == repo_id).count()
 
     def get_forks(self, repo_id):
         return self.sa.query(Repository)\
                 .filter(Repository.fork_id == repo_id).count()
 
     def _should_invalidate(self, repo_name):
         """
         Looks up database for invalidation signals for this repo_name
         :param repo_name:
         """
 
         ret = self.sa.query(CacheInvalidation)\
             .options(FromCache('sql_cache_short',
                            'get_invalidation_%s' % repo_name))\
             .filter(CacheInvalidation.cache_key == repo_name)\
             .filter(CacheInvalidation.cache_active == False)\
             .scalar()
 
         return ret
 
     def _mark_invalidated(self, cache_key):
         """
         Marks all occurences of cache to invaldation as already invalidated
         @param repo_name:
         """
         if cache_key:
             log.debug('marking %s as already invalidated', cache_key)
         try:
             cache_key.cache_active = True
             self.sa.add(cache_key)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
 

rhodecode/model/settings.py

@@ -1,79 +1,76
 #!/usr/bin/env python
 # encoding: utf-8
 # Model for RhodeCode settings
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 # 
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on Nov 17, 2010
 Model for RhodeCode
-@author: marcink
+:author: marcink
 """
+
 from rhodecode.lib import helpers as h
-from rhodecode.model import meta
+from rhodecode.model import BaseModel
 from rhodecode.model.caching_query import FromCache
 from rhodecode.model.db import  RhodeCodeSettings
 from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.session import make_transient
 import logging
 
 log = logging.getLogger(__name__)
 
-class SettingsModel(object):
+class SettingsModel(BaseModel):
     """
     Settings model
     """
 
-    def __init__(self):
-        self.sa = meta.Session()
-
-
     def get(self, settings_key, cache=False):
         r = self.sa.query(RhodeCodeSettings)\
             .filter(RhodeCodeSettings.app_settings_name == settings_key).scalar()
         if cache:
             r = r.options(FromCache("sql_cache_short",
                                           "get_setting_%s" % settings_key))
         return r
 
 
     def get_ldap_settings(self):
         """
         Returns ldap settings from database
         :returns:
         ldap_active
         ldap_host
         ldap_port 
         ldap_ldaps
         ldap_dn_user 
         ldap_dn_pass 
         ldap_base_dn
         """
 
         r = self.sa.query(RhodeCodeSettings)\
                 .filter(RhodeCodeSettings.app_settings_name\
                         .startswith('ldap_'))\
                 .all()
 
         fd = {}
 
         for row in r:
             v = row.app_settings_value
             if v in ['0', '1']:
                 v = v == '1'
             fd.update({row.app_settings_name:v})
 
         return fd

rhodecode/model/user.py

@@ -1,220 +1,217
 #!/usr/bin/env python
 # encoding: utf-8
 # Model for users
 # Copyright (C) 2009-2010 Marcin Kuzminski <[e-mail]>
 # 
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 """
 Created on April 9, 2010
 Model for users
 :author: marcink
 """
 
 from pylons.i18n.translation import _
+from rhodecode.model import BaseModel
 from rhodecode.model.caching_query import FromCache
 from rhodecode.model.db import User
-from rhodecode.model.meta import Session
 from rhodecode.lib.exceptions import *
 
 import logging
 import traceback
 
 log = logging.getLogger(__name__)
 
 
 
-class UserModel(object):
-
-    def __init__(self):
-        self.sa = Session()
+class UserModel(BaseModel):
 
     def get(self, user_id, cache=False):
         user = self.sa.query(User)
         if cache:
             user = user.options(FromCache("sql_cache_short",
                                           "get_user_%s" % user_id))
         return user.get(user_id)
 
 
     def get_by_username(self, username, cache=False, case_insensitive=False):
 
         if case_insensitive:
             user = self.sa.query(User).filter(User.username.ilike(username))
         else:
             user = self.sa.query(User)\
                 .filter(User.username == username)
         if cache:
             user = user.options(FromCache("sql_cache_short",
                                           "get_user_%s" % username))
         return user.scalar()
 
     def create(self, form_data):
         try:
             new_user = User()
             for k, v in form_data.items():
                 setattr(new_user, k, v)
 
             self.sa.add(new_user)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def create_ldap(self, username, password):
         """
         Checks if user is in database, if not creates this user marked
         as ldap user
         :param username:
         :param password:
         """
         from rhodecode.lib.auth import get_crypt_password
         if self.get_by_username(username) is None:
             try:
                 new_user = User()
                 new_user.username = username
                 new_user.password = get_crypt_password(password)
                 new_user.email = '[e-mail]' % username
                 new_user.active = True
                 new_user.is_ldap = True
                 new_user.name = '%s@ldap' % username
                 new_user.lastname = ''
 
 
                 self.sa.add(new_user)
                 self.sa.commit()
                 return True
             except:
                 log.error(traceback.format_exc())
                 self.sa.rollback()
                 raise
 
         return False
 
     def create_registration(self, form_data):
         from rhodecode.lib.celerylib import tasks, run_task
         try:
             new_user = User()
             for k, v in form_data.items():
                 if k != 'admin':
                     setattr(new_user, k, v)
 
             self.sa.add(new_user)
             self.sa.commit()
             body = ('New user registration\n'
                     'username: %s\n'
                     'email: %s\n')
             body = body % (form_data['username'], form_data['email'])
 
             run_task(tasks.send_email, None,
                      _('[RhodeCode] New User registration'),
                      body)
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def update(self, user_id, form_data):
         try:
             new_user = self.get(user_id, cache=False)
             if new_user.username == 'default':
                 raise DefaultUserException(
                                 _("You can't Edit this user since it's"
                                   " crucial for entire application"))
 
             for k, v in form_data.items():
                 if k == 'new_password' and v != '':
                     new_user.password = v
                 else:
                     setattr(new_user, k, v)
 
             self.sa.add(new_user)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def update_my_account(self, user_id, form_data):
         try:
             new_user = self.get(user_id, cache=False)
             if new_user.username == 'default':
                 raise DefaultUserException(
                                 _("You can't Edit this user since it's"
                                   " crucial for entire application"))
             for k, v in form_data.items():
                 if k == 'new_password' and v != '':
                     new_user.password = v
                 else:
                     if k not in ['admin', 'active']:
                         setattr(new_user, k, v)
 
             self.sa.add(new_user)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def delete(self, user_id):
         try:
             user = self.get(user_id, cache=False)
             if user.username == 'default':
                 raise DefaultUserException(
                                 _("You can't remove this user since it's"
                                   " crucial for entire application"))
             if user.repositories:
                 raise UserOwnsReposException(_('This user still owns %s '
                                                'repositories and cannot be '
                                                'removed. Switch owners or '
                                                'remove those repositories') \
                                                % user.repositories)
             self.sa.delete(user)
             self.sa.commit()
         except:
             log.error(traceback.format_exc())
             self.sa.rollback()
             raise
 
     def reset_password(self, data):
         from rhodecode.lib.celerylib import tasks, run_task
         run_task(tasks.reset_user_password, data['email'])
 
 
     def fill_data(self, user):
         """
         Fills user data with those from database and log out user if not 
         present in database
         :param user:
         """
 
         if not hasattr(user, 'user_id') or user.user_id is None:
             raise Exception('passed in user has to have the user_id attribute')
 
 
         log.debug('filling auth user data')
         try:
             dbuser = self.get(user.user_id)
             user.username = dbuser.username
             user.is_admin = dbuser.admin
             user.name = dbuser.name
             user.lastname = dbuser.lastname
             user.email = dbuser.email
         except:
             log.error(traceback.format_exc())
             user.is_authenticated = False
 
         return user