Merge with upstream

rhodecode/lib/db_manage.py

@@ -1,500 +1,505
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.db_manage
     ~~~~~~~~~~~~~~~~~~~~~~~
 
     Database creation, and setup module for RhodeCode. Used for creation
     of database as well as for migration operations
 
     :created_on: Apr 10, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
 import sys
 import uuid
 import logging
 from os.path import dirname as dn, join as jn
 
 from rhodecode import __dbversion__
 from rhodecode.model import meta
 
 from rhodecode.lib.auth import get_crypt_password, generate_api_key
 from rhodecode.lib.utils import ask_ok
 from rhodecode.model import init_model
 from rhodecode.model.db import User, Permission, RhodeCodeUi, \
     RhodeCodeSettings, UserToPerm, DbMigrateVersion
 
 from sqlalchemy.engine import create_engine
 
 log = logging.getLogger(__name__)
 
 
 class DbManage(object):
     def __init__(self, log_sql, dbconf, root, tests=False):
         self.dbname = dbconf.split('/')[-1]
         self.tests = tests
         self.root = root
         self.dburi = dbconf
         self.log_sql = log_sql
         self.db_exists = False
         self.init_db()
 
     def init_db(self):
         engine = create_engine(self.dburi, echo=self.log_sql)
         init_model(engine)
         self.sa = meta.Session()
 
     def create_tables(self, override=False):
         """Create a auth database
         """
 
         log.info("Any existing database is going to be destroyed")
         if self.tests:
             destroy = True
         else:
             destroy = ask_ok('Are you sure to destroy old database ? [y/n]')
         if not destroy:
             sys.exit()
         if destroy:
             meta.Base.metadata.drop_all()
 
         checkfirst = not override
         meta.Base.metadata.create_all(checkfirst=checkfirst)
         log.info('Created tables for %s', self.dbname)
 
     def set_db_version(self):
         try:
             ver = DbMigrateVersion()
             ver.version = __dbversion__
             ver.repository_id = 'rhodecode_db_migrations'
             ver.repository_path = 'versions'
             self.sa.add(ver)
             self.sa.commit()
         except:
             self.sa.rollback()
             raise
         log.info('db version set to: %s', __dbversion__)
 
     def upgrade(self):
         """Upgrades given database schema to given revision following
         all needed steps, to perform the upgrade
 
         """
 
         from rhodecode.lib.dbmigrate.migrate.versioning import api
         from rhodecode.lib.dbmigrate.migrate.exceptions import \
             DatabaseNotControlledError
 
         upgrade = ask_ok('You are about to perform database upgrade, make '
                          'sure You backed up your database before. '
                          'Continue ? [y/n]')
         if not upgrade:
             sys.exit('Nothing done')
 
         repository_path = jn(dn(dn(dn(os.path.realpath(__file__)))),
                              'rhodecode/lib/dbmigrate')
         db_uri = self.dburi
 
         try:
             curr_version = api.db_version(db_uri, repository_path)
             msg = ('Found current database under version'
                  ' control with version %s' % curr_version)
 
         except (RuntimeError, DatabaseNotControlledError):
             curr_version = 1
             msg = ('Current database is not under version control. Setting'
                    ' as version %s' % curr_version)
             api.version_control(db_uri, repository_path, curr_version)
 
         print (msg)
 
         if curr_version == __dbversion__:
             sys.exit('This database is already at the newest version')
 
         #======================================================================
         # UPGRADE STEPS
         #======================================================================
         class UpgradeSteps(object):
             """Those steps follow schema versions so for example schema
             for example schema with seq 002 == step_2 and so on.
             """
 
             def __init__(self, klass):
                 self.klass = klass
 
             def step_0(self):
                 #step 0 is the schema upgrade, and than follow proper upgrades
                 print ('attempting to do database upgrade to version %s' \
                                 % __dbversion__)
                 api.upgrade(db_uri, repository_path, __dbversion__)
                 print ('Schema upgrade completed')
 
             def step_1(self):
                 pass
 
             def step_2(self):
                 print ('Patching repo paths for newer version of RhodeCode')
                 self.klass.fix_repo_paths()
 
                 print ('Patching default user of RhodeCode')
                 self.klass.fix_default_user()
 
                 log.info('Changing ui settings')
                 self.klass.create_ui_settings()
 
             def step_3(self):
                 print ('Adding additional settings into RhodeCode db')
                 self.klass.fix_settings()
+                print ('Adding ldap defaults')
+                self.klass.create_ldap_options(skip_existing=True)
 
         upgrade_steps = [0] + range(curr_version + 1, __dbversion__ + 1)
 
         #CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
         for step in upgrade_steps:
             print ('performing upgrade step %s' % step)
             callable = getattr(UpgradeSteps(self), 'step_%s' % step)()
 
     def fix_repo_paths(self):
         """Fixes a old rhodecode version path into new one without a '*'
         """
 
         paths = self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
 
         paths.ui_value = paths.ui_value.replace('*', '')
 
         try:
             self.sa.add(paths)
             self.sa.commit()
         except:
             self.sa.rollback()
             raise
 
     def fix_default_user(self):
         """Fixes a old default user with some 'nicer' default values,
         used mostly for anonymous access
         """
         def_user = self.sa.query(User)\
                 .filter(User.username == 'default')\
                 .one()
 
         def_user.name = 'Anonymous'
         def_user.lastname = 'User'
         def_user.email = '[e-mail]'
 
         try:
             self.sa.add(def_user)
             self.sa.commit()
         except:
             self.sa.rollback()
             raise
 
     def fix_settings(self):
         """Fixes rhodecode settings adds ga_code key for google analytics
         """
 
         hgsettings3 = RhodeCodeSettings('ga_code', '')
 
         try:
             self.sa.add(hgsettings3)
             self.sa.commit()
         except:
             self.sa.rollback()
             raise
 
     def admin_prompt(self, second=False):
         if not self.tests:
             import getpass
 
             def get_password():
                 password = getpass.getpass('Specify admin password '
                                            '(min 6 chars):')
                 confirm = getpass.getpass('Confirm password:')
 
                 if password != confirm:
                     log.error('passwords mismatch')
                     return False
                 if len(password) < 6:
                     log.error('password is to short use at least 6 characters')
                     return False
 
                 return password
 
             username = raw_input('Specify admin username:')
 
             password = get_password()
             if not password:
                 #second try
                 password = get_password()
                 if not password:
                     sys.exit()
 
             email = raw_input('Specify admin email:')
             self.create_user(username, password, email, True)
         else:
             log.info('creating admin and regular test users')
             self.create_user('test_admin', 'test12',
                              '[e-mail]', True)
             self.create_user('test_regular', 'test12',
                              '[e-mail]', False)
             self.create_user('test_regular2', 'test12',
                              '[e-mail]', False)
 
     def create_ui_settings(self):
         """Creates ui settings, fills out hooks
         and disables dotencode
 
         """
         #HOOKS
         hooks1_key = RhodeCodeUi.HOOK_UPDATE
         hooks1_ = self.sa.query(RhodeCodeUi)\
             .filter(RhodeCodeUi.ui_key == hooks1_key).scalar()
 
         hooks1 = RhodeCodeUi() if hooks1_ is None else hooks1_
         hooks1.ui_section = 'hooks'
         hooks1.ui_key = hooks1_key
         hooks1.ui_value = 'hg update >&2'
         hooks1.ui_active = False
 
         hooks2_key = RhodeCodeUi.HOOK_REPO_SIZE
         hooks2_ = self.sa.query(RhodeCodeUi)\
             .filter(RhodeCodeUi.ui_key == hooks2_key).scalar()
 
         hooks2 = RhodeCodeUi() if hooks2_ is None else hooks2_
         hooks2.ui_section = 'hooks'
         hooks2.ui_key = hooks2_key
         hooks2.ui_value = 'python:rhodecode.lib.hooks.repo_size'
 
         hooks3 = RhodeCodeUi()
         hooks3.ui_section = 'hooks'
         hooks3.ui_key = RhodeCodeUi.HOOK_PUSH
         hooks3.ui_value = 'python:rhodecode.lib.hooks.log_push_action'
 
         hooks4 = RhodeCodeUi()
         hooks4.ui_section = 'hooks'
         hooks4.ui_key = RhodeCodeUi.HOOK_PULL
         hooks4.ui_value = 'python:rhodecode.lib.hooks.log_pull_action'
 
         #For mercurial 1.7 set backward comapatibility with format
         dotencode_disable = RhodeCodeUi()
         dotencode_disable.ui_section = 'format'
         dotencode_disable.ui_key = 'dotencode'
         dotencode_disable.ui_value = 'false'
 
         try:
             self.sa.add(hooks1)
             self.sa.add(hooks2)
             self.sa.add(hooks3)
             self.sa.add(hooks4)
             self.sa.add(dotencode_disable)
             self.sa.commit()
         except:
             self.sa.rollback()
             raise
 
-    def create_ldap_options(self):
+    def create_ldap_options(self,skip_existing=False):
         """Creates ldap settings"""
 
         try:
             for k, v in [('ldap_active', 'false'), ('ldap_host', ''),
                         ('ldap_port', '389'), ('ldap_tls_kind', 'PLAIN'),
                         ('ldap_tls_reqcert', ''), ('ldap_dn_user', ''),
                         ('ldap_dn_pass', ''), ('ldap_base_dn', ''),
                         ('ldap_filter', ''), ('ldap_search_scope', ''),
                         ('ldap_attr_login', ''), ('ldap_attr_firstname', ''),
                         ('ldap_attr_lastname', ''), ('ldap_attr_email', '')]:
 
+                if skip_existing and RhodeCodeSettings.get_by_name(k) != None:
+                    log.debug('Skipping option %s' % k)
+                    continue
                 setting = RhodeCodeSettings(k, v)
                 self.sa.add(setting)
             self.sa.commit()
         except:
             self.sa.rollback()
             raise
 
     def config_prompt(self, test_repo_path='', retries=3):
         if retries == 3:
             log.info('Setting up repositories config')
 
         if not self.tests and not test_repo_path:
             path = raw_input('Specify valid full path to your repositories'
                         ' you can change this later in application settings:')
         else:
             path = test_repo_path
         path_ok = True
 
         #check proper dir
         if not os.path.isdir(path):
             path_ok = False
             log.error('Given path %s is not a valid directory', path)
 
         #check write access
         if not os.access(path, os.W_OK) and path_ok:
             path_ok = False
             log.error('No write permission to given path %s', path)
 
 
         if retries == 0:
             sys.exit('max retries reached')
         if path_ok is False:
             retries -= 1
             return self.config_prompt(test_repo_path, retries)
 
         return path
 
     def create_settings(self, path):
 
         self.create_ui_settings()
 
         #HG UI OPTIONS
         web1 = RhodeCodeUi()
         web1.ui_section = 'web'
         web1.ui_key = 'push_ssl'
         web1.ui_value = 'false'
 
         web2 = RhodeCodeUi()
         web2.ui_section = 'web'
         web2.ui_key = 'allow_archive'
         web2.ui_value = 'gz zip bz2'
 
         web3 = RhodeCodeUi()
         web3.ui_section = 'web'
         web3.ui_key = 'allow_push'
         web3.ui_value = '*'
 
         web4 = RhodeCodeUi()
         web4.ui_section = 'web'
         web4.ui_key = 'baseurl'
         web4.ui_value = '/'
 
         paths = RhodeCodeUi()
         paths.ui_section = 'paths'
         paths.ui_key = '/'
         paths.ui_value = path
 
         hgsettings1 = RhodeCodeSettings('realm', 'RhodeCode authentication')
         hgsettings2 = RhodeCodeSettings('title', 'RhodeCode')
         hgsettings3 = RhodeCodeSettings('ga_code', '')
 
         try:
             self.sa.add(web1)
             self.sa.add(web2)
             self.sa.add(web3)
             self.sa.add(web4)
             self.sa.add(paths)
             self.sa.add(hgsettings1)
             self.sa.add(hgsettings2)
             self.sa.add(hgsettings3)
 
             self.sa.commit()
         except:
             self.sa.rollback()
             raise
 
         self.create_ldap_options()
 
         log.info('created ui config')
 
     def create_user(self, username, password, email='', admin=False):
         log.info('creating administrator user %s', username)
         
         form_data = dict(username=username,
                          password=password,
                          active=True,
                          admin=admin,
                          name='RhodeCode',
                          lastname='Admin',
                          email=email)
         User.create(form_data)
 
 
     def create_default_user(self):
         log.info('creating default user')
         #create default user for handling default permissions.
 
         form_data = dict(username='default',
                          password=str(uuid.uuid1())[:8],
                          active=False,
                          admin=False,
                          name='Anonymous',
                          lastname='User',
                          email='[e-mail]')
         User.create(form_data)
         
     def create_permissions(self):
         #module.(access|create|change|delete)_[name]
         #module.(read|write|owner)
         perms = [('repository.none', 'Repository no access'),
                  ('repository.read', 'Repository read access'),
                  ('repository.write', 'Repository write access'),
                  ('repository.admin', 'Repository admin access'),
                  ('hg.admin', 'Hg Administrator'),
                  ('hg.create.repository', 'Repository create'),
                  ('hg.create.none', 'Repository creation disabled'),
                  ('hg.register.none', 'Register disabled'),
                  ('hg.register.manual_activate', 'Register new user with '
                                                  'RhodeCode without manual'
                                                  'activation'),
 
                  ('hg.register.auto_activate', 'Register new user with '
                                                'RhodeCode without auto '
                                                'activation'),
                 ]
 
         for p in perms:
             new_perm = Permission()
             new_perm.permission_name = p[0]
             new_perm.permission_longname = p[1]
             try:
                 self.sa.add(new_perm)
                 self.sa.commit()
             except:
                 self.sa.rollback()
                 raise
 
     def populate_default_permissions(self):
         log.info('creating default user permissions')
 
         default_user = self.sa.query(User)\
         .filter(User.username == 'default').scalar()
 
         reg_perm = UserToPerm()
         reg_perm.user = default_user
         reg_perm.permission = self.sa.query(Permission)\
         .filter(Permission.permission_name == 'hg.register.manual_activate')\
         .scalar()
 
         create_repo_perm = UserToPerm()
         create_repo_perm.user = default_user
         create_repo_perm.permission = self.sa.query(Permission)\
         .filter(Permission.permission_name == 'hg.create.repository')\
         .scalar()
 
         default_repo_perm = UserToPerm()
         default_repo_perm.user = default_user
         default_repo_perm.permission = self.sa.query(Permission)\
         .filter(Permission.permission_name == 'repository.read')\
         .scalar()
 
         try:
             self.sa.add(reg_perm)
             self.sa.add(create_repo_perm)
             self.sa.add(default_repo_perm)
             self.sa.commit()
         except:
             self.sa.rollback()
             raise

rhodecode/lib/dbmigrate/versions/003_version_1_2_0.py

@@ -1,116 +1,119
 import logging
 import datetime
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper
 from sqlalchemy.orm.session import Session
 
 from rhodecode.lib.dbmigrate.migrate import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 
 from rhodecode.model.meta import Base
 
 log = logging.getLogger(__name__)
 
 def upgrade(migrate_engine):
     """ Upgrade operations go here.
     Don't create your own engine; bind migrate_engine to your metadata
     """
 
     #==========================================================================
     # Add table `groups``
     #==========================================================================
     from rhodecode.model.db import Group
     Group().__table__.create()
 
     #==========================================================================
     # Add table `group_to_perm`
     #==========================================================================
     from rhodecode.model.db import GroupToPerm
     GroupToPerm().__table__.create()
 
     #==========================================================================
     # Add table `users_groups`
     #==========================================================================
     from rhodecode.model.db import UsersGroup
     UsersGroup().__table__.create()
 
     #==========================================================================
     # Add table `users_groups_members`
     #==========================================================================
     from rhodecode.model.db import UsersGroupMember
     UsersGroupMember().__table__.create()
 
     #==========================================================================
     # Add table `users_group_repo_to_perm`
     #==========================================================================
     from rhodecode.model.db import UsersGroupRepoToPerm
     UsersGroupRepoToPerm().__table__.create()
 
     #==========================================================================
     # Add table `users_group_to_perm`
     #==========================================================================
     from rhodecode.model.db import UsersGroupToPerm
     UsersGroupToPerm().__table__.create()
 
     #==========================================================================
     # Upgrade of `users` table
     #==========================================================================
     from rhodecode.model.db import User
 
     #add column
     ldap_dn = Column("ldap_dn", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ldap_dn.create(User().__table__)
 
     api_key = Column("api_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     api_key.create(User().__table__)
 
     #remove old column
     is_ldap = Column("is_ldap", Boolean(), nullable=False, unique=None, default=False)
     is_ldap.drop(User().__table__)
 
 
     #==========================================================================
     # Upgrade of `repositories` table
     #==========================================================================
     from rhodecode.model.db import Repository
 
+    #ADD clone_uri column#
+
+    clone_uri = Column("clone_uri", String(length=255, convert_unicode=False,
+                                           assert_unicode=None),
+                        nullable=True, unique=False, default=None)
+
+    clone_uri.create(Repository().__table__)
+    
     #ADD downloads column#
     enable_downloads = Column("downloads", Boolean(), nullable=True, unique=None, default=True)
     enable_downloads.create(Repository().__table__)
 
     #ADD column created_on
     created_on = Column('created_on', DateTime(timezone=False), nullable=True,
                         unique=None, default=datetime.datetime.now)
     created_on.create(Repository().__table__)
 
     #ADD group_id column#
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'),
                   nullable=True, unique=False, default=None)
 
     group_id.create(Repository().__table__)
 
 
-    #ADD clone_uri column#
-
-    clone_uri = Column("clone_uri", String(length=255, convert_unicode=False,
-                                           assert_unicode=None),
-                        nullable=True, unique=False, default=None)
-
-    clone_uri.create(Repository().__table__)
-
-
     #==========================================================================
     # Upgrade of `user_followings` table
     #==========================================================================
 
-    follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
-    follows_from.create(Repository().__table__)
+    from rhodecode.model.db import UserFollowing
+
+    follows_from = Column('follows_from', DateTime(timezone=False), 
+                          nullable=True, unique=None, 
+                          default=datetime.datetime.now)
+    follows_from.create(UserFollowing().__table__)
 
     return
 
 
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine

setup.py

@@ -1,124 +1,125
 import sys
 from rhodecode import get_version
 from rhodecode import __platform__
 from rhodecode import __license__
 from rhodecode import PLATFORM_OTHERS
 
 py_version = sys.version_info
 
 if py_version < (2, 5):
     raise Exception('RhodeCode requires python 2.5 or later')
 
 requirements = [
         "Pylons==1.0.0",
         "WebHelpers>=1.2",
+        "formencode==1.2.4",
         "SQLAlchemy>=0.7.2,<0.8",
         "Mako>=0.4.2",
         "pygments>=1.4",
         "mercurial>=1.9,<2.0",
         "whoosh<1.8",
         "celery>=2.2.5,<2.3",
         "babel",
         "python-dateutil>=1.5.0,<2.0.0",
         "dulwich>=0.8.0",
         "vcs>=0.2.1.dev",
         "webob==1.0.8"    
     ]
 
 dependency_links = [
     "https://secure.rhodecode.org/vcs/archive/default.zip#egg=vcs-0.2.1.dev",
     "https://bitbucket.org/marcinkuzminski/vcs/get/default.zip#egg=vcs-0.2.1.dev",
 ]
 
 classifiers = ['Development Status :: 4 - Beta',
                'Environment :: Web Environment',
                'Framework :: Pylons',
                'Intended Audience :: Developers',
                'Operating System :: OS Independent',
                'Programming Language :: Python',
                'Programming Language :: Python :: 2.5',
                'Programming Language :: Python :: 2.6',
                'Programming Language :: Python :: 2.7', ]
 
 if py_version < (2, 6):
     requirements.append("simplejson")
     requirements.append("pysqlite")
 
 if __platform__ in PLATFORM_OTHERS:
     requirements.append("py-bcrypt")
 
 
 #additional files from project that goes somewhere in the filesystem
 #relative to sys.prefix
 data_files = []
 
 #additional files that goes into package itself
 package_data = {'rhodecode': ['i18n/*/LC_MESSAGES/*.mo', ], }
 
 description = ('Mercurial repository browser/management with '
                'build in push/pull server and full text search')
 keywords = ' '.join(['rhodecode', 'rhodiumcode', 'mercurial', 'git',
                       'repository management', 'hgweb replacement'
                       'hgwebdir', 'gitweb replacement', 'serving hgweb', ])
 #long description
 try:
     readme_file = 'README.rst'
     changelog_file = 'docs/changelog.rst'
     long_description = open(readme_file).read() + '\n\n' + \
         open(changelog_file).read()
 
 except IOError, err:
     sys.stderr.write("[WARNING] Cannot find file specified as "
         "long_description (%s)\n or changelog (%s) skipping that file" \
             % (readme_file, changelog_file))
     long_description = description
 
 
 try:
     from setuptools import setup, find_packages
 except ImportError:
     from ez_setup import use_setuptools
     use_setuptools()
     from setuptools import setup, find_packages
 #packages
 packages = find_packages(exclude=['ez_setup'])
 
 setup(
     name='RhodeCode',
     version=get_version(),
     description=description,
     long_description=long_description,
     keywords=keywords,
     license=__license__,
     author='Marcin Kuzminski',
     author_email='[e-mail]',
     dependency_links=dependency_links,
     url='http://rhodecode.org',
     install_requires=requirements,
     classifiers=classifiers,
     setup_requires=["PasteScript>=1.6.3"],
     data_files=data_files,
     packages=packages,
     include_package_data=True,
     test_suite='nose.collector',
     package_data=package_data,
     message_extractors={'rhodecode': [
             ('**.py', 'python', None),
             ('templates/**.mako', 'mako', {'input_encoding': 'utf-8'}),
             ('templates/**.html', 'mako', {'input_encoding': 'utf-8'}),
             ('public/**', 'ignore', None)]},
     zip_safe=False,
     paster_plugins=['PasteScript', 'Pylons'],
     entry_points="""
     [paste.app_factory]
     main = rhodecode.config.middleware:make_app
 
     [paste.app_install]
     main = pylons.util:PylonsInstaller
 
     [paste.global_paster_command]
     make-index = rhodecode.lib.indexers:MakeIndex
     upgrade-db = rhodecode.lib.dbmigrate:UpgradeDb
     celeryd=rhodecode.lib.celerypylons.commands:CeleryDaemonCommand
     """,
 )