core: use py3 compatible prints

rhodecode/api/tests/test_update_pull_request.py

@@ -1,213 +1,212 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.model.db import User
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_ok, assert_error)
 
 
 @pytest.mark.usefixtures("testuser_api", "app")
 class TestUpdatePullRequest(object):
 
     @pytest.mark.backends("git", "hg")
     def test_api_update_pull_request_title_or_description(
             self, pr_util, no_notifications):
         pull_request = pr_util.create_pull_request()
 
         id_, params = build_data(
             self.apikey, 'update_pull_request',
             repoid=pull_request.target_repo.repo_name,
             pullrequestid=pull_request.pull_request_id,
             title='New TITLE OF A PR',
             description='New DESC OF A PR',
         )
         response = api_call(self.app, params)
 
         expected = {
             "msg": "Updated pull request `{}`".format(
                 pull_request.pull_request_id),
             "pull_request": response.json['result']['pull_request'],
             "updated_commits": {"added": [], "common": [], "removed": []},
             "updated_reviewers": {"added": [], "removed": []},
         }
 
         response_json = response.json['result']
         assert response_json == expected
         pr = response_json['pull_request']
         assert pr['title'] == 'New TITLE OF A PR'
         assert pr['description'] == 'New DESC OF A PR'
 
     @pytest.mark.backends("git", "hg")
     def test_api_try_update_closed_pull_request(
             self, pr_util, no_notifications):
         pull_request = pr_util.create_pull_request()
         PullRequestModel().close_pull_request(
             pull_request, TEST_USER_ADMIN_LOGIN)
 
         id_, params = build_data(
             self.apikey, 'update_pull_request',
             repoid=pull_request.target_repo.repo_name,
             pullrequestid=pull_request.pull_request_id)
         response = api_call(self.app, params)
 
         expected = 'pull request `{}` update failed, pull request ' \
                    'is closed'.format(pull_request.pull_request_id)
 
         assert_error(id_, expected, response.body)
 
     @pytest.mark.backends("git", "hg")
     def test_api_update_update_commits(self, pr_util, no_notifications):
         commits = [
             {'message': 'a'},
             {'message': 'b', 'added': [FileNode('file_b', 'test_content\n')]},
             {'message': 'c', 'added': [FileNode('file_c', 'test_content\n')]},
         ]
         pull_request = pr_util.create_pull_request(
             commits=commits, target_head='a', source_head='b', revisions=['b'])
         pr_util.update_source_repository(head='c')
         repo = pull_request.source_repo.scm_instance()
         commits = [x for x in repo.get_commits()]
-        print commits
 
         added_commit_id = commits[-1].raw_id  # c commit
         common_commit_id = commits[1].raw_id  # b commit is common ancestor
         total_commits = [added_commit_id, common_commit_id]
 
         id_, params = build_data(
             self.apikey, 'update_pull_request',
             repoid=pull_request.target_repo.repo_name,
             pullrequestid=pull_request.pull_request_id,
             update_commits=True
         )
         response = api_call(self.app, params)
 
         expected = {
             "msg": "Updated pull request `{}`".format(
                 pull_request.pull_request_id),
             "pull_request": response.json['result']['pull_request'],
             "updated_commits": {"added": [added_commit_id],
                                 "common": [common_commit_id],
                                 "total": total_commits,
                                 "removed": []},
             "updated_reviewers": {"added": [], "removed": []},
         }
 
         assert_ok(id_, expected, response.body)
 
     @pytest.mark.backends("git", "hg")
     def test_api_update_change_reviewers(
             self, user_util, pr_util, no_notifications):
         a = user_util.create_user()
         b = user_util.create_user()
         c = user_util.create_user()
         new_reviewers = [
             {'username': b.username,'reasons': ['updated via API'],
              'mandatory':False},
             {'username': c.username, 'reasons': ['updated via API'],
              'mandatory':False},
         ]
 
         added = [b.username, c.username]
         removed = [a.username]
 
         pull_request = pr_util.create_pull_request(
             reviewers=[(a.username, ['added via API'], False, [])])
 
         id_, params = build_data(
             self.apikey, 'update_pull_request',
             repoid=pull_request.target_repo.repo_name,
             pullrequestid=pull_request.pull_request_id,
             reviewers=new_reviewers)
         response = api_call(self.app, params)
         expected = {
             "msg": "Updated pull request `{}`".format(
                 pull_request.pull_request_id),
             "pull_request": response.json['result']['pull_request'],
             "updated_commits": {"added": [], "common": [], "removed": []},
             "updated_reviewers": {"added": added, "removed": removed},
         }
 
         assert_ok(id_, expected, response.body)
 
     @pytest.mark.backends("git", "hg")
     def test_api_update_bad_user_in_reviewers(self, pr_util):
         pull_request = pr_util.create_pull_request()
 
         id_, params = build_data(
             self.apikey, 'update_pull_request',
             repoid=pull_request.target_repo.repo_name,
             pullrequestid=pull_request.pull_request_id,
             reviewers=[{'username': 'bad_name'}])
         response = api_call(self.app, params)
 
         expected = 'user `bad_name` does not exist'
 
         assert_error(id_, expected, response.body)
 
     @pytest.mark.backends("git", "hg")
     def test_api_update_repo_error(self, pr_util):
         pull_request = pr_util.create_pull_request()
         id_, params = build_data(
             self.apikey, 'update_pull_request',
             repoid='fake',
             pullrequestid=pull_request.pull_request_id,
             reviewers=[{'username': 'bad_name'}])
         response = api_call(self.app, params)
 
         expected = 'repository `fake` does not exist'
 
         response_json = response.json['error']
         assert response_json == expected
 
     @pytest.mark.backends("git", "hg")
     def test_api_update_pull_request_error(self, pr_util):
         pull_request = pr_util.create_pull_request()
 
         id_, params = build_data(
             self.apikey, 'update_pull_request',
             repoid=pull_request.target_repo.repo_name,
             pullrequestid=999999,
             reviewers=[{'username': 'bad_name'}])
         response = api_call(self.app, params)
 
         expected = 'pull request `999999` does not exist'
         assert_error(id_, expected, response.body)
 
     @pytest.mark.backends("git", "hg")
     def test_api_update_pull_request_no_perms_to_update(
             self, user_util, pr_util):
         user = user_util.create_user()
         pull_request = pr_util.create_pull_request()
 
         id_, params = build_data(
             user.api_key, 'update_pull_request',
             repoid=pull_request.target_repo.repo_name,
             pullrequestid=pull_request.pull_request_id,)
         response = api_call(self.app, params)
 
         expected = ('pull request `%s` update failed, '
                     'no permission to update.') % pull_request.pull_request_id
 
         assert_error(id_, expected, response.body)

rhodecode/lib/db_manage.py

@@ -1,621 +1,621 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Database creation, and setup module for RhodeCode Enterprise. Used for creation
 of database as well as for migration operations
 """
 
 import os
 import sys
 import time
 import uuid
 import logging
 import getpass
 from os.path import dirname as dn, join as jn
 
 from sqlalchemy.engine import create_engine
 
 from rhodecode import __dbversion__
 from rhodecode.model import init_model
 from rhodecode.model.user import UserModel
 from rhodecode.model.db import (
     User, Permission, RhodeCodeUi, RhodeCodeSetting, UserToPerm,
     DbMigrateVersion, RepoGroup, UserRepoGroupToPerm, CacheKey, Repository)
 from rhodecode.model.meta import Session, Base
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import SettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 def notify(msg):
     """
     Notification for migrations messages
     """
     ml = len(msg) + (4 * 2)
     print(('\n%s\n*** %s ***\n%s' % ('*' * ml, msg, '*' * ml)).upper())
 
 
 class DbManage(object):
 
     def __init__(self, log_sql, dbconf, root, tests=False,
                  SESSION=None, cli_args=None):
         self.dbname = dbconf.split('/')[-1]
         self.tests = tests
         self.root = root
         self.dburi = dbconf
         self.log_sql = log_sql
         self.db_exists = False
         self.cli_args = cli_args or {}
         self.init_db(SESSION=SESSION)
         self.ask_ok = self.get_ask_ok_func(self.cli_args.get('force_ask'))
 
     def get_ask_ok_func(self, param):
         if param not in [None]:
             # return a function lambda that has a default set to param
             return lambda *args, **kwargs: param
         else:
             from rhodecode.lib.utils import ask_ok
             return ask_ok
 
     def init_db(self, SESSION=None):
         if SESSION:
             self.sa = SESSION
         else:
             # init new sessions
             engine = create_engine(self.dburi, echo=self.log_sql)
             init_model(engine)
             self.sa = Session()
 
     def create_tables(self, override=False):
         """
         Create a auth database
         """
 
         log.info("Existing database with the same name is going to be destroyed.")
         log.info("Setup command will run DROP ALL command on that database.")
         if self.tests:
             destroy = True
         else:
             destroy = self.ask_ok('Are you sure that you want to destroy the old database? [y/n]')
         if not destroy:
             log.info('Nothing done.')
             sys.exit(0)
         if destroy:
             Base.metadata.drop_all()
 
         checkfirst = not override
         Base.metadata.create_all(checkfirst=checkfirst)
         log.info('Created tables for %s' % self.dbname)
 
     def set_db_version(self):
         ver = DbMigrateVersion()
         ver.version = __dbversion__
         ver.repository_id = 'rhodecode_db_migrations'
         ver.repository_path = 'versions'
         self.sa.add(ver)
         log.info('db version set to: %s' % __dbversion__)
 
     def run_pre_migration_tasks(self):
         """
         Run various tasks before actually doing migrations
         """
         # delete cache keys on each upgrade
         total = CacheKey.query().count()
         log.info("Deleting (%s) cache keys now...", total)
         CacheKey.delete_all_cache()
 
     def upgrade(self, version=None):
         """
         Upgrades given database schema to given revision following
         all needed steps, to perform the upgrade
 
         """
 
         from rhodecode.lib.dbmigrate.migrate.versioning import api
         from rhodecode.lib.dbmigrate.migrate.exceptions import \
             DatabaseNotControlledError
 
         if 'sqlite' in self.dburi:
-            print (
+            print(
                '********************** WARNING **********************\n'
                'Make sure your version of sqlite is at least 3.7.X.  \n'
                'Earlier versions are known to fail on some migrations\n'
                '*****************************************************\n')
 
         upgrade = self.ask_ok(
             'You are about to perform a database upgrade. Make '
             'sure you have backed up your database. '
             'Continue ? [y/n]')
         if not upgrade:
             log.info('No upgrade performed')
             sys.exit(0)
 
         repository_path = jn(dn(dn(dn(os.path.realpath(__file__)))),
                              'rhodecode/lib/dbmigrate')
         db_uri = self.dburi
 
         try:
             curr_version = version or api.db_version(db_uri, repository_path)
             msg = ('Found current database db_uri under version '
                    'control with version {}'.format(curr_version))
 
         except (RuntimeError, DatabaseNotControlledError):
             curr_version = 1
             msg = ('Current database is not under version control. Setting '
                    'as version %s' % curr_version)
             api.version_control(db_uri, repository_path, curr_version)
 
         notify(msg)
 
         self.run_pre_migration_tasks()
 
         if curr_version == __dbversion__:
             log.info('This database is already at the newest version')
             sys.exit(0)
 
         upgrade_steps = range(curr_version + 1, __dbversion__ + 1)
         notify('attempting to upgrade database from '
                'version %s to version %s' % (curr_version, __dbversion__))
 
         # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
         _step = None
         for step in upgrade_steps:
             notify('performing upgrade step %s' % step)
             time.sleep(0.5)
 
             api.upgrade(db_uri, repository_path, step)
             self.sa.rollback()
             notify('schema upgrade for step %s completed' % (step,))
 
             _step = step
 
         notify('upgrade to version %s successful' % _step)
 
     def fix_repo_paths(self):
         """
         Fixes an old RhodeCode version path into new one without a '*'
         """
 
         paths = self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
 
         paths.ui_value = paths.ui_value.replace('*', '')
 
         try:
             self.sa.add(paths)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_default_user(self):
         """
         Fixes an old default user with some 'nicer' default values,
         used mostly for anonymous access
         """
         def_user = self.sa.query(User)\
                 .filter(User.username == User.DEFAULT_USER)\
                 .one()
 
         def_user.name = 'Anonymous'
         def_user.lastname = 'User'
         def_user.email = User.DEFAULT_USER_EMAIL
 
         try:
             self.sa.add(def_user)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_settings(self):
         """
         Fixes rhodecode settings and adds ga_code key for google analytics
         """
 
         hgsettings3 = RhodeCodeSetting('ga_code', '')
 
         try:
             self.sa.add(hgsettings3)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def create_admin_and_prompt(self):
 
         # defaults
         defaults = self.cli_args
         username = defaults.get('username')
         password = defaults.get('password')
         email = defaults.get('email')
 
         if username is None:
             username = raw_input('Specify admin username:')
         if password is None:
             password = self._get_admin_password()
             if not password:
                 # second try
                 password = self._get_admin_password()
                 if not password:
                     sys.exit()
         if email is None:
             email = raw_input('Specify admin email:')
         api_key = self.cli_args.get('api_key')
         self.create_user(username, password, email, True,
                          strict_creation_check=False,
                          api_key=api_key)
 
     def _get_admin_password(self):
         password = getpass.getpass('Specify admin password '
                                    '(min 6 chars):')
         confirm = getpass.getpass('Confirm password:')
 
         if password != confirm:
             log.error('passwords mismatch')
             return False
         if len(password) < 6:
             log.error('password is too short - use at least 6 characters')
             return False
 
         return password
 
     def create_test_admin_and_users(self):
         log.info('creating admin and regular test users')
         from rhodecode.tests import TEST_USER_ADMIN_LOGIN, \
             TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, \
             TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, \
             TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR2_LOGIN, \
             TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL
 
         self.create_user(TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
                          TEST_USER_ADMIN_EMAIL, True, api_key=True)
 
         self.create_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
                          TEST_USER_REGULAR_EMAIL, False, api_key=True)
 
         self.create_user(TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS,
                          TEST_USER_REGULAR2_EMAIL, False, api_key=True)
 
     def create_ui_settings(self, repo_store_path):
         """
         Creates ui settings, fills out hooks
         and disables dotencode
         """
         settings_model = SettingsModel(sa=self.sa)
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         # Build HOOKS
         hooks = [
             (RhodeCodeUi.HOOK_REPO_SIZE, 'python:vcsserver.hooks.repo_size'),
 
             # HG
             (RhodeCodeUi.HOOK_PRE_PULL, 'python:vcsserver.hooks.pre_pull'),
             (RhodeCodeUi.HOOK_PULL, 'python:vcsserver.hooks.log_pull_action'),
             (RhodeCodeUi.HOOK_PRE_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PRETX_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PUSH, 'python:vcsserver.hooks.log_push_action'),
             (RhodeCodeUi.HOOK_PUSH_KEY, 'python:vcsserver.hooks.key_push'),
 
         ]
 
         for key, value in hooks:
             hook_obj = settings_model.get_ui_by_key(key)
             hooks2 = hook_obj if hook_obj else RhodeCodeUi()
             hooks2.ui_section = 'hooks'
             hooks2.ui_key = key
             hooks2.ui_value = value
             self.sa.add(hooks2)
 
         # enable largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'extensions'
         largefiles.ui_key = 'largefiles'
         largefiles.ui_value = ''
         self.sa.add(largefiles)
 
         # set default largefiles cache dir, defaults to
         # /repo_store_location/.cache/largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'largefiles'
         largefiles.ui_key = 'usercache'
         largefiles.ui_value = largefiles_store(repo_store_path)
 
         self.sa.add(largefiles)
 
         # set default lfs cache dir, defaults to
         # /repo_store_location/.cache/lfs_store
         lfsstore = RhodeCodeUi()
         lfsstore.ui_section = 'vcs_git_lfs'
         lfsstore.ui_key = 'store_location'
         lfsstore.ui_value = lfs_store(repo_store_path)
 
         self.sa.add(lfsstore)
 
         # enable hgsubversion disabled by default
         hgsubversion = RhodeCodeUi()
         hgsubversion.ui_section = 'extensions'
         hgsubversion.ui_key = 'hgsubversion'
         hgsubversion.ui_value = ''
         hgsubversion.ui_active = False
         self.sa.add(hgsubversion)
 
         # enable hgevolve disabled by default
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'extensions'
         hgevolve.ui_key = 'evolve'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         # enable hggit disabled by default
         hggit = RhodeCodeUi()
         hggit.ui_section = 'extensions'
         hggit.ui_key = 'hggit'
         hggit.ui_value = ''
         hggit.ui_active = False
         self.sa.add(hggit)
 
         # set svn branch defaults
         branches = ["/branches/*", "/trunk"]
         tags = ["/tags/*"]
 
         for branch in branches:
             settings_model.create_ui_section_value(
                 RhodeCodeUi.SVN_BRANCH_ID, branch)
 
         for tag in tags:
             settings_model.create_ui_section_value(RhodeCodeUi.SVN_TAG_ID, tag)
 
     def create_auth_plugin_options(self, skip_existing=False):
         """
         Create default auth plugin settings, and make it active
 
         :param skip_existing:
         """
 
         for k, v, t in [('auth_plugins', 'egg:rhodecode-enterprise-ce#rhodecode', 'list'),
                         ('auth_rhodecode_enabled', 'True', 'bool')]:
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s' % k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def create_default_options(self, skip_existing=False):
         """Creates default settings"""
 
         for k, v, t in [
             ('default_repo_enable_locking',  False, 'bool'),
             ('default_repo_enable_downloads', False, 'bool'),
             ('default_repo_enable_statistics', False, 'bool'),
             ('default_repo_private', False, 'bool'),
             ('default_repo_type', 'hg', 'unicode')]:
 
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s' % k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def fixup_groups(self):
         def_usr = User.get_default_user()
         for g in RepoGroup.query().all():
             g.group_name = g.get_new_name(g.name)
             self.sa.add(g)
             # get default perm
             default = UserRepoGroupToPerm.query()\
                 .filter(UserRepoGroupToPerm.group == g)\
                 .filter(UserRepoGroupToPerm.user == def_usr)\
                 .scalar()
 
             if default is None:
                 log.debug('missing default permission for group %s adding' % g)
                 perm_obj = RepoGroupModel()._create_default_perms(g)
                 self.sa.add(perm_obj)
 
     def reset_permissions(self, username):
         """
         Resets permissions to default state, useful when old systems had
         bad permissions, we must clean them up
 
         :param username:
         """
         default_user = User.get_by_username(username)
         if not default_user:
             return
 
         u2p = UserToPerm.query()\
             .filter(UserToPerm.user == default_user).all()
         fixed = False
         if len(u2p) != len(Permission.DEFAULT_USER_PERMISSIONS):
             for p in u2p:
                 Session().delete(p)
             fixed = True
             self.populate_default_permissions()
         return fixed
 
     def update_repo_info(self):
         RepoModel.update_repoinfo()
 
     def config_prompt(self, test_repo_path='', retries=3):
         defaults = self.cli_args
         _path = defaults.get('repos_location')
         if retries == 3:
             log.info('Setting up repositories config')
 
         if _path is not None:
             path = _path
         elif not self.tests and not test_repo_path:
             path = raw_input(
                  'Enter a valid absolute path to store repositories. '
                  'All repositories in that path will be added automatically:'
             )
         else:
             path = test_repo_path
         path_ok = True
 
         # check proper dir
         if not os.path.isdir(path):
             path_ok = False
             log.error('Given path %s is not a valid directory' % (path,))
 
         elif not os.path.isabs(path):
             path_ok = False
             log.error('Given path %s is not an absolute path' % (path,))
 
         # check if path is at least readable.
         if not os.access(path, os.R_OK):
             path_ok = False
             log.error('Given path %s is not readable' % (path,))
 
         # check write access, warn user about non writeable paths
         elif not os.access(path, os.W_OK) and path_ok:
             log.warning('No write permission to given path %s' % (path,))
 
             q = ('Given path %s is not writeable, do you want to '
                  'continue with read only mode ? [y/n]' % (path,))
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         if retries == 0:
             sys.exit('max retries reached')
         if not path_ok:
             retries -= 1
             return self.config_prompt(test_repo_path, retries)
 
         real_path = os.path.normpath(os.path.realpath(path))
 
         if real_path != os.path.normpath(path):
             q = ('Path looks like a symlink, RhodeCode Enterprise will store '
                  'given path as %s ? [y/n]') % (real_path,)
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         return real_path
 
     def create_settings(self, path):
 
         self.create_ui_settings(path)
 
         ui_config = [
             ('web', 'push_ssl', 'False'),
             ('web', 'allow_archive', 'gz zip bz2'),
             ('web', 'allow_push', '*'),
             ('web', 'baseurl', '/'),
             ('paths', '/', path),
             ('phases', 'publish', 'True')
         ]
         for section, key, value in ui_config:
             ui_conf = RhodeCodeUi()
             setattr(ui_conf, 'ui_section', section)
             setattr(ui_conf, 'ui_key', key)
             setattr(ui_conf, 'ui_value', value)
             self.sa.add(ui_conf)
 
         # rhodecode app settings
         settings = [
             ('realm', 'RhodeCode', 'unicode'),
             ('title', '', 'unicode'),
             ('pre_code', '', 'unicode'),
             ('post_code', '', 'unicode'),
             ('show_public_icon', True, 'bool'),
             ('show_private_icon', True, 'bool'),
             ('stylify_metatags', False, 'bool'),
             ('dashboard_items', 100, 'int'),
             ('admin_grid_items', 25, 'int'),
             ('show_version', True, 'bool'),
             ('use_gravatar', False, 'bool'),
             ('gravatar_url', User.DEFAULT_GRAVATAR_URL, 'unicode'),
             ('clone_uri_tmpl', Repository.DEFAULT_CLONE_URI, 'unicode'),
             ('support_url', '', 'unicode'),
             ('update_url', RhodeCodeSetting.DEFAULT_UPDATE_URL, 'unicode'),
             ('show_revision_number', True, 'bool'),
             ('show_sha_length', 12, 'int'),
         ]
 
         for key, val, type_ in settings:
             sett = RhodeCodeSetting(key, val, type_)
             self.sa.add(sett)
 
         self.create_auth_plugin_options()
         self.create_default_options()
 
         log.info('created ui config')
 
     def create_user(self, username, password, email='', admin=False,
                     strict_creation_check=True, api_key=None):
         log.info('creating user `%s`' % username)
         user = UserModel().create_or_update(
             username, password, email, firstname=u'RhodeCode', lastname=u'Admin',
             active=True, admin=admin, extern_type="rhodecode",
             strict_creation_check=strict_creation_check)
 
         if api_key:
             log.info('setting a new default auth token for user `%s`', username)
             UserModel().add_auth_token(
                 user=user, lifetime_minutes=-1,
                 role=UserModel.auth_token_role.ROLE_ALL,
                 description=u'BUILTIN TOKEN')
 
     def create_default_user(self):
         log.info('creating default user')
         # create default user for handling default permissions.
         user = UserModel().create_or_update(username=User.DEFAULT_USER,
                                             password=str(uuid.uuid1())[:20],
                                             email=User.DEFAULT_USER_EMAIL,
                                             firstname=u'Anonymous',
                                             lastname=u'User',
                                             strict_creation_check=False)
         # based on configuration options activate/de-activate this user which
         # controlls anonymous access
         if self.cli_args.get('public_access') is False:
             log.info('Public access disabled')
             user.active = False
             Session().add(user)
             Session().commit()
 
     def create_permissions(self):
         """
         Creates all permissions defined in the system
         """
         # module.(access|create|change|delete)_[name]
         # module.(none|read|write|admin)
         log.info('creating permissions')
         PermissionModel(self.sa).create_permissions()
 
     def populate_default_permissions(self):
         """
         Populate default permissions. It will create only the default
         permissions that are missing, and not alter already defined ones
         """
         log.info('creating default user permissions')
         PermissionModel(self.sa).create_default_user_permissions(user=User.DEFAULT_USER)

rhodecode/lib/dbmigrate/migrate/versioning/migrate_repository.py

@@ -1,100 +1,100 b''
 """
    Script to migrate repository from sqlalchemy <= 0.4.4 to the new
    repository schema. This shouldn't use any other migrate modules, so
    that it can work in any version.
 """
 
 import os
 import sys
 import logging
 
 log = logging.getLogger(__name__)
 
 
 def usage():
     """Gives usage information."""
-    print """Usage: %(prog)s repository-to-migrate
+    print("""Usage: %(prog)s repository-to-migrate
 
     Upgrade your repository to the new flat format.
 
     NOTE: You should probably make a backup before running this.
-    """ % {'prog': sys.argv[0]}
+    """ % {'prog': sys.argv[0]})
 
     sys.exit(1)
 
 
 def delete_file(filepath):
     """Deletes a file and prints a message."""
     log.info('Deleting file: %s' % filepath)
     os.remove(filepath)
 
 
 def move_file(src, tgt):
     """Moves a file and prints a message."""
     log.info('Moving file %s to %s' % (src, tgt))
     if os.path.exists(tgt):
         raise Exception(
             'Cannot move file %s because target %s already exists' % \
                 (src, tgt))
     os.rename(src, tgt)
 
 
 def delete_directory(dirpath):
     """Delete a directory and print a message."""
     log.info('Deleting directory: %s' % dirpath)
     os.rmdir(dirpath)
 
 
 def migrate_repository(repos):
     """Does the actual migration to the new repository format."""
     log.info('Migrating repository at: %s to new format' % repos)
     versions = '%s/versions' % repos
     dirs = os.listdir(versions)
     # Only use int's in list.
     numdirs = [int(dirname) for dirname in dirs if dirname.isdigit()]
     numdirs.sort()  # Sort list.
     for dirname in numdirs:
         origdir = '%s/%s' % (versions, dirname)
         log.info('Working on directory: %s' % origdir)
         files = os.listdir(origdir)
         files.sort()
         for filename in files:
             # Delete compiled Python files.
             if filename.endswith('.pyc') or filename.endswith('.pyo'):
                 delete_file('%s/%s' % (origdir, filename))
 
             # Delete empty __init__.py files.
             origfile = '%s/__init__.py' % origdir
             if os.path.exists(origfile) and len(open(origfile).read()) == 0:
                 delete_file(origfile)
 
             # Move sql upgrade scripts.
             if filename.endswith('.sql'):
                 version, dbms, operation = filename.split('.', 3)[0:3]
                 origfile = '%s/%s' % (origdir, filename)
                 # For instance:  2.postgres.upgrade.sql ->
                 #  002_postgres_upgrade.sql
                 tgtfile = '%s/%03d_%s_%s.sql' % (
                     versions, int(version), dbms, operation)
                 move_file(origfile, tgtfile)
 
         # Move Python upgrade script.
         pyfile = '%s.py' % dirname
         pyfilepath = '%s/%s' % (origdir, pyfile)
         if os.path.exists(pyfilepath):
             tgtfile = '%s/%03d.py' % (versions, int(dirname))
             move_file(pyfilepath, tgtfile)
 
         # Try to remove directory. Will fail if it's not empty.
         delete_directory(origdir)
 
 
 def main():
     """Main function to be called when using this script."""
     if len(sys.argv) != 2:
         usage()
     migrate_repository(sys.argv[1])
 
 
 if __name__ == '__main__':
     main()

rhodecode/lib/dbmigrate/versions/005_version_1_3_0.py

@@ -1,79 +1,79 b''
 import logging
 import datetime
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper
 from sqlalchemy.orm.session import Session
 
 from rhodecode.lib.dbmigrate.migrate import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 
 from rhodecode.model.meta import Base
 
 log = logging.getLogger(__name__)
 
 
 def upgrade(migrate_engine):
     """ Upgrade operations go here.
     Don't create your own engine; bind migrate_engine to your metadata
     """
 
     #==========================================================================
     # Change unique constraints of table `repo_to_perm`
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_3_0 import UserRepoToPerm
     tbl = UserRepoToPerm().__table__
     new_cons = UniqueConstraint('user_id', 'repository_id', 'permission_id', table=tbl)
     new_cons.create()
     old_cons = None
     if migrate_engine.name in ['mysql']:
         old_cons = UniqueConstraint('user_id', 'repository_id', table=tbl, name="user_id")
     elif migrate_engine.name in ['postgresql']:
         old_cons = UniqueConstraint('user_id', 'repository_id', table=tbl)
     else:
         # sqlite doesn't support dropping constraints...
-        print """Please manually drop UniqueConstraint('user_id', 'repository_id')"""
+        print("""Please manually drop UniqueConstraint('user_id', 'repository_id')""")
 
     if old_cons:
         try:
             old_cons.drop()
         except Exception as e:
             # we don't care if this fails really... better to pass migration than
             # leave this in intermidiate state
-            print 'Failed to remove Unique for user_id, repository_id reason %s' % e
+            print('Failed to remove Unique for user_id, repository_id reason %s' % e)
 
 
     #==========================================================================
     # fix uniques of table `user_repo_group_to_perm`
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_3_0 import UserRepoGroupToPerm
     tbl = UserRepoGroupToPerm().__table__
     new_cons = UniqueConstraint('group_id', 'permission_id', 'user_id', table=tbl)
     new_cons.create()
     old_cons = None
 
     # fix uniqueConstraints
     if migrate_engine.name in ['mysql']:
         #mysql is givinig troubles here...
         old_cons = UniqueConstraint('group_id', 'permission_id', table=tbl, name="group_id")
     elif migrate_engine.name in ['postgresql']:
         old_cons = UniqueConstraint('group_id', 'permission_id', table=tbl, name='group_to_perm_group_id_permission_id_key')
     else:
         # sqlite doesn't support dropping constraints...
-        print """Please manually drop UniqueConstraint('group_id', 'permission_id')"""
+        print("""Please manually drop UniqueConstraint('group_id', 'permission_id')""")
 
     if old_cons:
         try:
             old_cons.drop()
         except Exception as e:
             # we don't care if this fails really... better to pass migration than
             # leave this in intermidiate state
-            print 'Failed to remove Unique for user_id, repository_id reason %s' % e
+            print('Failed to remove Unique for user_id, repository_id reason %s' % e)
 
     return
 
 
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine

rhodecode/lib/dbmigrate/versions/008_version_1_5_0.py

@@ -1,135 +1,135 b''
 import logging
 import datetime
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper, joinedload
 from sqlalchemy.orm.session import Session
 from sqlalchemy.ext.declarative import declarative_base
 
 from rhodecode.lib.dbmigrate.migrate import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 
 from rhodecode.model.meta import Base
 from rhodecode.model import meta
 from rhodecode.lib.dbmigrate.versions import _reset_base
 
 log = logging.getLogger(__name__)
 
 
 def upgrade(migrate_engine):
     """
     Upgrade operations go here.
     Don't create your own engine; bind migrate_engine to your metadata
     """
     _reset_base(migrate_engine)
     from rhodecode.lib.dbmigrate.schema import db_1_5_0
     #==========================================================================
     # USER LOGS
     #==========================================================================
 
     tbl = db_1_5_0.UserLog.__table__
     username = Column("username", String(255, convert_unicode=False),
                       nullable=True, unique=None, default=None)
     # create username column
     username.create(table=tbl)
 
     _Session = meta.Session()
     ## after adding that column fix all usernames
     users_log = _Session.query(db_1_5_0.UserLog)\
             .options(joinedload(db_1_5_0.UserLog.user))\
             .options(joinedload(db_1_5_0.UserLog.repository)).all()
 
     for entry in users_log:
         entry.username = entry.user.username
         _Session.add(entry)
     _Session.commit()
 
     #alter username to not null
     tbl_name = db_1_5_0.UserLog.__tablename__
     tbl = Table(tbl_name,
                 MetaData(bind=migrate_engine), autoload=True,
                 autoload_with=migrate_engine)
     col = tbl.columns.username
 
     # remove nullability from revision field
     col.alter(nullable=False)
 
     # issue fixups
     fixups(db_1_5_0, meta.Session)
 
 
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine
 
 
 def get_by_key(cls, key):
     return cls.query().filter(cls.permission_name == key).scalar()
 
 
 def get_by_name(cls, key):
     return cls.query().filter(cls.app_settings_name == key).scalar()
 
 
 def fixups(models, _SESSION):
     # ** create default permissions ** #
     #=====================================
     for p in models.Permission.PERMS:
         if not get_by_key(models.Permission, p[0]):
             new_perm = models.Permission()
             new_perm.permission_name = p[0]
             new_perm.permission_longname = p[0]  #translation err with p[1]
-            print 'Creating new permission %s' % p[0]
+            print('Creating new permission %s' % p[0])
             _SESSION().add(new_perm)
 
     _SESSION().commit()
 
     # ** populate default permissions ** #
     #=====================================
 
     user = models.User.query().filter(models.User.username == 'default').scalar()
 
     def _make_perm(perm):
         new_perm = models.UserToPerm()
         new_perm.user = user
         new_perm.permission = get_by_key(models.Permission, perm)
         return new_perm
 
     def _get_group(perm_name):
         return '.'.join(perm_name.split('.')[:1])
 
     perms = models.UserToPerm.query().filter(models.UserToPerm.user == user).all()
     defined_perms_groups = map(
         _get_group, (x.permission.permission_name for x in perms))
     log.debug('GOT ALREADY DEFINED:%s' % perms)
     DEFAULT_PERMS = models.Permission.DEFAULT_USER_PERMISSIONS
 
     # for every default permission that needs to be created, we check if
     # it's group is already defined, if it's not we create default perm
     for perm_name in DEFAULT_PERMS:
         gr = _get_group(perm_name)
         if gr not in defined_perms_groups:
             log.debug('GR:%s not found, creating permission %s'
                       % (gr, perm_name))
             new_perm = _make_perm(perm_name)
             _SESSION().add(new_perm)
     _SESSION().commit()
 
     # ** create default options ** #
     #===============================
     skip_existing = True
     for k, v in [
         ('default_repo_enable_locking',  False),
         ('default_repo_enable_downloads', False),
         ('default_repo_enable_statistics', False),
         ('default_repo_private', False),
         ('default_repo_type', 'hg')]:
 
         if skip_existing and get_by_name(models.RhodeCodeSetting, k) is not None:
             log.debug('Skipping option %s' % k)
             continue
         setting = models.RhodeCodeSetting(k, v)
         _SESSION().add(setting)
 
     _SESSION().commit()

rhodecode/lib/dbmigrate/versions/011_version_1_6_0.py

@@ -1,49 +1,49 b''
 import logging
 import datetime
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper, joinedload
 from sqlalchemy.orm.session import Session
 from sqlalchemy.ext.declarative import declarative_base
 
 from rhodecode.lib.dbmigrate.migrate import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 
 from rhodecode.model.meta import Base
 from rhodecode.model import meta
 from rhodecode.lib.dbmigrate.versions import _reset_base, notify
 
 log = logging.getLogger(__name__)
 
 
 def upgrade(migrate_engine):
     """
     Upgrade operations go here.
     Don't create your own engine; bind migrate_engine to your metadata
     """
     _reset_base(migrate_engine)
     from rhodecode.lib.dbmigrate.schema import db_1_6_0
 
     #==========================================================================
     # USER LOGS
     #==========================================================================
     tbl = db_1_6_0.RepositoryField.__table__
     tbl.create()
 
     # issue fixups
     fixups(db_1_6_0, meta.Session)
 
 
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine
 
 
 def fixups(models, _SESSION):
     notify('Upgrading repositories Caches')
     repositories = models.Repository.getAll()
     for repo in repositories:
-        print repo
+        print(repo)
         repo.update_commit_cache()
         _SESSION().commit()

rhodecode/lib/dbmigrate/versions/021_version_2_0_2.py

@@ -1,85 +1,85 b''
 import os
 import logging
 import datetime
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper, joinedload
 from sqlalchemy.orm.session import Session
 from sqlalchemy.ext.declarative import declarative_base
 
 from rhodecode.lib.dbmigrate.migrate import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 from rhodecode.lib.utils2 import str2bool
 
 from rhodecode.model.meta import Base
 from rhodecode.model import meta
 from rhodecode.lib.dbmigrate.versions import _reset_base, notify
 
 log = logging.getLogger(__name__)
 
 
 def get_by_key(cls, key):
     return cls.query().filter(cls.ui_key == key).scalar()
 
 
 def get_repos_location(cls):
     return get_by_key(cls, '/').ui_value
 
 
 def upgrade(migrate_engine):
     """
     Upgrade operations go here.
     Don't create your own engine; bind migrate_engine to your metadata
     """
     _reset_base(migrate_engine)
     from rhodecode.lib.dbmigrate.schema import db_2_0_1
     tbl = db_2_0_1.RepoGroup.__table__
 
     created_on = Column('created_on', DateTime(timezone=False), nullable=True,
                         default=datetime.datetime.now)
     created_on.create(table=tbl)
 
     #fix null values on certain columns when upgrading from older releases
     tbl = db_2_0_1.UserLog.__table__
     col = tbl.columns.user_id
     col.alter(nullable=True)
 
     tbl = db_2_0_1.UserFollowing.__table__
     col = tbl.columns.follows_repository_id
     col.alter(nullable=True)
 
     tbl = db_2_0_1.UserFollowing.__table__
     col = tbl.columns.follows_user_id
     col.alter(nullable=True)
 
     # issue fixups
     fixups(db_2_0_1, meta.Session)
 
 
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine
 
 
 def fixups(models, _SESSION):
     notify('Fixing default created on for repo groups')
 
     for gr in models.RepoGroup.get_all():
         gr.created_on = datetime.datetime.now()
         _SESSION().add(gr)
         _SESSION().commit()
 
     repo_store_path = get_repos_location(models.RhodeCodeUi)
     _store = os.path.join(repo_store_path, '.cache', 'largefiles')
     notify('Setting largefiles usercache')
-    print _store
+    print(_store)
 
     if not models.RhodeCodeUi.query().filter(
                     models.RhodeCodeUi.ui_key == 'usercache').scalar():
         largefiles = models.RhodeCodeUi()
         largefiles.ui_section = 'largefiles'
         largefiles.ui_key = 'usercache'
         largefiles.ui_value = _store
         _SESSION().add(largefiles)
         _SESSION().commit()

rhodecode/lib/dbmigrate/versions/022_version_2_0_2.py

@@ -1,70 +1,70 b''
 import logging
 import datetime
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper, joinedload
 from sqlalchemy.orm.session import Session
 from sqlalchemy.ext.declarative import declarative_base
 
 from rhodecode.lib.dbmigrate.migrate import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 from rhodecode.lib.utils2 import str2bool
 
 from rhodecode.model.meta import Base
 from rhodecode.model import meta
 from rhodecode.lib.dbmigrate.versions import _reset_base, notify
 
 log = logging.getLogger(__name__)
 
 
 def upgrade(migrate_engine):
     """
     Upgrade operations go here.
     Don't create your own engine; bind migrate_engine to your metadata
     """
     _reset_base(migrate_engine)
     from rhodecode.lib.dbmigrate.schema import db_2_0_2
 
     # issue fixups
     fixups(db_2_0_2, meta.Session)
 
 
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine
 
 
 def fixups(models, _SESSION):
     notify('fixing new schema for landing_rev')
 
     for repo in models.Repository.get_all():
-        print u'repo %s old landing rev is: %s' % (repo, repo.landing_rev)
+        print(u'repo %s old landing rev is: %s' % (repo, repo.landing_rev))
         _rev = repo.landing_rev[1]
         _rev_type = 'rev'  # default
 
         if _rev in ['default', 'master']:
             _rev_type = 'branch'
         elif _rev in ['tip']:
             _rev_type = 'rev'
         else:
             try:
                 scm = repo.scm_instance
                 if scm:
                     known_branches = scm.branches.keys()
                     known_bookmarks = scm.bookmarks.keys()
                     if _rev in known_branches:
                         _rev_type = 'branch'
                     elif _rev in known_bookmarks:
                         _rev_type = 'book'
             except Exception as e:
-                print e
-                print 'continue...'
+                print(e)
+                print('continue...')
                 #we don't want any error to break the process
                 pass
 
         _new_landing_rev = '%s:%s' % (_rev_type, _rev)
-        print u'setting to %s' % _new_landing_rev
+        print(u'setting to %s' % _new_landing_rev)
         repo.landing_rev = _new_landing_rev
         _SESSION().add(repo)
         _SESSION().commit()

rhodecode/lib/dbmigrate/versions/027_version_2_2_0.py

@@ -1,65 +1,65 b''
 import logging
 import datetime
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper, joinedload
 from sqlalchemy.orm.session import Session
 from sqlalchemy.ext.declarative import declarative_base
 
 from rhodecode.lib.dbmigrate.migrate import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 from rhodecode.lib.utils2 import str2bool
 
 from rhodecode.model.meta import Base
 from rhodecode.model import meta
 from rhodecode.lib.dbmigrate.versions import _reset_base, notify
 
 log = logging.getLogger(__name__)
 
 
 def get_by_key(cls, key):
     return cls.query().filter(cls.permission_name == key).scalar()
 
 
 def upgrade(migrate_engine):
     """
     Upgrade operations go here.
     Don't create your own engine; bind migrate_engine to your metadata
     """
     _reset_base(migrate_engine)
     from rhodecode.lib.dbmigrate.schema import db_2_2_0
 
     # issue fixups
     fixups(db_2_2_0, meta.Session)
 
 
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine
 
 
 def fixups(models, _SESSION):
     # ** create default permissions ** #
     #=====================================
     for p in models.Permission.PERMS:
         if not get_by_key(models.Permission, p[0]):
             new_perm = models.Permission()
             new_perm.permission_name = p[0]
             new_perm.permission_longname = p[0]  #translation err with p[1]
-            print 'Creating new permission %s' % p[0]
+            print('Creating new permission %s' % p[0])
             _SESSION().add(new_perm)
 
     _SESSION().commit()
 
     # ** set default create_on_write to active
     user = models.User.query().filter(
         models.User.username == 'default').scalar()
 
     _def = 'hg.create.write_on_repogroup.true'
     new = models.UserToPerm()
     new.user = user
     new.permission = get_by_key(models.Permission, _def)
-    print 'Setting default to %s' % _def
+    print('Setting default to %s' % _def)
     _SESSION().add(new)
     _SESSION().commit()

rhodecode/lib/dbmigrate/versions/036_version_2_3_0.py

@@ -1,44 +1,44 b''
 import logging
 import datetime
 
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper, joinedload
 from sqlalchemy.orm.session import Session
 from sqlalchemy.ext.declarative import declarative_base
 
 from rhodecode.lib.dbmigrate.migrate import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 from rhodecode.lib.utils2 import str2bool
 
 from rhodecode.model.meta import Base
 from rhodecode.model import meta
 from rhodecode.lib.dbmigrate.versions import _reset_base, notify
 
 log = logging.getLogger(__name__)
 
 
 def upgrade(migrate_engine):
     """
     Upgrade operations go here.
     Don't create your own engine; bind migrate_engine to your metadata
     """
     _reset_base(migrate_engine)
     from rhodecode.lib.dbmigrate.schema import db_2_3_0_1
 
     # issue fixups
     fixups(db_2_3_0_1, meta.Session)
 
 
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine
 
 
 def fixups(models, _SESSION):
     notify('Setting default renderer to rst')
     for cs_comment in models.ChangesetComment.get_all():
-        print 'comment_id %s renderer rst' % (cs_comment.comment_id)
+        print('comment_id %s renderer rst' % (cs_comment.comment_id))
         cs_comment.renderer = 'rst'
         _SESSION().add(cs_comment)
         _SESSION().commit()

rhodecode/lib/pidlock.py

@@ -1,149 +1,149 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import errno
 
 from multiprocessing.util import Finalize
 
 from rhodecode.lib.compat import kill
 
 
 class LockHeld(Exception):
     pass
 
 
 class DaemonLock(object):
     """daemon locking
     USAGE:
     try:
         l = DaemonLock(file_='/path/tolockfile',desc='test lock')
         main()
         l.release()
     except LockHeld:
         sys.exit(1)
     """
 
     def __init__(self, file_=None, callbackfn=None,
                  desc='daemon lock', debug=False):
 
         lock_name = os.path.join(os.path.dirname(__file__), 'running.lock')
         self.pidfile = file_ if file_ else lock_name
         self.callbackfn = callbackfn
         self.desc = desc
         self.debug = debug
         self.held = False
         #run the lock automatically !
         self.lock()
         self._finalize = Finalize(self, DaemonLock._on_finalize,
                                   args=(self, debug), exitpriority=10)
 
     @staticmethod
     def _on_finalize(lock, debug):
         if lock.held:
             if debug:
-                print 'leck held finilazing and running lock.release()'
+                print('leck held finilazing and running lock.release()')
             lock.release()
 
     def lock(self):
         """
         locking function, if lock is present it
         will raise LockHeld exception
         """
         lockname = '%s' % (os.getpid())
         if self.debug:
-            print 'running lock'
+            print('running lock')
         self.trylock()
         self.makelock(lockname, self.pidfile)
         return True
 
     def trylock(self):
         running_pid = False
         if self.debug:
-            print 'checking for already running process'
+            print('checking for already running process')
         try:
             with open(self.pidfile, 'r') as f:
                 try:
                     running_pid = int(f.readline())
                 except ValueError:
                     running_pid = -1
 
             if self.debug:
-                print ('lock file present running_pid: %s, '
-                       'checking for execution' % (running_pid,))
+                print('lock file present running_pid: %s, '
+                      'checking for execution' % (running_pid,))
             # Now we check the PID from lock file matches to the current
             # process PID
             if running_pid:
                 try:
                     kill(running_pid, 0)
                 except OSError as exc:
                     if exc.errno in (errno.ESRCH, errno.EPERM):
-                        print ("Lock File is there but"
-                               " the program is not running")
-                        print "Removing lock file for the: %s" % running_pid
+                        print("Lock File is there but"
+                              " the program is not running")
+                        print("Removing lock file for the: %s" % running_pid)
                         self.release()
                     else:
                         raise
                 else:
-                    print "You already have an instance of the program running"
-                    print "It is running as process %s" % running_pid
+                    print("You already have an instance of the program running")
+                    print("It is running as process %s" % running_pid)
                     raise LockHeld()
 
         except IOError as e:
             if e.errno != 2:
                 raise
 
     def release(self):
         """releases the pid by removing the pidfile
         """
         if self.debug:
-            print 'trying to release the pidlock'
+            print('trying to release the pidlock')
 
         if self.callbackfn:
-            #execute callback function on release
+            # execute callback function on release
             if self.debug:
-                print 'executing callback function %s' % self.callbackfn
+                print('executing callback function %s' % self.callbackfn)
             self.callbackfn()
         try:
             if self.debug:
-                print 'removing pidfile %s' % self.pidfile
+                print('removing pidfile %s' % self.pidfile)
             os.remove(self.pidfile)
             self.held = False
         except OSError as e:
             if self.debug:
-                print 'removing pidfile failed %s' % e
+                print('removing pidfile failed %s' % e)
             pass
 
     def makelock(self, lockname, pidfile):
         """
         this function will make an actual lock
 
         :param lockname: acctual pid of file
         :param pidfile: the file to write the pid in
         """
         if self.debug:
-            print 'creating a file %s and pid: %s' % (pidfile, lockname)
+            print('creating a file %s and pid: %s' % (pidfile, lockname))
 
         dir_, file_ = os.path.split(pidfile)
         if not os.path.isdir(dir_):
             os.makedirs(dir_)
         with open(self.pidfile, 'wb') as f:
             f.write(lockname)
         self.held = True

rhodecode/rcserver.py

@@ -1,1025 +1,1025 b''
 # (c) 2005 Ian Bicking and contributors; written for Paste
 # (http://pythonpaste.org) Licensed under the MIT license:
 # http://www.opensource.org/licenses/mit-license.php
 #
 # For discussion of daemonizing:
 # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/278731
 #
 # Code taken also from QP: http://www.mems-exchange.org/software/qp/ From
 # lib/site.py
 
 import atexit
 import errno
 import fnmatch
 import logging
 import optparse
 import os
 import re
 import subprocess32
 import sys
 import textwrap
 import threading
 import time
 import traceback
 
 from logging.config import fileConfig
 import ConfigParser as configparser
 from paste.deploy import loadserver
 from paste.deploy import loadapp
 
 import rhodecode
 from rhodecode.lib.compat import kill
 
 
 def make_web_build_callback(filename):
     p = subprocess32.Popen('make web-build', shell=True,
                          stdout=subprocess32.PIPE,
                          stderr=subprocess32.PIPE,
                          cwd=os.path.dirname(os.path.dirname(__file__)))
     stdout, stderr = p.communicate()
     stdout = ''.join(stdout)
     stderr = ''.join(stderr)
     if stdout:
-        print stdout
+        print(stdout)
     if stderr:
-        print ('%s %s %s' % ('-' * 20, 'ERRORS', '-' * 20))
-        print stderr
+        print('%s %s %s' % ('-' * 20, 'ERRORS', '-' * 20))
+        print(stderr)
 
 
 MAXFD = 1024
 HERE = os.path.dirname(os.path.abspath(__file__))
 SERVER_RUNNING_FILE = None
 
 
 # watch those extra files for changes, server gets restarted if file changes
 GLOBAL_EXTRA_FILES = {
     'rhodecode/public/css/*.less': make_web_build_callback,
     'rhodecode/public/js/src/**/*.js': make_web_build_callback,
 }
 
 
 
 ## HOOKS - inspired by gunicorn #
 
 def when_ready(server):
     """
     Called just after the server is started.
     """
 
     def _remove_server_running_file():
         if os.path.isfile(SERVER_RUNNING_FILE):
             os.remove(SERVER_RUNNING_FILE)
 
     if SERVER_RUNNING_FILE:
         with open(SERVER_RUNNING_FILE, 'wb') as f:
             f.write(str(os.getpid()))
         # register cleanup of that file when server exits
         atexit.register(_remove_server_running_file)
 
 
 def setup_logging(config_uri, fileConfig=fileConfig,
                   configparser=configparser):
     """
     Set up logging via the logging module's fileConfig function with the
     filename specified via ``config_uri`` (a string in the form
     ``filename#sectionname``).
 
     ConfigParser defaults are specified for the special ``__file__``
     and ``here`` variables, similar to PasteDeploy config loading.
     """
     path, _ = _getpathsec(config_uri, None)
     parser = configparser.ConfigParser()
     parser.read([path])
     if parser.has_section('loggers'):
         config_file = os.path.abspath(path)
         return fileConfig(
             config_file,
             {'__file__': config_file, 'here': os.path.dirname(config_file)}
             )
 
 
 def set_rhodecode_is_test(config_uri):
     """If is_test is defined in the config file sets rhodecode.is_test."""
     path, _ = _getpathsec(config_uri, None)
     parser = configparser.ConfigParser()
     parser.read(path)
     rhodecode.is_test = (
         parser.has_option('app:main', 'is_test') and
         parser.getboolean('app:main', 'is_test'))
 
 
 def _getpathsec(config_uri, name):
     if '#' in config_uri:
         path, section = config_uri.split('#', 1)
     else:
         path, section = config_uri, 'main'
     if name:
         section = name
     return path, section
 
 
 def parse_vars(args):
     """
     Given variables like ``['a=b', 'c=d']`` turns it into ``{'a':
     'b', 'c': 'd'}``
     """
     result = {}
     for arg in args:
         if '=' not in arg:
             raise ValueError(
                 'Variable assignment %r invalid (no "=")'
                 % arg)
         name, value = arg.split('=', 1)
         result[name] = value
     return result
 
 
 def _match_pattern(filename):
     for pattern in GLOBAL_EXTRA_FILES:
         if fnmatch.fnmatch(filename, pattern):
             return pattern
     return False
 
 
 def generate_extra_file_list():
 
     extra_list = []
     for root, dirs, files in os.walk(HERE, topdown=True):
         for fname in files:
             stripped_src = os.path.join(
                 'rhodecode', os.path.relpath(os.path.join(root, fname), HERE))
 
             if _match_pattern(stripped_src):
                 extra_list.append(stripped_src)
 
     return extra_list
 
 
 def run_callback_for_pattern(filename):
     pattern = _match_pattern(filename)
     if pattern:
         _file_callback = GLOBAL_EXTRA_FILES.get(pattern)
         if callable(_file_callback):
             _file_callback(filename)
 
 
 class DaemonizeException(Exception):
     pass
 
 
 class RcServerCommand(object):
 
     usage = '%prog config_uri [start|stop|restart|status] [var=value]'
     description = """\
     This command serves a web application that uses a PasteDeploy
     configuration file for the server and application.
 
     If start/stop/restart is given, then --daemon is implied, and it will
     start (normal operation), stop (--stop-daemon), or do both.
 
     You can also include variable assignments like 'http_port=8080'
     and then use %(http_port)s in your config files.
     """
     default_verbosity = 1
 
     parser = optparse.OptionParser(
         usage,
         description=textwrap.dedent(description)
         )
     parser.add_option(
         '-n', '--app-name',
         dest='app_name',
         metavar='NAME',
         help="Load the named application (default main)")
     parser.add_option(
         '-s', '--server',
         dest='server',
         metavar='SERVER_TYPE',
         help="Use the named server.")
     parser.add_option(
         '--server-name',
         dest='server_name',
         metavar='SECTION_NAME',
         help=("Use the named server as defined in the configuration file "
               "(default: main)"))
     parser.add_option(
         '--with-vcsserver',
         dest='vcs_server',
         action='store_true',
         help=("Start the vcsserver instance together with the RhodeCode server"))
     if hasattr(os, 'fork'):
         parser.add_option(
             '--daemon',
             dest="daemon",
             action="store_true",
             help="Run in daemon (background) mode")
     parser.add_option(
         '--pid-file',
         dest='pid_file',
         metavar='FILENAME',
         help=("Save PID to file (default to pyramid.pid if running in "
               "daemon mode)"))
     parser.add_option(
         '--running-file',
         dest='running_file',
         metavar='RUNNING_FILE',
         help="Create a running file after the server is initalized with "
              "stored PID of process")
     parser.add_option(
         '--log-file',
         dest='log_file',
         metavar='LOG_FILE',
         help="Save output to the given log file (redirects stdout)")
     parser.add_option(
         '--reload',
         dest='reload',
         action='store_true',
         help="Use auto-restart file monitor")
     parser.add_option(
         '--reload-interval',
         dest='reload_interval',
         default=1,
         help=("Seconds between checking files (low number can cause "
               "significant CPU usage)"))
     parser.add_option(
         '--monitor-restart',
         dest='monitor_restart',
         action='store_true',
         help="Auto-restart server if it dies")
     parser.add_option(
         '--status',
         action='store_true',
         dest='show_status',
         help="Show the status of the (presumably daemonized) server")
     parser.add_option(
         '-v', '--verbose',
         default=default_verbosity,
         dest='verbose',
         action='count',
         help="Set verbose level (default "+str(default_verbosity)+")")
     parser.add_option(
         '-q', '--quiet',
         action='store_const',
         const=0,
         dest='verbose',
         help="Suppress verbose output")
 
     if hasattr(os, 'setuid'):
         # I don't think these are available on Windows
         parser.add_option(
             '--user',
             dest='set_user',
             metavar="USERNAME",
             help="Set the user (usually only possible when run as root)")
         parser.add_option(
             '--group',
             dest='set_group',
             metavar="GROUP",
             help="Set the group (usually only possible when run as root)")
 
     parser.add_option(
         '--stop-daemon',
         dest='stop_daemon',
         action='store_true',
         help=('Stop a daemonized server (given a PID file, or default '
               'pyramid.pid file)'))
 
     _scheme_re = re.compile(r'^[a-z][a-z]+:', re.I)
 
     _reloader_environ_key = 'PYTHON_RELOADER_SHOULD_RUN'
     _monitor_environ_key = 'PASTE_MONITOR_SHOULD_RUN'
 
     possible_subcommands = ('start', 'stop', 'restart', 'status')
 
     def __init__(self, argv, quiet=False):
         self.options, self.args = self.parser.parse_args(argv[1:])
         if quiet:
             self.options.verbose = 0
 
     def out(self, msg): # pragma: no cover
         if self.options.verbose > 0:
             print(msg)
 
     def get_options(self):
         if (len(self.args) > 1
             and self.args[1] in self.possible_subcommands):
             restvars = self.args[2:]
         else:
             restvars = self.args[1:]
 
         return parse_vars(restvars)
 
     def run(self):  # pragma: no cover
         if self.options.stop_daemon:
             return self.stop_daemon()
 
         if not hasattr(self.options, 'set_user'):
             # Windows case:
             self.options.set_user = self.options.set_group = None
 
         # @@: Is this the right stage to set the user at?
         self.change_user_group(
             self.options.set_user, self.options.set_group)
 
         if not self.args:
             self.out('Please provide configuration file as first argument, '
                      'most likely it should be production.ini')
             return 2
         app_spec = self.args[0]
 
         if (len(self.args) > 1
             and self.args[1] in self.possible_subcommands):
             cmd = self.args[1]
         else:
             cmd = None
 
         if self.options.reload:
             if os.environ.get(self._reloader_environ_key):
                 if self.options.verbose > 1:
                     self.out('Running reloading file monitor')
 
                 install_reloader(int(self.options.reload_interval),
                                  [app_spec] + generate_extra_file_list())
                 # if self.requires_config_file:
                 #     watch_file(self.args[0])
             else:
                 return self.restart_with_reloader()
 
         if cmd not in (None, 'start', 'stop', 'restart', 'status'):
             self.out(
                 'Error: must give start|stop|restart (not %s)' % cmd)
             return 2
 
         if cmd == 'status' or self.options.show_status:
             return self.show_status()
 
         if cmd == 'restart' or cmd == 'stop':
             result = self.stop_daemon()
             if result:
                 if cmd == 'restart':
                     self.out("Could not stop daemon; aborting")
                 else:
                     self.out("Could not stop daemon")
                 return result
             if cmd == 'stop':
                 return result
             self.options.daemon = True
 
         if cmd == 'start':
             self.options.daemon = True
 
         app_name = self.options.app_name
 
         vars = self.get_options()
 
         if self.options.vcs_server:
             vars['vcs.start_server'] = 'true'
 
         if self.options.running_file:
             global SERVER_RUNNING_FILE
             SERVER_RUNNING_FILE = self.options.running_file
 
         if not self._scheme_re.search(app_spec):
             app_spec = 'config:' + app_spec
         server_name = self.options.server_name
         if self.options.server:
             server_spec = 'egg:pyramid'
             assert server_name is None
             server_name = self.options.server
         else:
             server_spec = app_spec
         base = os.getcwd()
 
         if getattr(self.options, 'daemon', False):
             if not self.options.pid_file:
                 self.options.pid_file = 'pyramid.pid'
             if not self.options.log_file:
                 self.options.log_file = 'pyramid.log'
 
         # Ensure the log file is writeable
         if self.options.log_file:
             try:
                 writeable_log_file = open(self.options.log_file, 'a')
             except IOError as ioe:
                 msg = 'Error: Unable to write to log file: %s' % ioe
                 raise ValueError(msg)
             writeable_log_file.close()
 
         # Ensure the pid file is writeable
         if self.options.pid_file:
             try:
                 writeable_pid_file = open(self.options.pid_file, 'a')
             except IOError as ioe:
                 msg = 'Error: Unable to write to pid file: %s' % ioe
                 raise ValueError(msg)
             writeable_pid_file.close()
 
 
         if getattr(self.options, 'daemon', False):
             try:
                 self.daemonize()
             except DaemonizeException as ex:
                 if self.options.verbose > 0:
                     self.out(str(ex))
                 return 2
 
         if (self.options.monitor_restart
             and not os.environ.get(self._monitor_environ_key)):
             return self.restart_with_monitor()
 
         if self.options.pid_file:
             self.record_pid(self.options.pid_file)
 
         if self.options.log_file:
             stdout_log = LazyWriter(self.options.log_file, 'a')
             sys.stdout = stdout_log
             sys.stderr = stdout_log
             logging.basicConfig(stream=stdout_log)
 
         log_fn = app_spec
         if log_fn.startswith('config:'):
             log_fn = app_spec[len('config:'):]
         elif log_fn.startswith('egg:'):
             log_fn = None
         if log_fn:
             log_fn = os.path.join(base, log_fn)
             setup_logging(log_fn)
             set_rhodecode_is_test(log_fn)
 
         server = self.loadserver(server_spec, name=server_name,
                                  relative_to=base, global_conf=vars)
         # starting hooks
         app = self.loadapp(app_spec, name=app_name, relative_to=base,
                            global_conf=vars)
 
         if self.options.verbose > 0:
             if hasattr(os, 'getpid'):
                 msg = 'Starting %s in PID %i.' % (__name__, os.getpid())
             else:
                 msg = 'Starting %s.' % (__name__,)
             self.out(msg)
             if SERVER_RUNNING_FILE:
                 self.out('PID file written as %s' % (SERVER_RUNNING_FILE, ))
             elif not self.options.pid_file:
                 self.out('No PID file written by default.')
 
         try:
             when_ready(server)
             server(app)
         except (SystemExit, KeyboardInterrupt) as e:
             if self.options.verbose > 1:
                 raise
             if str(e):
                 msg = ' ' + str(e)
             else:
                 msg = ''
             self.out('Exiting%s (-v to see traceback)' % msg)
 
 
     def loadapp(self, app_spec, name, relative_to, **kw):  # pragma: no cover
         return loadapp(app_spec, name=name, relative_to=relative_to, **kw)
 
     def loadserver(self, server_spec, name, relative_to, **kw):  # pragma:no cover
         return loadserver(
             server_spec, name=name, relative_to=relative_to, **kw)
 
     def quote_first_command_arg(self, arg): # pragma: no cover
         """
         There's a bug in Windows when running an executable that's
         located inside a path with a space in it.  This method handles
         that case, or on non-Windows systems or an executable with no
         spaces, it just leaves well enough alone.
         """
         if sys.platform != 'win32' or ' ' not in arg:
             # Problem does not apply:
             return arg
         try:
             import win32api
         except ImportError:
             raise ValueError(
                 "The executable %r contains a space, and in order to "
                 "handle this issue you must have the win32api module "
                 "installed" % arg)
         arg = win32api.GetShortPathName(arg)
         return arg
 
     def daemonize(self):  # pragma: no cover
         pid = live_pidfile(self.options.pid_file)
         if pid:
             raise DaemonizeException(
                 "Daemon is already running (PID: %s from PID file %s)"
                 % (pid, self.options.pid_file))
 
         if self.options.verbose > 0:
             self.out('Entering daemon mode')
         pid = os.fork()
         if pid:
             # The forked process also has a handle on resources, so we
             # *don't* want proper termination of the process, we just
             # want to exit quick (which os._exit() does)
             os._exit(0)
         # Make this the session leader
         os.setsid()
         # Fork again for good measure!
         pid = os.fork()
         if pid:
             os._exit(0)
 
         # @@: Should we set the umask and cwd now?
 
         import resource  # Resource usage information.
         maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
         if maxfd == resource.RLIM_INFINITY:
             maxfd = MAXFD
         # Iterate through and close all file descriptors.
         for fd in range(0, maxfd):
             try:
                 os.close(fd)
             except OSError:  # ERROR, fd wasn't open to begin with (ignored)
                 pass
 
         if hasattr(os, "devnull"):
             REDIRECT_TO = os.devnull
         else:
             REDIRECT_TO = "/dev/null"
         os.open(REDIRECT_TO, os.O_RDWR)  # standard input (0)
         # Duplicate standard input to standard output and standard error.
         os.dup2(0, 1)  # standard output (1)
         os.dup2(0, 2)  # standard error (2)
 
     def _remove_pid_file(self, written_pid, filename, verbosity):
         current_pid = os.getpid()
         if written_pid != current_pid:
             # A forked process must be exiting, not the process that
             # wrote the PID file
             return
         if not os.path.exists(filename):
             return
         with open(filename) as f:
             content = f.read().strip()
         try:
             pid_in_file = int(content)
         except ValueError:
             pass
         else:
             if pid_in_file != current_pid:
                 msg = "PID file %s contains %s, not expected PID %s"
                 self.out(msg % (filename, pid_in_file, current_pid))
                 return
         if verbosity > 0:
             self.out("Removing PID file %s" % filename)
         try:
             os.unlink(filename)
             return
         except OSError as e:
             # Record, but don't give traceback
             self.out("Cannot remove PID file: (%s)" % e)
         # well, at least lets not leave the invalid PID around...
         try:
             with open(filename, 'w') as f:
                 f.write('')
         except OSError as e:
             self.out('Stale PID left in file: %s (%s)' % (filename, e))
         else:
             self.out('Stale PID removed')
 
     def record_pid(self, pid_file):
         pid = os.getpid()
         if self.options.verbose > 1:
             self.out('Writing PID %s to %s' % (pid, pid_file))
         with open(pid_file, 'w') as f:
             f.write(str(pid))
         atexit.register(self._remove_pid_file, pid, pid_file, self.options.verbose)
 
     def stop_daemon(self):  # pragma: no cover
         pid_file = self.options.pid_file or 'pyramid.pid'
         if not os.path.exists(pid_file):
             self.out('No PID file exists in %s' % pid_file)
             return 1
         pid = read_pidfile(pid_file)
         if not pid:
             self.out("Not a valid PID file in %s" % pid_file)
             return 1
         pid = live_pidfile(pid_file)
         if not pid:
             self.out("PID in %s is not valid (deleting)" % pid_file)
             try:
                 os.unlink(pid_file)
             except (OSError, IOError) as e:
                 self.out("Could not delete: %s" % e)
                 return 2
             return 1
         for j in range(10):
             if not live_pidfile(pid_file):
                 break
             import signal
             kill(pid, signal.SIGTERM)
             time.sleep(1)
         else:
             self.out("failed to kill web process %s" % pid)
             return 3
         if os.path.exists(pid_file):
             os.unlink(pid_file)
         return 0
 
     def show_status(self):  # pragma: no cover
         pid_file = self.options.pid_file or 'pyramid.pid'
         if not os.path.exists(pid_file):
             self.out('No PID file %s' % pid_file)
             return 1
         pid = read_pidfile(pid_file)
         if not pid:
             self.out('No PID in file %s' % pid_file)
             return 1
         pid = live_pidfile(pid_file)
         if not pid:
             self.out('PID %s in %s is not running' % (pid, pid_file))
             return 1
         self.out('Server running in PID %s' % pid)
         return 0
 
     def restart_with_reloader(self):  # pragma: no cover
         self.restart_with_monitor(reloader=True)
 
     def restart_with_monitor(self, reloader=False):  # pragma: no cover
         if self.options.verbose > 0:
             if reloader:
                 self.out('Starting subprocess with file monitor')
             else:
                 self.out('Starting subprocess with monitor parent')
         while 1:
             args = [self.quote_first_command_arg(sys.executable)] + sys.argv
             new_environ = os.environ.copy()
             if reloader:
                 new_environ[self._reloader_environ_key] = 'true'
             else:
                 new_environ[self._monitor_environ_key] = 'true'
             proc = None
             try:
                 try:
                     _turn_sigterm_into_systemexit()
                     proc = subprocess32.Popen(args, env=new_environ)
                     exit_code = proc.wait()
                     proc = None
                 except KeyboardInterrupt:
                     self.out('^C caught in monitor process')
                     if self.options.verbose > 1:
                         raise
                     return 1
             finally:
                 if proc is not None:
                     import signal
                     try:
                         kill(proc.pid, signal.SIGTERM)
                     except (OSError, IOError):
                         pass
 
             if reloader:
                 # Reloader always exits with code 3; but if we are
                 # a monitor, any exit code will restart
                 if exit_code != 3:
                     return exit_code
             if self.options.verbose > 0:
                 self.out('%s %s %s' % ('-' * 20, 'Restarting', '-' * 20))
 
     def change_user_group(self, user, group):  # pragma: no cover
         if not user and not group:
             return
         import pwd
         import grp
         uid = gid = None
         if group:
             try:
                 gid = int(group)
                 group = grp.getgrgid(gid).gr_name
             except ValueError:
                 try:
                     entry = grp.getgrnam(group)
                 except KeyError:
                     raise ValueError(
                         "Bad group: %r; no such group exists" % group)
                 gid = entry.gr_gid
         try:
             uid = int(user)
             user = pwd.getpwuid(uid).pw_name
         except ValueError:
             try:
                 entry = pwd.getpwnam(user)
             except KeyError:
                 raise ValueError(
                     "Bad username: %r; no such user exists" % user)
             if not gid:
                 gid = entry.pw_gid
             uid = entry.pw_uid
         if self.options.verbose > 0:
             self.out('Changing user to %s:%s (%s:%s)' % (
                 user, group or '(unknown)', uid, gid))
         if gid:
             os.setgid(gid)
         if uid:
             os.setuid(uid)
 
 
 class LazyWriter(object):
 
     """
     File-like object that opens a file lazily when it is first written
     to.
     """
 
     def __init__(self, filename, mode='w'):
         self.filename = filename
         self.fileobj = None
         self.lock = threading.Lock()
         self.mode = mode
 
     def open(self):
         if self.fileobj is None:
             with self.lock:
                 self.fileobj = open(self.filename, self.mode)
         return self.fileobj
 
     def close(self):
         fileobj = self.fileobj
         if fileobj is not None:
             fileobj.close()
 
     def __del__(self):
         self.close()
 
     def write(self, text):
         fileobj = self.open()
         fileobj.write(text)
         fileobj.flush()
 
     def writelines(self, text):
         fileobj = self.open()
         fileobj.writelines(text)
         fileobj.flush()
 
     def flush(self):
         self.open().flush()
 
 
 def live_pidfile(pidfile):  # pragma: no cover
     """
     (pidfile:str) -> int | None
     Returns an int found in the named file, if there is one,
     and if there is a running process with that process id.
     Return None if no such process exists.
     """
     pid = read_pidfile(pidfile)
     if pid:
         try:
             kill(int(pid), 0)
             return pid
         except OSError as e:
             if e.errno == errno.EPERM:
                 return pid
     return None
 
 
 def read_pidfile(filename):
     if os.path.exists(filename):
         try:
             with open(filename) as f:
                 content = f.read()
             return int(content.strip())
         except (ValueError, IOError):
             return None
     else:
         return None
 
 
 def ensure_port_cleanup(
     bound_addresses, maxtries=30, sleeptime=2):  # pragma: no cover
     """
     This makes sure any open ports are closed.
 
     Does this by connecting to them until they give connection
     refused.  Servers should call like::
 
         ensure_port_cleanup([80, 443])
     """
     atexit.register(_cleanup_ports, bound_addresses, maxtries=maxtries,
                     sleeptime=sleeptime)
 
 
 def _cleanup_ports(
     bound_addresses, maxtries=30, sleeptime=2):  # pragma: no cover
     # Wait for the server to bind to the port.
     import socket
     import errno
     for bound_address in bound_addresses:
         for attempt in range(maxtries):
             sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             try:
                 sock.connect(bound_address)
             except socket.error as e:
                 if e.args[0] != errno.ECONNREFUSED:
                     raise
                 break
             else:
                 time.sleep(sleeptime)
         else:
             raise SystemExit('Timeout waiting for port.')
         sock.close()
 
 
 def _turn_sigterm_into_systemexit():  # pragma: no cover
     """
     Attempts to turn a SIGTERM exception into a SystemExit exception.
     """
     try:
         import signal
     except ImportError:
         return
     def handle_term(signo, frame):
         raise SystemExit
     signal.signal(signal.SIGTERM, handle_term)
 
 
 def install_reloader(poll_interval=1, extra_files=None):  # pragma: no cover
     """
     Install the reloading monitor.
 
     On some platforms server threads may not terminate when the main
     thread does, causing ports to remain open/locked.  The
     ``raise_keyboard_interrupt`` option creates a unignorable signal
     which causes the whole application to shut-down (rudely).
     """
     mon = Monitor(poll_interval=poll_interval)
     if extra_files is None:
         extra_files = []
     mon.extra_files.extend(extra_files)
     t = threading.Thread(target=mon.periodic_reload)
     t.setDaemon(True)
     t.start()
 
 
 class classinstancemethod(object):
     """
     Acts like a class method when called from a class, like an
     instance method when called by an instance.  The method should
     take two arguments, 'self' and 'cls'; one of these will be None
     depending on how the method was called.
     """
 
     def __init__(self, func):
         self.func = func
         self.__doc__ = func.__doc__
 
     def __get__(self, obj, type=None):
         return _methodwrapper(self.func, obj=obj, type=type)
 
 
 class _methodwrapper(object):
 
     def __init__(self, func, obj, type):
         self.func = func
         self.obj = obj
         self.type = type
 
     def __call__(self, *args, **kw):
         assert not 'self' in kw and not 'cls' in kw, (
             "You cannot use 'self' or 'cls' arguments to a "
             "classinstancemethod")
         return self.func(*((self.obj, self.type) + args), **kw)
 
 
 class Monitor(object): # pragma: no cover
     """
     A file monitor and server restarter.
 
     Use this like:
 
     ..code-block:: Python
 
         install_reloader()
 
     Then make sure your server is installed with a shell script like::
 
         err=3
         while test "$err" -eq 3 ; do
             python server.py
             err="$?"
         done
 
     or is run from this .bat file (if you use Windows)::
 
         @echo off
         :repeat
             python server.py
         if %errorlevel% == 3 goto repeat
 
     or run a monitoring process in Python (``pserve --reload`` does
     this).
 
     Use the ``watch_file(filename)`` function to cause a reload/restart for
     other non-Python files (e.g., configuration files).  If you have
     a dynamic set of files that grows over time you can use something like::
 
         def watch_config_files():
             return CONFIG_FILE_CACHE.keys()
         add_file_callback(watch_config_files)
 
     Then every time the reloader polls files it will call
     ``watch_config_files`` and check all the filenames it returns.
     """
     instances = []
     global_extra_files = []
     global_file_callbacks = []
 
     def __init__(self, poll_interval):
         self.module_mtimes = {}
         self.keep_running = True
         self.poll_interval = poll_interval
         self.extra_files = list(self.global_extra_files)
         self.instances.append(self)
         self.file_callbacks = list(self.global_file_callbacks)
 
     def _exit(self):
         # use os._exit() here and not sys.exit() since within a
         # thread sys.exit() just closes the given thread and
         # won't kill the process; note os._exit does not call
         # any atexit callbacks, nor does it do finally blocks,
         # flush open files, etc.  In otherwords, it is rude.
         os._exit(3)
 
     def periodic_reload(self):
         while True:
             if not self.check_reload():
                 self._exit()
                 break
             time.sleep(self.poll_interval)
 
     def check_reload(self):
         filenames = list(self.extra_files)
         for file_callback in self.file_callbacks:
             try:
                 filenames.extend(file_callback())
             except:
                 print(
                     "Error calling reloader callback %r:" % file_callback)
                 traceback.print_exc()
         for module in list(sys.modules.values()):
             try:
                 filename = module.__file__
             except (AttributeError, ImportError):
                 continue
             if filename is not None:
                 filenames.append(filename)
 
         for filename in filenames:
             try:
                 stat = os.stat(filename)
                 if stat:
                     mtime = stat.st_mtime
                 else:
                     mtime = 0
             except (OSError, IOError):
                 continue
             if filename.endswith('.pyc') and os.path.exists(filename[:-1]):
                 mtime = max(os.stat(filename[:-1]).st_mtime, mtime)
             if not filename in self.module_mtimes:
                 self.module_mtimes[filename] = mtime
             elif self.module_mtimes[filename] < mtime:
                 print("%s changed; reloading..." % filename)
                 run_callback_for_pattern(filename)
                 return False
         return True
 
     def watch_file(self, cls, filename):
         """Watch the named file for changes"""
         filename = os.path.abspath(filename)
         if self is None:
             for instance in cls.instances:
                 instance.watch_file(filename)
             cls.global_extra_files.append(filename)
         else:
             self.extra_files.append(filename)
 
     watch_file = classinstancemethod(watch_file)
 
     def add_file_callback(self, cls, callback):
         """Add a callback -- a function that takes no parameters -- that will
         return a list of filenames to watch for changes."""
         if self is None:
             for instance in cls.instances:
                 instance.add_file_callback(callback)
             cls.global_file_callbacks.append(callback)
         else:
             self.file_callbacks.append(callback)
 
     add_file_callback = classinstancemethod(add_file_callback)
 
 watch_file = Monitor.watch_file
 add_file_callback = Monitor.add_file_callback
 
 
 def main(argv=sys.argv, quiet=False):
     command = RcServerCommand(argv, quiet=quiet)
     return command.run()
 
 if __name__ == '__main__':  # pragma: no cover
     sys.exit(main() or 0)

rhodecode/tests/fixture.py

@@ -1,349 +1,349 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Helpers for fixture generation
 """
 
 import os
 import time
 import tempfile
 import shutil
 
 import configobj
 
 from rhodecode.tests import *
 from rhodecode.model.db import Repository, User, RepoGroup, UserGroup, Gist, UserEmailMap
 from rhodecode.model.meta import Session
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.gist import GistModel
 from rhodecode.model.auth_token import AuthTokenModel
 
 dn = os.path.dirname
 FIXTURES = os.path.join(dn(dn(os.path.abspath(__file__))), 'tests', 'fixtures')
 
 
 def error_function(*args, **kwargs):
     raise Exception('Total Crash !')
 
 
 class TestINI(object):
     """
     Allows to create a new test.ini file as a copy of existing one with edited
     data. Example usage::
 
         with TestINI('test.ini', [{'section':{'key':val'}]) as new_test_ini_path:
-            print 'paster server %s' % new_test_ini
+            print('paster server %s' % new_test_ini)
     """
 
     def __init__(self, ini_file_path, ini_params, new_file_prefix='DEFAULT',
                  destroy=True, dir=None):
         self.ini_file_path = ini_file_path
         self.ini_params = ini_params
         self.new_path = None
         self.new_path_prefix = new_file_prefix
         self._destroy = destroy
         self._dir = dir
 
     def __enter__(self):
         return self.create()
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.destroy()
 
     def create(self):
         config = configobj.ConfigObj(
             self.ini_file_path, file_error=True, write_empty_values=True)
 
         for data in self.ini_params:
             section, ini_params = data.items()[0]
             for key, val in ini_params.items():
                 config[section][key] = val
         with tempfile.NamedTemporaryFile(
                 prefix=self.new_path_prefix, suffix='.ini', dir=self._dir,
                 delete=False) as new_ini_file:
             config.write(new_ini_file)
             self.new_path = new_ini_file.name
 
         return self.new_path
 
     def destroy(self):
         if self._destroy:
             os.remove(self.new_path)
 
 
 class Fixture(object):
 
     def anon_access(self, status):
         """
         Context process for disabling anonymous access. use like:
         fixture = Fixture()
         with fixture.anon_access(False):
             #tests
 
         after this block anon access will be set to `not status`
         """
 
         class context(object):
             def __enter__(self):
                 anon = User.get_default_user()
                 anon.active = status
                 Session().add(anon)
                 Session().commit()
                 time.sleep(1.5)  # must sleep for cache (1s to expire)
 
             def __exit__(self, exc_type, exc_val, exc_tb):
                 anon = User.get_default_user()
                 anon.active = not status
                 Session().add(anon)
                 Session().commit()
 
         return context()
 
     def _get_repo_create_params(self, **custom):
         defs = {
             'repo_name': None,
             'repo_type': 'hg',
             'clone_uri': '',
             'push_uri': '',
             'repo_group': '-1',
             'repo_description': 'DESC',
             'repo_private': False,
             'repo_landing_rev': 'rev:tip',
             'repo_copy_permissions': False,
             'repo_state': Repository.STATE_CREATED,
         }
         defs.update(custom)
         if 'repo_name_full' not in custom:
             defs.update({'repo_name_full': defs['repo_name']})
 
         # fix the repo name if passed as repo_name_full
         if defs['repo_name']:
             defs['repo_name'] = defs['repo_name'].split('/')[-1]
 
         return defs
 
     def _get_group_create_params(self, **custom):
         defs = {
             'group_name': None,
             'group_description': 'DESC',
             'perm_updates': [],
             'perm_additions': [],
             'perm_deletions': [],
             'group_parent_id': -1,
             'enable_locking': False,
             'recursive': False,
         }
         defs.update(custom)
 
         return defs
 
     def _get_user_create_params(self, name, **custom):
         defs = {
             'username': name,
             'password': 'qweqwe',
             'email': '[e-mail]' % name,
             'firstname': 'TestUser',
             'lastname': 'Test',
             'active': True,
             'admin': False,
             'extern_type': 'rhodecode',
             'extern_name': None,
         }
         defs.update(custom)
 
         return defs
 
     def _get_user_group_create_params(self, name, **custom):
         defs = {
             'users_group_name': name,
             'user_group_description': 'DESC',
             'users_group_active': True,
             'user_group_data': {},
         }
         defs.update(custom)
 
         return defs
 
     def create_repo(self, name, **kwargs):
         repo_group = kwargs.get('repo_group')
         if isinstance(repo_group, RepoGroup):
             kwargs['repo_group'] = repo_group.group_id
             name = name.split(Repository.NAME_SEP)[-1]
             name = Repository.NAME_SEP.join((repo_group.group_name, name))
 
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             r = Repository.get_by_repo_name(name)
             if r:
                 return r
 
         form_data = self._get_repo_create_params(repo_name=name, **kwargs)
         cur_user = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         RepoModel().create(form_data, cur_user)
         Session().commit()
         repo = Repository.get_by_repo_name(name)
         assert repo
         return repo
 
     def create_fork(self, repo_to_fork, fork_name, **kwargs):
         repo_to_fork = Repository.get_by_repo_name(repo_to_fork)
 
         form_data = self._get_repo_create_params(repo_name=fork_name,
                                             fork_parent_id=repo_to_fork.repo_id,
                                             repo_type=repo_to_fork.repo_type,
                                             **kwargs)
         #TODO: fix it !!
         form_data['description'] = form_data['repo_description']
         form_data['private'] = form_data['repo_private']
         form_data['landing_rev'] = form_data['repo_landing_rev']
 
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         RepoModel().create_fork(form_data, cur_user=owner)
         Session().commit()
         r = Repository.get_by_repo_name(fork_name)
         assert r
         return r
 
     def destroy_repo(self, repo_name, **kwargs):
         RepoModel().delete(repo_name, **kwargs)
         Session().commit()
 
     def destroy_repo_on_filesystem(self, repo_name):
         rm_path = os.path.join(RepoModel().repos_path, repo_name)
         if os.path.isdir(rm_path):
             shutil.rmtree(rm_path)
 
     def create_repo_group(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             gr = RepoGroup.get_by_group_name(group_name=name)
             if gr:
                 return gr
         form_data = self._get_group_create_params(group_name=name, **kwargs)
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         gr = RepoGroupModel().create(
             group_name=form_data['group_name'],
             group_description=form_data['group_name'],
             owner=owner)
         Session().commit()
         gr = RepoGroup.get_by_group_name(gr.group_name)
         return gr
 
     def destroy_repo_group(self, repogroupid):
         RepoGroupModel().delete(repogroupid)
         Session().commit()
 
     def create_user(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             user = User.get_by_username(name)
             if user:
                 return user
         form_data = self._get_user_create_params(name, **kwargs)
         user = UserModel().create(form_data)
 
         # create token for user
         AuthTokenModel().create(
                 user=user, description=u'TEST_USER_TOKEN')
 
         Session().commit()
         user = User.get_by_username(user.username)
         return user
 
     def destroy_user(self, userid):
         UserModel().delete(userid)
         Session().commit()
 
     def create_additional_user_email(self, user, email):
         uem = UserEmailMap()
         uem.user = user
         uem.email = email
         Session().add(uem)
         return uem
 
     def destroy_users(self, userid_iter):
         for user_id in userid_iter:
             if User.get_by_username(user_id):
                 UserModel().delete(user_id)
                 Session().commit()
 
     def create_user_group(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             gr = UserGroup.get_by_group_name(group_name=name)
             if gr:
                 return gr
         # map active flag to the real attribute. For API consistency of fixtures
         if 'active' in kwargs:
             kwargs['users_group_active'] = kwargs['active']
             del kwargs['active']
         form_data = self._get_user_group_create_params(name, **kwargs)
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         user_group = UserGroupModel().create(
             name=form_data['users_group_name'],
             description=form_data['user_group_description'],
             owner=owner, active=form_data['users_group_active'],
             group_data=form_data['user_group_data'])
         Session().commit()
         user_group = UserGroup.get_by_group_name(user_group.users_group_name)
         return user_group
 
     def destroy_user_group(self, usergroupid):
         UserGroupModel().delete(user_group=usergroupid, force=True)
         Session().commit()
 
     def create_gist(self, **kwargs):
         form_data = {
             'description': 'new-gist',
             'owner': TEST_USER_ADMIN_LOGIN,
             'gist_type': GistModel.cls.GIST_PUBLIC,
             'lifetime': -1,
             'acl_level': Gist.ACL_LEVEL_PUBLIC,
             'gist_mapping': {'filename1.txt': {'content': 'hello world'},}
         }
         form_data.update(kwargs)
         gist = GistModel().create(
             description=form_data['description'], owner=form_data['owner'],
             gist_mapping=form_data['gist_mapping'], gist_type=form_data['gist_type'],
             lifetime=form_data['lifetime'], gist_acl_level=form_data['acl_level']
         )
         Session().commit()
         return gist
 
     def destroy_gists(self, gistid=None):
         for g in GistModel.cls.get_all():
             if gistid:
                 if gistid == g.gist_access_id:
                     GistModel().delete(g)
             else:
                 GistModel().delete(g)
         Session().commit()
 
     def load_resource(self, resource_name, strip=False):
         with open(os.path.join(FIXTURES, resource_name)) as f:
             source = f.read()
             if strip:
                 source = source.strip()
 
         return source

rhodecode/tests/lib/middleware/test_simplesvn.py

@@ -1,201 +1,201 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from StringIO import StringIO
 
 import pytest
 from mock import patch, Mock
 
 from rhodecode.lib.middleware.simplesvn import SimpleSvn, SimpleSvnApp
 from rhodecode.lib.utils import get_rhodecode_base_path
 
 
 class TestSimpleSvn(object):
     @pytest.fixture(autouse=True)
     def simple_svn(self, baseapp, request_stub):
         base_path = get_rhodecode_base_path()
         self.app = SimpleSvn(
             config={'auth_ret_code': '', 'base_path': base_path},
             registry=request_stub.registry)
 
     def test_get_config(self):
         extras = {'foo': 'FOO', 'bar': 'BAR'}
         config = self.app._create_config(extras, repo_name='test-repo')
         assert config == extras
 
     @pytest.mark.parametrize(
         'method', ['OPTIONS', 'PROPFIND', 'GET', 'REPORT'])
     def test_get_action_returns_pull(self, method):
         environment = {'REQUEST_METHOD': method}
         action = self.app._get_action(environment)
         assert action == 'pull'
 
     @pytest.mark.parametrize(
         'method', [
             'MKACTIVITY', 'PROPPATCH', 'PUT', 'CHECKOUT', 'MKCOL', 'MOVE',
             'COPY', 'DELETE', 'LOCK', 'UNLOCK', 'MERGE'
         ])
     def test_get_action_returns_push(self, method):
         environment = {'REQUEST_METHOD': method}
         action = self.app._get_action(environment)
         assert action == 'push'
 
     @pytest.mark.parametrize(
         'path, expected_name', [
             ('/hello-svn', 'hello-svn'),
             ('/hello-svn/', 'hello-svn'),
             ('/group/hello-svn/', 'group/hello-svn'),
             ('/group/hello-svn/!svn/vcc/default', 'group/hello-svn'),
         ])
     def test_get_repository_name(self, path, expected_name):
         environment = {'PATH_INFO': path}
         name = self.app._get_repository_name(environment)
         assert name == expected_name
 
     def test_get_repository_name_subfolder(self, backend_svn):
         repo = backend_svn.repo
         environment = {
             'PATH_INFO': '/{}/path/with/subfolders'.format(repo.repo_name)}
         name = self.app._get_repository_name(environment)
         assert name == repo.repo_name
 
     def test_create_wsgi_app(self):
         with patch.object(SimpleSvn, '_is_svn_enabled') as mock_method:
             mock_method.return_value = False
             with patch('rhodecode.lib.middleware.simplesvn.DisabledSimpleSvnApp') as (
                     wsgi_app_mock):
                 config = Mock()
                 wsgi_app = self.app._create_wsgi_app(
                     repo_path='', repo_name='', config=config)
 
             wsgi_app_mock.assert_called_once_with(config)
             assert wsgi_app == wsgi_app_mock()
 
     def test_create_wsgi_app_when_enabled(self):
         with patch.object(SimpleSvn, '_is_svn_enabled') as mock_method:
             mock_method.return_value = True
             with patch('rhodecode.lib.middleware.simplesvn.SimpleSvnApp') as (
                     wsgi_app_mock):
                 config = Mock()
                 wsgi_app = self.app._create_wsgi_app(
                     repo_path='', repo_name='', config=config)
 
             wsgi_app_mock.assert_called_once_with(config)
             assert wsgi_app == wsgi_app_mock()
 
 
 class TestSimpleSvnApp(object):
     data = '<xml></xml>'
     path = '/group/my-repo'
     wsgi_input = StringIO(data)
     environment = {
         'HTTP_DAV': (
             'http://subversion.tigris.org/xmlns/dav/svn/depth,'
             ' http://subversion.tigris.org/xmlns/dav/svn/mergeinfo'),
         'HTTP_USER_AGENT': 'SVN/1.8.11 (x86_64-linux) serf/1.3.8',
         'REQUEST_METHOD': 'OPTIONS',
         'PATH_INFO': path,
         'wsgi.input': wsgi_input,
         'CONTENT_TYPE': 'text/xml',
         'CONTENT_LENGTH': '130'
     }
 
     def setup_method(self, method):
         self.host = 'http://localhost/'
         base_path = get_rhodecode_base_path()
         self.app = SimpleSvnApp(
             config={'subversion_http_server_url': self.host,
                     'base_path': base_path})
 
     def test_get_request_headers_with_content_type(self):
         expected_headers = {
             'Dav': self.environment['HTTP_DAV'],
             'User-Agent': self.environment['HTTP_USER_AGENT'],
             'Content-Type': self.environment['CONTENT_TYPE'],
             'Content-Length': self.environment['CONTENT_LENGTH']
         }
         headers = self.app._get_request_headers(self.environment)
         assert headers == expected_headers
 
     def test_get_request_headers_without_content_type(self):
         environment = self.environment.copy()
         environment.pop('CONTENT_TYPE')
         expected_headers = {
             'Dav': environment['HTTP_DAV'],
             'Content-Length': self.environment['CONTENT_LENGTH'],
             'User-Agent': environment['HTTP_USER_AGENT'],
         }
         request_headers = self.app._get_request_headers(environment)
         assert request_headers == expected_headers
 
     def test_get_response_headers(self):
         headers = {
             'Connection': 'keep-alive',
             'Keep-Alive': 'timeout=5, max=100',
             'Transfer-Encoding': 'chunked',
             'Content-Encoding': 'gzip',
             'MS-Author-Via': 'DAV',
             'SVN-Supported-Posts': 'create-txn-with-props'
         }
         expected_headers = [
             ('MS-Author-Via', 'DAV'),
             ('SVN-Supported-Posts', 'create-txn-with-props'),
         ]
         response_headers = self.app._get_response_headers(headers)
         assert sorted(response_headers) == sorted(expected_headers)
 
     def test_get_url(self):
         url = self.app._get_url(self.path)
         expected_url = '{}{}'.format(self.host.strip('/'), self.path)
         assert url == expected_url
 
     def test_call(self):
         start_response = Mock()
         response_mock = Mock()
         response_mock.headers = {
             'Content-Encoding': 'gzip',
             'MS-Author-Via': 'DAV',
             'SVN-Supported-Posts': 'create-txn-with-props'
         }
         response_mock.status_code = 200
         response_mock.reason = 'OK'
         with patch('rhodecode.lib.middleware.simplesvn.requests.request') as (
                 request_mock):
             request_mock.return_value = response_mock
             self.app(self.environment, start_response)
 
         expected_url = '{}{}'.format(self.host.strip('/'), self.path)
         expected_request_headers = {
             'Dav': self.environment['HTTP_DAV'],
             'User-Agent': self.environment['HTTP_USER_AGENT'],
             'Content-Type': self.environment['CONTENT_TYPE'],
             'Content-Length': self.environment['CONTENT_LENGTH']
         }
         expected_response_headers = [
             ('SVN-Supported-Posts', 'create-txn-with-props'),
             ('MS-Author-Via', 'DAV'),
         ]
         request_mock.assert_called_once_with(
             self.environment['REQUEST_METHOD'], expected_url,
-            data=self.data, headers=expected_request_headers)
+            data=self.data, headers=expected_request_headers, stream=False)
         response_mock.iter_content.assert_called_once_with(chunk_size=1024)
         args, _ = start_response.call_args
         assert args[0] == '200 OK'
         assert sorted(args[1]) == sorted(expected_response_headers)

rhodecode/tests/lib/test_codeblocks.py

@@ -1,311 +1,312 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 from pygments.lexers import get_lexer_by_name
 
 from rhodecode.tests import no_newline_id_generator
 from rhodecode.lib.codeblocks import (
     tokenize_string, split_token_stream, rollup_tokenstream,
     render_tokenstream)
 
 
 class TestTokenizeString(object):
 
     python_code = '''
     import this
 
     var = 6
-    print "this"
+    print("this")
 
     '''
 
     def test_tokenize_as_python(self):
         lexer = get_lexer_by_name('python')
         tokens = list(tokenize_string(self.python_code, lexer))
 
         assert tokens == [
             ('',    u'\n'),
             ('',    u'    '),
             ('kn',  u'import'),
             ('',    u' '),
             ('nn',  u'this'),
             ('',    u'\n'),
             ('',    u'\n'),
             ('',    u'    '),
             ('n',   u'var'),
             ('',    u' '),
             ('o',   u'='),
             ('',    u' '),
             ('mi',  u'6'),
             ('',    u'\n'),
             ('',    u'    '),
             ('k',   u'print'),
-            ('',    u' '),
-            ('s2',  u'"'),
-            ('s2',  u'this'),
-            ('s2',  u'"'),
+            ('p', u'('),
+            ('s2', u'"'),
+            ('s2', u'this'),
+            ('s2', u'"'),
+            ('p', u')'),
             ('',    u'\n'),
             ('',    u'\n'),
             ('',    u'    ')
         ]
 
     def test_tokenize_as_text(self):
         lexer = get_lexer_by_name('text')
         tokens = list(tokenize_string(self.python_code, lexer))
 
         assert tokens == [
             ('',
-            u'\n    import this\n\n    var = 6\n    print "this"\n\n    ')
+            u'\n    import this\n\n    var = 6\n    print("this")\n\n    ')
         ]
 
 
 class TestSplitTokenStream(object):
 
     def test_split_token_stream(self):
         lines = list(split_token_stream(
             [('type1', 'some\ntext'), ('type2', 'more\n')]))
 
         assert lines == [
             [('type1', u'some')],
             [('type1', u'text'), ('type2', u'more')],
             [('type2', u'')],
         ]
 
     def test_split_token_stream_single(self):
         lines = list(split_token_stream(
             [('type1', '\n')]))
 
         assert lines == [
             [('type1', '')],
             [('type1', '')],
         ]
 
     def test_split_token_stream_single_repeat(self):
         lines = list(split_token_stream(
             [('type1', '\n\n\n')]))
 
         assert lines == [
             [('type1', '')],
             [('type1', '')],
             [('type1', '')],
             [('type1', '')],
         ]
 
     def test_split_token_stream_multiple_repeat(self):
         lines = list(split_token_stream(
             [('type1', '\n\n'), ('type2', '\n\n')]))
 
         assert lines == [
             [('type1', '')],
             [('type1', '')],
             [('type1', ''), ('type2', '')],
             [('type2', '')],
             [('type2', '')],
         ]
 
 
 class TestRollupTokens(object):
 
     @pytest.mark.parametrize('tokenstream,output', [
         ([],
             []),
         ([('A', 'hell'), ('A', 'o')], [
             ('A', [
                 ('', 'hello')]),
         ]),
         ([('A', 'hell'), ('B', 'o')], [
             ('A', [
                 ('', 'hell')]),
             ('B', [
                 ('', 'o')]),
         ]),
         ([('A', 'hel'), ('A', 'lo'), ('B', ' '), ('A', 'there')], [
             ('A', [
                 ('', 'hello')]),
             ('B', [
                 ('', ' ')]),
             ('A', [
                 ('', 'there')]),
         ]),
     ])
     def test_rollup_tokenstream_without_ops(self, tokenstream, output):
         assert list(rollup_tokenstream(tokenstream)) == output
 
     @pytest.mark.parametrize('tokenstream,output', [
         ([],
             []),
         ([('A', '', 'hell'), ('A', '', 'o')], [
             ('A', [
                 ('', 'hello')]),
         ]),
         ([('A', '', 'hell'), ('B', '', 'o')], [
             ('A', [
                 ('', 'hell')]),
             ('B', [
                 ('', 'o')]),
         ]),
         ([('A', '', 'h'), ('B', '', 'e'), ('C', '', 'y')], [
             ('A', [
                 ('', 'h')]),
             ('B', [
                 ('', 'e')]),
             ('C', [
                 ('', 'y')]),
         ]),
         ([('A', '', 'h'), ('A', '', 'e'), ('C', '', 'y')], [
             ('A', [
                 ('', 'he')]),
             ('C', [
                 ('', 'y')]),
         ]),
         ([('A', 'ins', 'h'), ('A', 'ins', 'e')], [
             ('A', [
                 ('ins', 'he')
             ]),
         ]),
         ([('A', 'ins', 'h'), ('A', 'del', 'e')], [
             ('A', [
                 ('ins', 'h'),
                 ('del', 'e')
             ]),
         ]),
         ([('A', 'ins', 'h'), ('B', 'del', 'e'), ('B', 'del', 'y')], [
             ('A', [
                 ('ins', 'h'),
             ]),
             ('B', [
                 ('del', 'ey'),
             ]),
         ]),
         ([('A', 'ins', 'h'), ('A', 'del', 'e'), ('B', 'del', 'y')], [
             ('A', [
                 ('ins', 'h'),
                 ('del', 'e'),
             ]),
             ('B', [
                 ('del', 'y'),
             ]),
         ]),
         ([('A', '', 'some'), ('A', 'ins', 'new'), ('A', '', 'name')], [
             ('A', [
                 ('', 'some'),
                 ('ins', 'new'),
                 ('', 'name'),
             ]),
         ]),
     ])
     def test_rollup_tokenstream_with_ops(self, tokenstream, output):
         assert list(rollup_tokenstream(tokenstream)) == output
 
 
 class TestRenderTokenStream(object):
 
     @pytest.mark.parametrize('tokenstream,output', [
         (
             [],
             '',
         ),
         (
             [('', '', u'')],
             '<span></span>',
         ),
         (
             [('', '', u'text')],
             '<span>text</span>',
         ),
         (
             [('A', '', u'')],
             '<span class="A"></span>',
         ),
         (
             [('A', '', u'hello')],
             '<span class="A">hello</span>',
         ),
         (
             [('A', '', u'hel'), ('A', '', u'lo')],
             '<span class="A">hello</span>',
         ),
         (
             [('A', '', u'two\n'), ('A', '', u'lines')],
             '<span class="A">two\nlines</span>',
         ),
         (
             [('A', '', u'\nthree\n'), ('A', '', u'lines')],
             '<span class="A">\nthree\nlines</span>',
         ),
         (
             [('', '', u'\n'), ('A', '', u'line')],
             '<span>\n</span><span class="A">line</span>',
         ),
         (
             [('', 'ins', u'\n'), ('A', '', u'line')],
             '<span><ins>\n</ins></span><span class="A">line</span>',
         ),
         (
             [('A', '', u'hel'), ('A', 'ins', u'lo')],
             '<span class="A">hel<ins>lo</ins></span>',
         ),
         (
             [('A', '', u'hel'), ('A', 'ins', u'l'), ('A', 'ins', u'o')],
             '<span class="A">hel<ins>lo</ins></span>',
         ),
         (
             [('A', '', u'hel'), ('A', 'ins', u'l'), ('A', 'del', u'o')],
             '<span class="A">hel<ins>l</ins><del>o</del></span>',
         ),
         (
             [('A', '', u'hel'), ('B', '', u'lo')],
             '<span class="A">hel</span><span class="B">lo</span>',
         ),
         (
             [('A', '', u'hel'), ('B', 'ins', u'lo')],
             '<span class="A">hel</span><span class="B"><ins>lo</ins></span>',
         ),
     ], ids=no_newline_id_generator)
     def test_render_tokenstream_with_ops(self, tokenstream, output):
         html = render_tokenstream(tokenstream)
         assert html == output
 
     @pytest.mark.parametrize('tokenstream,output', [
         (
             [('A', u'hel'), ('A', u'lo')],
             '<span class="A">hello</span>',
         ),
         (
             [('A', u'hel'), ('A', u'l'), ('A', u'o')],
             '<span class="A">hello</span>',
         ),
         (
             [('A', u'hel'), ('A', u'l'), ('A', u'o')],
             '<span class="A">hello</span>',
         ),
         (
             [('A', u'hel'), ('B', u'lo')],
             '<span class="A">hel</span><span class="B">lo</span>',
         ),
         (
             [('A', u'hel'), ('B', u'lo')],
             '<span class="A">hel</span><span class="B">lo</span>',
         ),
     ])
     def test_render_tokenstream_without_ops(self, tokenstream, output):
         html = render_tokenstream(tokenstream)
         assert html == output

rhodecode/tests/load/http_performance.py

@@ -1,463 +1,463 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Module to test the performance of pull, push and clone operations.
 
 It works by replaying a group of commits to the repo.
 """
 
 import argparse
 import collections
 import ConfigParser
 import functools
 import itertools
 import os
 import pprint
 import shutil
 import subprocess32
 import sys
 import time
 
 import api
 
 
 def mean(container):
     """Return the mean of the container."""
     if not container:
         return -1.0
     return sum(container) / len(container)
 
 
 def keep_cwd(f):
     """Decorator that keeps track of the starting working directory."""
     @functools.wraps(f)
     def wrapped_f(*args, **kwargs):
         cur_dir = os.getcwd()
         try:
             return f(*args, **kwargs)
         finally:
             os.chdir(cur_dir)
 
     return wrapped_f
 
 
 def timed(f):
     """Decorator that returns the time it took to execute the function."""
     @functools.wraps(f)
     def wrapped_f(*args, **kwargs):
         start_time = time.time()
         try:
             f(*args, **kwargs)
         finally:
             return time.time() - start_time
 
     return wrapped_f
 
 
 def execute(*popenargs, **kwargs):
     """Extension of subprocess.check_output to support writing to stdin."""
     input = kwargs.pop('stdin', None)
     stdin = None
     if input:
         stdin = subprocess32.PIPE
     #if 'stderr' not in kwargs:
     #    kwargs['stderr'] = subprocess32.PIPE
     if 'stdout' in kwargs:
         raise ValueError('stdout argument not allowed, it will be overridden.')
     process = subprocess32.Popen(stdin=stdin, stdout=subprocess32.PIPE,
                                *popenargs, **kwargs)
     output, error = process.communicate(input=input)
     retcode = process.poll()
     if retcode:
         cmd = kwargs.get("args")
         if cmd is None:
             cmd = popenargs[0]
-        print cmd, output, error
+        print('{} {} {} '.format(cmd, output, error))
         raise subprocess32.CalledProcessError(retcode, cmd, output=output)
     return output
 
 
 def get_repo_name(repo_url):
     """Extract the repo name from its url."""
     repo_url = repo_url.rstrip('/')
     return repo_url.split('/')[-1].split('.')[0]
 
 
 class TestPerformanceBase(object):
     def __init__(self, base_dir, repo_url, n_commits, max_commits,
                  skip_commits):
         self.repo_url = repo_url
         self.repo_name = get_repo_name(self.repo_url)
         self.upstream_repo_name = '%s_upstream' % self.repo_name
         self.base_dir = os.path.abspath(base_dir)
         self.n_commits = n_commits
         self.max_commits = max_commits
         self.skip_commits = skip_commits
         self.push_times = []
         self.pull_times = []
         self.empty_pull_times = []
         self.clone_time = -1.0
         self.last_commit = None
 
         self.cloned_repo = ''
         self.pull_repo = ''
         self.orig_repo = ''
 
     def run(self):
         try:
             self.test()
         except Exception as error:
-            print error
+            print(error)
         finally:
             self.cleanup()
 
-            print 'Clone time     :', self.clone_time
-            print 'Push time      :', mean(self.push_times)
-            print 'Pull time      :', mean(self.pull_times)
-            print 'Empty pull time:', mean(self.empty_pull_times)
+            print('Clone time     :{}'.format(self.clone_time))
+            print('Push time      :{}'.format(mean(self.push_times)))
+            print('Pull time      :{}'.format(mean(self.pull_times)))
+            print('Empty pull time:{}'.format(mean(self.empty_pull_times)))
 
             return {
                 'clone': self.clone_time,
                 'push': mean(self.push_times),
                 'pull': mean(self.pull_times),
                 'empty_pull': mean(self.empty_pull_times),
             }
 
     @keep_cwd
     def test(self):
         os.chdir(self.base_dir)
 
         self.orig_repo = os.path.join(self.base_dir, self.repo_name)
         if not os.path.exists(self.orig_repo):
             self.clone_repo(self.repo_url, default_only=True)
 
         upstream_url = self.create_repo(self.upstream_repo_name, self.repo_type)
 
         self.add_remote(self.orig_repo, upstream_url)
 
         self.pull_repo = os.path.join(self.base_dir, '%s_pull' % self.repo_name)
         self.clone_repo(upstream_url, self.pull_repo)
 
         commits = self.get_commits(self.orig_repo)
         self.last_commit = commits[-1]
         if self.skip_commits:
             self.push(
                 self.orig_repo, commits[self.skip_commits - 1], 'upstream')
         commits = commits[self.skip_commits:self.max_commits]
 
-        print 'Working with %d commits' % len(commits)
-        for i in xrange(self.n_commits - 1, len(commits), self.n_commits):
+        print('Working with %d commits' % len(commits))
+        for i in range(self.n_commits - 1, len(commits), self.n_commits):
             commit = commits[i]
-            print 'Processing commit %s (%d)' % (commit, i + 1)
+            print('Processing commit %s (%d)' % (commit, i + 1))
             self.push_times.append(
                 self.push(self.orig_repo, commit, 'upstream'))
             self.check_remote_last_commit_is(commit, upstream_url)
 
             self.pull_times.append(self.pull(self.pull_repo))
             self.check_local_last_commit_is(commit, self.pull_repo)
 
             self.empty_pull_times.append(self.pull(self.pull_repo))
 
         self.cloned_repo = os.path.join(self.base_dir,
                                         '%s_clone' % self.repo_name)
         self.clone_time = self.clone_repo(upstream_url, self.cloned_repo)
 
     def cleanup(self):
         try:
             self.delete_repo(self.upstream_repo_name)
         except api.ApiError:
             # Continue in case we could not delete the repo. Maybe we did not
             # create it in the first place.
             pass
 
         shutil.rmtree(self.pull_repo, ignore_errors=True)
         shutil.rmtree(self.cloned_repo, ignore_errors=True)
 
         if os.path.exists(self.orig_repo):
             self.remove_remote(self.orig_repo)
 
 
 class RhodeCodeMixin(object):
     """Mixin providing the methods to create and delete repos in RhodeCode."""
     def __init__(self, api_key):
         self.api = api.RCApi(api_key=api_key)
 
     def create_repo(self, repo_name, repo_type):
         return self.api.create_repo(repo_name, repo_type,
                                     'Repo for perfomance testing')
 
     def delete_repo(self, repo_name):
         return self.api.delete_repo(repo_name)
 
 
 class GitMixin(object):
     """Mixin providing the git operations."""
     @timed
     def clone_repo(self, repo_url, destination=None, default_only=False):
         args = ['git', 'clone']
         if default_only:
             args.extend(['--branch', 'master', '--single-branch'])
         args.append(repo_url)
         if destination:
             args.append(destination)
         execute(args)
 
     @keep_cwd
     def add_remote(self, repo, remote_url, remote_name='upstream'):
         self.remove_remote(repo, remote_name)
         os.chdir(repo)
         execute(['git', 'remote', 'add', remote_name, remote_url])
 
     @keep_cwd
     def remove_remote(self, repo, remote_name='upstream'):
         os.chdir(repo)
         remotes = execute(['git', 'remote']).split('\n')
         if remote_name in remotes:
             execute(['git', 'remote', 'remove', remote_name])
 
     @keep_cwd
     def get_commits(self, repo, branch='master'):
         os.chdir(repo)
         commits_list = execute(
             ['git', 'log', '--first-parent', branch, '--pretty=%H'])
         return commits_list.strip().split('\n')[::-1]
 
     @timed
     def push(self, repo, commit, remote_name=None):
         os.chdir(repo)
         try:
             execute(['git', 'reset', '--soft', commit])
             args = ['git', 'push']
             if remote_name:
                 args.append(remote_name)
             execute(args)
         finally:
             execute(['git', 'reset', '--soft', 'HEAD@{1}'])
 
     @timed
     def pull(self, repo):
         os.chdir(repo)
         execute(['git', 'pull'])
 
     def _remote_last_commit(self, repo_url):
         output = execute(['git', 'ls-remote', repo_url, 'HEAD'])
         return output.split()[0]
 
     def check_remote_last_commit_is(self, commit, repo_url):
         last_remote_commit = self._remote_last_commit(repo_url)
         if last_remote_commit != commit:
             raise Exception('Push did not work, expected commit %s but got %s' %
                             (commit, last_remote_commit))
 
     @keep_cwd
     def _local_last_commit(self, repo):
         os.chdir(repo)
         return execute(['git', 'rev-parse', 'HEAD']).strip()
 
     def check_local_last_commit_is(self, commit, repo):
         last_local_commit = self._local_last_commit(repo)
         if last_local_commit != commit:
             raise Exception('Pull did not work, expected commit %s but got %s' %
                             (commit, last_local_commit))
 
 
 class HgMixin(object):
     """Mixin providing the mercurial operations."""
     @timed
     def clone_repo(self, repo_url, destination=None, default_only=False):
         args = ['hg', 'clone']
         if default_only:
             args.extend(['--branch', 'default'])
         args.append(repo_url)
         if destination:
             args.append(destination)
         execute(args)
 
     @keep_cwd
     def add_remote(self, repo, remote_url, remote_name='upstream'):
         self.remove_remote(repo, remote_name)
         os.chdir(repo)
         hgrc = ConfigParser.RawConfigParser()
         hgrc.read('.hg/hgrc')
         hgrc.set('paths', remote_name, remote_url)
         with open('.hg/hgrc', 'w') as f:
             hgrc.write(f)
 
     @keep_cwd
     def remove_remote(self, repo, remote_name='upstream'):
         os.chdir(repo)
         hgrc = ConfigParser.RawConfigParser()
         hgrc.read('.hg/hgrc')
         hgrc.remove_option('paths', remote_name)
         with open('.hg/hgrc', 'w') as f:
             hgrc.write(f)
 
     @keep_cwd
     def get_commits(self, repo, branch='default'):
         os.chdir(repo)
         # See http://stackoverflow.com/questions/15376649/is-there-a-mercurial-equivalent-to-git-log-first-parent
         commits_list = execute(['hg', 'log', '--branch', branch, '--template',
                                 '{node}\n', '--follow-first'])
         return commits_list.strip().split('\n')[::-1]
 
     @timed
     def push(self, repo, commit, remote_name=None):
         os.chdir(repo)
         args = ['hg', 'push', '--rev', commit, '--new-branch']
         if remote_name:
             args.append(remote_name)
         execute(args)
 
     @timed
     def pull(self, repo):
         os.chdir(repo)
         execute(['hg', '--config', 'alias.pull=pull', 'pull', '-u'])
 
     def _remote_last_commit(self, repo_url):
         return execute(['hg', 'identify', repo_url])[:12]
 
     def check_remote_last_commit_is(self, commit, repo_url):
         last_remote_commit = self._remote_last_commit(repo_url)
         if not commit.startswith(last_remote_commit):
             raise Exception('Push did not work, expected commit %s but got %s' %
                             (commit, last_remote_commit))
 
     @keep_cwd
     def _local_last_commit(self, repo):
         os.chdir(repo)
         return execute(['hg', 'identify'])[:12]
 
     def check_local_last_commit_is(self, commit, repo):
         last_local_commit = self._local_last_commit(repo)
         if not commit.startswith(last_local_commit):
             raise Exception('Pull did not work, expected commit %s but got %s' %
                             (commit, last_local_commit))
 
 
 class GitTestPerformance(GitMixin, RhodeCodeMixin, TestPerformanceBase):
     def __init__(self, base_dir, repo_url, n_commits, max_commits, skip_commits,
                  api_key):
         TestPerformanceBase.__init__(self, base_dir, repo_url, n_commits,
                                      max_commits, skip_commits)
         RhodeCodeMixin.__init__(self, api_key)
         self.repo_type = 'git'
 
 
 class HgTestPerformance(HgMixin, RhodeCodeMixin, TestPerformanceBase):
     def __init__(self, base_dir, repo_url, n_commits, max_commits, skip_commits,
                  api_key):
         TestPerformanceBase.__init__(self, base_dir, repo_url, n_commits,
                                      max_commits, skip_commits)
         RhodeCodeMixin.__init__(self, api_key)
         self.repo_type = 'hg'
 
 
 def get_test(base_dir, repo_url, repo_type, step, max_commits, skip_commits,
              api_key):
     max_commits = min(10 * step,
                       int((max_commits - skip_commits) / step) * step)
     max_commits += skip_commits
     if repo_type == 'git':
         return GitTestPerformance(
             base_dir, repo_url, step, max_commits, skip_commits, api_key)
     elif repo_type == 'hg':
         return HgTestPerformance(
             base_dir, repo_url, step, max_commits, skip_commits, api_key)
 
 
 def main(argv):
     parser = argparse.ArgumentParser(
         description='Performance tests for push/pull/clone for git and ' +
                     'mercurial repos.')
     parser.add_argument(
         '--tests', dest='tests', action='store', required=False, default='all',
         help='The tests to run. Default: all. But could be any comma ' +
              'separated list with python, hg, kernel or git')
     parser.add_argument(
         '--sizes', dest='sizes', action='store', required=False,
         default='1,10,100,1000,2500',
         help='The sizes to use. Default: 1,10,100,1000,2500')
     parser.add_argument(
         '--dir', dest='dir', action='store', required=True,
         help='The dir where to store the repos')
     parser.add_argument(
         '--api-key', dest='api_key', action='store', required=True,
         help='The api key of RhodeCode')
     options = parser.parse_args(argv[1:])
-    print options
+    print(options)
 
     test_config = {
         'python': {
             'url': 'https://hg.python.org/cpython/',
             'limit': 23322,
             'type': 'hg',
             # Do not time the first commit, as it is HUGE!
             'skip': 1,
         },
         'hg': {
             'url': 'http://selenic.com/hg',
             'limit': 14396,
             'type': 'hg',
         },
         'kernel': {
             'url': 'https://github.com/torvalds/linux.git',
             'limit': 46271,
             'type': 'git',
         },
         'git': {
             'url': 'https://github.com/git/git.git',
             'limit': 13525,
             'type': 'git',
         }
 
     }
 
     test_names = options.tests.split(',')
     if test_names == ['all']:
         test_names = test_config.keys()
     if not set(test_names) <= set(test_config.keys()):
-        print ('Invalid tests: only %s are valid but specified %s' %
-               (test_config.keys(), test_names))
+        print('Invalid tests: only %s are valid but specified %s' %
+              (test_config.keys(), test_names))
         return 1
 
     sizes = options.sizes.split(',')
     sizes = map(int, sizes)
 
     base_dir = options.dir
     api_key = options.api_key
     results = collections.defaultdict(dict)
     for test_name, size in itertools.product(test_names, sizes):
         test = get_test(base_dir,
                         test_config[test_name]['url'],
                         test_config[test_name]['type'],
                         size,
                         test_config[test_name]['limit'],
                         test_config[test_name].get('skip', 0),
                         api_key)
-        print '*' * 80
-        print 'Running performance test: %s with size %d' % (test_name, size)
-        print '*' * 80
+        print('*' * 80)
+        print('Running performance test: %s with size %d' % (test_name, size))
+        print('*' * 80)
         results[test_name][size] = test.run()
     pprint.pprint(dict(results))
 
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv))

rhodecode/tests/load/profile-mem.py

@@ -1,135 +1,135 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Utility to gather certain statistics about a process.
 
 Used to generate data about the memory consumption of the vcsserver. It is
 quite generic and should work for every process. Use the parameter `--help`
 to see all options.
 
 Example call::
 
   python profile-mem.py --pid=89816 --ae --ae-key=YOUR_API_KEY
 
 """
 
 
 import argparse
 import json
 import sys
 import time
 
 import datetime
 import requests
 import psutil
 
 import logging
 import socket
 logging.basicConfig(level=logging.DEBUG)
 
 
 def profile():
     config = parse_options()
     try:
         process = psutil.Process(config.pid)
     except psutil.NoSuchProcess:
-        print "Process {pid} does not exist!".format(pid=config.pid)
+        print("Process {pid} does not exist!".format(pid=config.pid))
         sys.exit(1)
 
     while True:
         stats = process_stats(process)
         dump_stats(stats)
         if config.appenlight:
             client = AppenlightClient(
                 url=config.appenlight_url,
                 api_key=config.appenlight_api_key)
             client.dump_stats(stats)
         time.sleep(config.interval)
 
 
 def parse_options():
     parser = argparse.ArgumentParser(
         description=__doc__)
     parser.add_argument(
         '--pid', required=True, type=int,
         help="Process ID to monitor.")
     parser.add_argument(
         '--interval', '-i', type=float, default=5,
         help="Interval in secods.")
     parser.add_argument(
         '--appenlight', '--ae', action='store_true')
     parser.add_argument(
         '--appenlight-url', '--ae-url',
         default='https://ae.rhodecode.com/api/logs',
         help='URL of the Appenlight API endpoint, defaults to "%(default)s".')
     parser.add_argument(
         '--appenlight-api-key', '--ae-key',
         help='API key to use when sending data to appenlight. This has to be '
              'set if Appenlight is enabled.')
     return parser.parse_args()
 
 
 def process_stats(process):
     mem = process.memory_info()
     iso_now = datetime.datetime.utcnow().isoformat()
     stats = [
         {'message': 'Memory stats of process {pid}'.format(pid=process.pid),
          'namespace': 'process.{pid}'.format(pid=process.pid),
          'server': socket.getfqdn(socket.gethostname()),
          'tags': [
             ['rss', mem.rss],
             ['vms', mem.vms]],
          'date': iso_now,
          },
     ]
     return stats
 
 
 def dump_stats(stats):
     for sample in stats:
-        print json.dumps(sample)
+        print(json.dumps(sample))
 
 
 class AppenlightClient():
 
     url_template = '{url}?protocol_version=0.5'
 
     def __init__(self, url, api_key):
         self.url = self.url_template.format(url=url)
         self.api_key = api_key
 
     def dump_stats(self, stats):
         response = requests.post(
             self.url,
             headers={
                 'X-appenlight-api-key': self.api_key},
             data=json.dumps(stats))
         if not response.status_code == 200:
             logging.error(
                 'Sending to appenlight failed\n%s\n%s',
                 response.headers, response.text)
 
 
 if __name__ == '__main__':
     try:
         profile()
     except KeyboardInterrupt:
         pass

rhodecode/tests/load/profile.py

@@ -1,155 +1,155 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 This is a standalone script which will start VCS and RC.
 
 Performance numbers will be written on each interval to:
     vcs_profileX.csv
     rc_profileX.csv
 
 To stop the script by press Ctrl-C
 """
 
 import datetime
 import os
 import psutil
 import subprocess32
 import sys
 import time
 import traceback
 import urllib
 
 PROFILING_INTERVAL = 5
 RC_WEBSITE = "http://localhost:5001/"
 
 
 def get_file(prefix):
     out_file = None
-    for i in xrange(100):
+    for i in range(100):
         file_path = "%s_profile%.3d.csv" % (prefix, i)
         if os.path.exists(file_path):
             continue
         out_file = open(file_path, "w")
         out_file.write("Time; CPU %; Memory (MB); Total FDs; Dulwich FDs; Threads\n")
         break
     return out_file
 
 
 def dump_system():
-    print "System Overview..."
-    print "\nCPU Count: %d (%d real)" % \
-          (psutil.cpu_count(), psutil.cpu_count(logical=False))
-    print "\nDisk:"
-    print psutil.disk_usage(os.sep)
-    print "\nMemory:"
-    print psutil.virtual_memory()
-    print "\nMemory (swap):"
-    print psutil.swap_memory()
+    print("System Overview...")
+    print("\nCPU Count: %d (%d real)" %
+          (psutil.cpu_count(), psutil.cpu_count(logical=False)))
+    print("\nDisk:")
+    print(psutil.disk_usage(os.sep))
+    print("\nMemory:")
+    print(psutil.virtual_memory())
+    print("\nMemory (swap):")
+    print(psutil.swap_memory())
 
 
 def count_dulwich_fds(proc):
     p = subprocess32.Popen(["lsof", "-p", proc.pid], stdout=subprocess32.PIPE)
     out, err = p.communicate()
 
     count = 0
     for line in out.splitlines():
         content = line.split()
         # http://git-scm.com/book/en/Git-Internals-Packfiles
         if content[-1].endswith(".idx"):
             count += 1
 
     return count
 
 def dump_process(pid, out_file):
     now = datetime.datetime.now()
     cpu = pid.cpu_percent()
     mem = pid.memory_info()
     fds = pid.num_fds()
     dulwich_fds = count_dulwich_fds(pid)
     threads = pid.num_threads()
 
     content = [now.strftime('%m/%d/%y %H:%M:%S'),
                cpu,
                "%.2f" % (mem[0]/1024.0/1024.0),
                fds, dulwich_fds, threads]
     out_file.write("; ".join([str(item) for item in content]))
     out_file.write("\n")
 
 
 # Open output files
 vcs_out = get_file("vcs")
 if vcs_out is None:
-    print "Unable to enumerate output file for VCS"
+    print("Unable to enumerate output file for VCS")
     sys.exit(1)
 rc_out = get_file("rc")
 if rc_out is None:
-    print "Unable to enumerate output file for RC"
+    print("Unable to enumerate output file for RC")
     sys.exit(1)
 
 # Show system information
 dump_system()
 
-print "\nStarting VCS..."
+print("\nStarting VCS...")
 vcs = psutil.Popen(["vcsserver"])
 time.sleep(1)
 if not vcs.is_running():
-    print "VCS - Failed to start"
+    print("VCS - Failed to start")
     sys.exit(1)
-print "VCS - Ok"
+print("VCS - Ok")
 
-print "\nStarting RhodeCode..."
+print("\nStarting RhodeCode...")
 rc = psutil.Popen("RC_VCSSERVER_TEST_DISABLE=1 paster serve test.ini",
                   shell=True, stdin=subprocess32.PIPE)
 time.sleep(1)
 if not rc.is_running():
-    print "RC - Failed to start"
+    print("RC - Failed to start")
     vcs.terminate()
     sys.exit(1)
 
 # Send command to create the databases
 rc.stdin.write("y\n")
 
 # Verify that the website is up
 time.sleep(4)
 try:
     urllib.urlopen(RC_WEBSITE)
 except IOError:
-    print "RC - Website not started"
+    print("RC - Website not started")
     vcs.terminate()
     sys.exit(1)
-print "RC - Ok"
+print("RC - Ok")
 
-print "\nProfiling...\n%s\n" % ("-"*80)
+print("\nProfiling...\n%s\n" % ("-"*80))
 while True:
     try:
         dump_process(vcs, vcs_out)
         dump_process(rc, rc_out)
         time.sleep(PROFILING_INTERVAL)
     except Exception:
-        print traceback.format_exc()
+        print(traceback.format_exc())
         break
 
 # Finalize the profiling
 vcs_out.close()
 rc_out.close()
 
 vcs.terminate()
 rc.terminate()

rhodecode/tests/load/time_urls.py

@@ -1,69 +1,69 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import timeit
 
 server = "localhost:5000"
 
 pages = [
     "cpython",
     "cpython/annotate/74236c8bf064188516b32bf95016971227ec72a9/Makefile.pre.in",
     "cpython/changelog",
     "cpython/changeset/e0f681f4ade3af52915d5f32daac97ada580d71a",
     "cpython/compare/tag@v3.4.1rc1...tag@v3.4.1?target_repo=cpython",
     "cpython/files/tip/",
     "cpython/files/74236c8bf064188516b32bf95016971227ec72a9/Grammar",
     "",
     "git",
     "git/annotate/6c4ab27f2378ce67940b4496365043119d7ffff2/gitk-git/.gitignore",
     "git/changelog",
     "git/changeset/d299e9e550c1bf8640907fdba1f03cc585ee71df",
     "git/compare/rev@1200...rev@1300?target_repo=git",
     "git/files/tip/",
     "git/files/6c4ab27f2378ce67940b4496365043119d7ffff2/.gitignore"
 ]
 
 svn_pages = [
     "svn-apache",
     "svn-apache/annotate/672129/cocoon/trunk/README.txt",
     "svn-apache/changelog",
     "svn-apache/changeset/1164362",
     "svn-apache/compare/rev@1164350...rev@1164360?target_repo=svn-apache",
     "svn-apache/compare/rev@1164300...rev@1164360?target_repo=svn-apache",
     "svn-apache/files/tip/",
     "svn-apache/files/1164363/cocoon/trunk/README.txt",
 ]
 
 # Uncomment to check also svn performance
 # pages = pages + svn_pages
 
 repeat = 10
 
-print "Repeating each URL x%d\n" % repeat
+print("Repeating each URL x%d\n" % repeat)
 for page in pages:
     url = "http://%s/%s" % (server, page)
-    print url
+    print(url)
 
     stmt = "urllib2.urlopen('%s', timeout=120)" % url
     t = timeit.Timer(stmt=stmt, setup="import urllib2")
 
     result = t.repeat(repeat=repeat, number=1)
-    print "\t%.3f (min) - %.3f (max) - %.3f (avg)\n" % \
-          (min(result), max(result), sum(result)/len(result))
+    print("\t%.3f (min) - %.3f (max) - %.3f (avg)\n" %
+          (min(result), max(result), sum(result)/len(result)))

rhodecode/tests/scripts/test_crawler.py

@@ -1,187 +1,187 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Test for crawling a project for memory usage
 This should be runned just as regular script together
 with a watch script that will show memory usage.
 
 watch -n1 ./rhodecode/tests/mem_watch
 """
 
 
 import cookielib
 import urllib
 import urllib2
 import time
 import os
 import sys
 from os.path import join as jn
 from os.path import dirname as dn
 
 from sqlalchemy.util import OrderedSet
 
 __here__ = os.path.abspath(__file__)
 __root__ = dn(dn(dn(__here__)))
 sys.path.append(__root__)
 
 from rhodecode.lib import vcs
 from rhodecode.lib.vcs.exceptions import RepositoryError
 
 PASES = 3
 HOST = 'http://127.0.0.1'
 PORT = 5001
 BASE_URI = '%s:%s/' % (HOST, PORT)
 
 if len(sys.argv) == 2:
     BASE_URI = sys.argv[1]
 
 if not BASE_URI.endswith('/'):
     BASE_URI += '/'
 
-print 'Crawling @ %s' % BASE_URI
+print('Crawling @ %s' % BASE_URI)
 BASE_URI += '%s'
 PROJECT_PATH = jn('/', 'home', 'marcink', 'repos')
 PROJECTS = [
     #'linux-magx-pbranch',
     'CPython',
     'rhodecode_tip',
 ]
 
 
 cj = cookielib.FileCookieJar('/tmp/rc_test_cookie.txt')
 o = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
 o.addheaders = [
     ('User-agent', 'rhodecode-crawler'),
     ('Accept-Language', 'en - us, en;q = 0.5')
 ]
 
 urllib2.install_opener(o)
 
 
 def _get_repo(proj):
     if isinstance(proj, basestring):
         repo = vcs.get_repo(jn(PROJECT_PATH, proj))
         proj = proj
     else:
         repo = proj
         proj = repo.name
 
     return repo, proj
 
 
 def test_changelog_walk(proj, pages=100):
     repo, proj = _get_repo(proj)
 
     total_time = 0
     for i in range(1, pages):
 
         page = '/'.join((proj, 'changelog',))
 
         full_uri = (BASE_URI % page) + '?' + urllib.urlencode({'page': i})
         s = time.time()
         f = o.open(full_uri)
 
         assert f.url == full_uri, 'URL:%s does not match %s' % (f.url, full_uri)
 
         size = len(f.read())
         e = time.time() - s
         total_time += e
-        print 'visited %s size:%s req:%s ms' % (full_uri, size, e)
+        print('visited %s size:%s req:%s ms' % (full_uri, size, e))
 
-    print 'total_time', total_time
-    print 'average on req', total_time / float(pages)
+    print('total_time {}'.format(total_time))
+    print('average on req {}'.format(total_time / float(pages)))
 
 
 def test_commit_walk(proj, limit=None):
     repo, proj = _get_repo(proj)
 
-    print 'processing', jn(PROJECT_PATH, proj)
+    print('processing', jn(PROJECT_PATH, proj))
     total_time = 0
 
     cnt = 0
     for i in repo:
         cnt += 1
         raw_cs = '/'.join((proj, 'changeset', i.raw_id))
         if limit and limit == cnt:
             break
 
         full_uri = (BASE_URI % raw_cs)
-        print '%s visiting %s\%s' % (cnt, full_uri, i)
+        print('%s visiting %s\%s' % (cnt, full_uri, i))
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
-        print '%s visited %s\%s size:%s req:%s ms' % (cnt, full_uri, i, size, e)
+        print('%s visited %s\%s size:%s req:%s ms' % (cnt, full_uri, i, size, e))
 
-    print 'total_time', total_time
-    print 'average on req', total_time / float(cnt)
+    print('total_time {}'.format(total_time))
+    print('average on req {}'.format(total_time / float(cnt)))
 
 
 def test_files_walk(proj, limit=100):
     repo, proj = _get_repo(proj)
 
-    print 'processing', jn(PROJECT_PATH, proj)
+    print('processing {}'.format(jn(PROJECT_PATH, proj)))
     total_time = 0
 
     paths_ = OrderedSet([''])
     try:
         tip = repo.get_commit('tip')
         for topnode, dirs, files in tip.walk('/'):
 
             for dir in dirs:
                 paths_.add(dir.path)
                 for f in dir:
                     paths_.add(f.path)
 
             for f in files:
                 paths_.add(f.path)
 
     except RepositoryError as e:
         pass
 
     cnt = 0
     for f in paths_:
         cnt += 1
         if limit and limit == cnt:
             break
 
         file_path = '/'.join((proj, 'files', 'tip', f))
         full_uri = (BASE_URI % file_path)
-        print '%s visiting %s' % (cnt, full_uri)
+        print('%s visiting %s' % (cnt, full_uri))
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
-        print '%s visited OK size:%s req:%s ms' % (cnt, size, e)
+        print('%s visited OK size:%s req:%s ms' % (cnt, size, e))
 
-    print 'total_time', total_time
-    print 'average on req', total_time / float(cnt)
+    print('total_time {}'.format(total_time))
+    print('average on req {}'.format(total_time / float(cnt)))
 
 if __name__ == '__main__':
     for path in PROJECTS:
         repo = vcs.get_repo(jn(PROJECT_PATH, path))
         for i in range(PASES):
-            print 'PASS %s/%s' % (i, PASES)
+            print('PASS %s/%s' % (i, PASES))
             test_changelog_walk(repo, pages=80)
             test_commit_walk(repo, limit=100)
             test_files_walk(repo, limit=100)