# HG changeset patch # User RhodeCode Admin # Date 2023-07-18 09:46:31 # Node ID 0a27bd22c7956f22fdfa136eb0fb55ed06bb6a10 # Parent 62af941ab9856ade732f2afb1638b7c7d3498b7f tests: fixed all tests for python3 BIG changes diff --git a/conftest.py b/conftest.py --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -18,8 +17,14 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import pytest -from rhodecode.lib import ext_json +import pytest # noqa + +# keep the imports to have a toplevel conftest.py but still importable from EE edition +from rhodecode.tests.conftest_common import ( # noqa + pytest_generate_tests, + pytest_runtest_makereport, + pytest_addoption +) pytest_plugins = [ @@ -29,121 +34,7 @@ pytest_plugins = [ def pytest_configure(config): - from rhodecode.config import patches - - -def pytest_addoption(parser): - - def _parse_json(value): - return ext_json.str_json(value) if value else None - - def _split_comma(value): - return value.split(',') - - parser.addoption( - '--keep-tmp-path', action='store_true', - help="Keep the test temporary directories") - - parser.addoption( - '--backends', action='store', type=_split_comma, - default=['git', 'hg', 'svn'], - help="Select which backends to test for backend specific tests.") - parser.addoption( - '--dbs', action='store', type=_split_comma, - default=['sqlite'], - help="Select which database to test for database specific tests. " - "Possible options are sqlite,postgres,mysql") - parser.addoption( - '--appenlight', '--ae', action='store_true', - help="Track statistics in appenlight.") - parser.addoption( - '--appenlight-api-key', '--ae-key', - help="API key for Appenlight.") - parser.addoption( - '--appenlight-url', '--ae-url', - default="https://ae.rhodecode.com", - help="Appenlight service URL, defaults to https://ae.rhodecode.com") - parser.addoption( - '--sqlite-connection-string', action='store', - default='', help="Connection string for the dbs tests with SQLite") - parser.addoption( - '--postgres-connection-string', action='store', - default='', help="Connection string for the dbs tests with Postgres") - parser.addoption( - '--mysql-connection-string', action='store', - default='', help="Connection string for the dbs tests with MySQL") - parser.addoption( - '--repeat', type=int, default=100, - help="Number of repetitions in performance tests.") - - parser.addoption( - '--test-loglevel', dest='test_loglevel', - help="Set default Logging level for tests, critical(default), error, warn , info, debug") - group = parser.getgroup('pylons') - group.addoption( - '--with-pylons', dest='pyramid_config', - help="Set up a Pylons environment with the specified config file.") - group.addoption( - '--ini-config-override', action='store', type=_parse_json, - default=None, dest='pyramid_config_override', help=( - "Overrides the .ini file settings. Should be specified in JSON" - " format, e.g. '{\"section\": {\"parameter\": \"value\", ...}}'" - ) - ) - parser.addini( - 'pyramid_config', - "Set up a Pyramid environment with the specified config file.") - - vcsgroup = parser.getgroup('vcs') - vcsgroup.addoption( - '--without-vcsserver', dest='with_vcsserver', action='store_false', - help="Do not start the VCSServer in a background process.") - vcsgroup.addoption( - '--with-vcsserver-http', dest='vcsserver_config_http', - help="Start the HTTP VCSServer with the specified config file.") - vcsgroup.addoption( - '--vcsserver-protocol', dest='vcsserver_protocol', - help="Start the VCSServer with HTTP protocol support.") - vcsgroup.addoption( - '--vcsserver-config-override', action='store', type=_parse_json, - default=None, dest='vcsserver_config_override', help=( - "Overrides the .ini file settings for the VCSServer. " - "Should be specified in JSON " - "format, e.g. '{\"section\": {\"parameter\": \"value\", ...}}'" - ) - ) - vcsgroup.addoption( - '--vcsserver-port', action='store', type=int, - default=None, help=( - "Allows to set the port of the vcsserver. Useful when testing " - "against an already running server and random ports cause " - "trouble.")) - parser.addini( - 'vcsserver_config_http', - "Start the HTTP VCSServer with the specified config file.") - parser.addini( - 'vcsserver_protocol', - "Start the VCSServer with HTTP protocol support.") - - -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_makereport(item, call): - """ - Adding the remote traceback if the exception has this information. - - VCSServer attaches this information as the attribute `_vcs_server_traceback` - to the exception instance. - """ - outcome = yield - report = outcome.get_result() - - if call.excinfo: - exc = call.excinfo.value - vcsserver_traceback = getattr(exc, '_vcs_server_traceback', None) - - if vcsserver_traceback and report.outcome == 'failed': - section = f'VCSServer remote traceback {report.when}' - report.sections.append((section, vcsserver_traceback)) + from rhodecode.config import patches # noqa def pytest_collection_modifyitems(session, config, items): @@ -152,7 +43,7 @@ def pytest_collection_modifyitems(sessio i for i in items if getattr(i.obj, '__test__', True)] items[:] = remaining - # NOTE(marcink): custom test ordering, db tests and vcstests are slowes and should + # NOTE(marcink): custom test ordering, db tests and vcstests are slowest and should # be executed at the end for faster test feedback def sorter(item): pos = 0 @@ -165,37 +56,3 @@ def pytest_collection_modifyitems(sessio return pos items.sort(key=sorter) - - -def get_backends_from_metafunc(metafunc): - requested_backends = set(metafunc.config.getoption('--backends')) - backend_mark = metafunc.definition.get_closest_marker('backends') - if backend_mark: - # Supported backends by this test function, created from - # pytest.mark.backends - backends = backend_mark.args - elif hasattr(metafunc.cls, 'backend_alias'): - # Support class attribute "backend_alias", this is mainly - # for legacy reasons for tests not yet using pytest.mark.backends - backends = [metafunc.cls.backend_alias] - else: - backends = metafunc.config.getoption('--backends') - return requested_backends.intersection(backends) - - -def pytest_generate_tests(metafunc): - - # Support test generation based on --backend parameter - if 'backend_alias' in metafunc.fixturenames: - backends = get_backends_from_metafunc(metafunc) - scope = None - if not backends: - pytest.skip("Not enabled for any of selected backends") - - metafunc.parametrize('backend_alias', backends, scope=scope) - - backend_mark = metafunc.definition.get_closest_marker('backends') - if backend_mark: - backends = get_backends_from_metafunc(metafunc) - if not backends: - pytest.skip("Not enabled for any of selected backends") diff --git a/pytest.ini b/pytest.ini --- a/pytest.ini +++ b/pytest.ini @@ -13,6 +13,8 @@ addopts = --capture=no --show-capture=all +# --test-loglevel=INFO, show log-level during execution + markers = vcs_operations: Mark tests depending on a running RhodeCode instance. xfail_backends: Mark tests as xfail for given backends. diff --git a/rhodecode/apps/admin/tests/test_admin_audit_logs.py b/rhodecode/apps/admin/tests/test_admin_audit_logs.py --- a/rhodecode/apps/admin/tests/test_admin_audit_logs.py +++ b/rhodecode/apps/admin/tests/test_admin_audit_logs.py @@ -23,15 +23,17 @@ import datetime import pytest +from rhodecode.lib.str_utils import safe_str from rhodecode.tests import * from rhodecode.tests.fixture import FIXTURES from rhodecode.model.db import UserLog from rhodecode.model.meta import Session -from rhodecode.lib.utils2 import safe_unicode def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { @@ -69,7 +71,7 @@ class TestAdminController(object): for row in csv.DictReader(f): ul = UserLog() for k, v in row.items(): - v = safe_unicode(v) + v = safe_str(v) if k == 'action_date': v = strptime(v) if k in ['user_id', 'repository_id']: diff --git a/rhodecode/apps/admin/tests/test_admin_defaults.py b/rhodecode/apps/admin/tests/test_admin_defaults.py --- a/rhodecode/apps/admin/tests/test_admin_defaults.py +++ b/rhodecode/apps/admin/tests/test_admin_defaults.py @@ -24,7 +24,9 @@ from rhodecode.model.settings import Set def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { diff --git a/rhodecode/apps/admin/tests/test_admin_main_views.py b/rhodecode/apps/admin/tests/test_admin_main_views.py --- a/rhodecode/apps/admin/tests/test_admin_main_views.py +++ b/rhodecode/apps/admin/tests/test_admin_main_views.py @@ -26,7 +26,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { @@ -50,7 +52,10 @@ class TestAdminMainView(TestController): response = self.app.get(route_path('admin_home'), status=200) response.mustcontain("Administration area") - def test_redirect_pull_request_view(self, view): + @pytest.mark.parametrize('view', [ + 'pull_requests_global', + ]) + def test_redirect_pull_request_view_global(self, view): self.log_user() self.app.get( route_path(view, pull_request_id='xxxx'), diff --git a/rhodecode/apps/admin/tests/test_admin_permissions.py b/rhodecode/apps/admin/tests/test_admin_permissions.py --- a/rhodecode/apps/admin/tests/test_admin_permissions.py +++ b/rhodecode/apps/admin/tests/test_admin_permissions.py @@ -28,7 +28,9 @@ from rhodecode.tests import ( def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { diff --git a/rhodecode/apps/admin/tests/test_admin_repos.py b/rhodecode/apps/admin/tests/test_admin_repos.py --- a/rhodecode/apps/admin/tests/test_admin_repos.py +++ b/rhodecode/apps/admin/tests/test_admin_repos.py @@ -17,7 +17,9 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import urllib.request, urllib.parse, urllib.error +import urllib.request +import urllib.parse +import urllib.error import mock import pytest @@ -42,7 +44,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repos': ADMIN_PREFIX + '/repos', @@ -92,12 +96,14 @@ class TestAdminRepos(object): assert ['hg', 'git', 'svn'] == [x.get('value') for x in assert_response.get_elements('[name=repo_type]')] @pytest.mark.parametrize( - "suffix", [u'', u'xxa'], ids=['', 'non-ascii']) + "suffix", ['', 'xxa'], ids=['', 'non-ascii']) def test_create(self, autologin_user, backend, suffix, csrf_token): repo_name_unicode = backend.new_repo_name(suffix=suffix) - repo_name = repo_name_unicode.encode('utf8') - description_unicode = u'description for newly created repo' + suffix - description = description_unicode.encode('utf8') + repo_name = repo_name_unicode + + description_unicode = 'description for newly created repo' + suffix + description = description_unicode + response = self.app.post( route_path('repo_create'), fixture._get_repo_create_params( @@ -127,20 +133,20 @@ class TestAdminRepos(object): self.assert_repository_is_created_correctly( repo_name, description, backend) - @pytest.mark.parametrize("suffix", [u'', u'ąćę'], ids=['', 'non-ascii']) + @pytest.mark.parametrize("suffix", ['', '_ąćę'], ids=['', 'non-ascii']) def test_create_in_group( self, autologin_user, backend, suffix, csrf_token): # create GROUP - group_name = 'sometest_%s' % backend.alias + group_name = f'sometest_{backend.alias}' gr = RepoGroupModel().create(group_name=group_name, group_description='test', owner=TEST_USER_ADMIN_LOGIN) Session().commit() - repo_name = u'ingroup' + suffix - repo_name_full = RepoGroup.url_sep().join( - [group_name, repo_name]) - description = u'description for newly created repo' + repo_name = f'ingroup{suffix}' + repo_name_full = RepoGroup.url_sep().join([group_name, repo_name]) + description = 'description for newly created repo' + self.app.post( route_path('repo_create'), fixture._get_repo_create_params( @@ -483,17 +489,15 @@ class TestAdminRepos(object): # repo must not be in filesystem ! assert not repo_on_filesystem(repo_name) - def assert_repository_is_created_correctly( - self, repo_name, description, backend): - repo_name_utf8 = safe_str(repo_name) + def assert_repository_is_created_correctly(self, repo_name, description, backend): + url_quoted_repo_name = urllib.parse.quote(repo_name) # run the check page that triggers the flash message response = self.app.get( - route_path('repo_creating_check', repo_name=safe_str(repo_name))) - assert response.json == {u'result': True} + route_path('repo_creating_check', repo_name=repo_name)) + assert response.json == {'result': True} - flash_msg = u'Created repository {}'.format( - urllib.parse.quote(repo_name_utf8), repo_name) + flash_msg = 'Created repository {}'.format(url_quoted_repo_name, repo_name) assert_session_flash(response, flash_msg) # test if the repo was created in the database @@ -504,7 +508,7 @@ class TestAdminRepos(object): # test if the repository is visible in the list ? response = self.app.get( - h.route_path('repo_summary', repo_name=safe_str(repo_name))) + h.route_path('repo_summary', repo_name=repo_name)) response.mustcontain(repo_name) response.mustcontain(backend.alias) diff --git a/rhodecode/apps/admin/tests/test_admin_repository_groups.py b/rhodecode/apps/admin/tests/test_admin_repository_groups.py --- a/rhodecode/apps/admin/tests/test_admin_repository_groups.py +++ b/rhodecode/apps/admin/tests/test_admin_repository_groups.py @@ -33,7 +33,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repo_groups': ADMIN_PREFIX + '/repo_groups', @@ -106,7 +108,7 @@ class TestAdminRepositoryGroups(object): 'hg_repo_ąć', ]) def test_create(self, autologin_user, repo_group_name, csrf_token): - repo_group_name_unicode = repo_group_name.decode('utf8') + repo_group_name_non_ascii = repo_group_name description = 'description for newly created repo group' response = self.app.post( @@ -123,14 +125,14 @@ class TestAdminRepositoryGroups(object): assert_session_flash( response, 'Created repository group %s' % ( - repo_gr_url, repo_group_name_unicode)) + repo_gr_url, repo_group_name_non_ascii)) # # test if the repo group was created in the database new_repo_group = RepoGroupModel()._get_repo_group( - repo_group_name_unicode) + repo_group_name_non_ascii) assert new_repo_group is not None - assert new_repo_group.group_name == repo_group_name_unicode + assert new_repo_group.group_name == repo_group_name_non_ascii assert new_repo_group.group_description == description # test if the repository is visible in the list ? @@ -143,7 +145,7 @@ class TestAdminRepositoryGroups(object): if not is_on_filesystem: self.fail('no repo group %s in filesystem' % repo_group_name) - RepoGroupModel().delete(repo_group_name_unicode) + RepoGroupModel().delete(repo_group_name_non_ascii) Session().commit() @pytest.mark.parametrize('repo_group_name', [ @@ -159,7 +161,7 @@ class TestAdminRepositoryGroups(object): expected_group_name = '{}/{}'.format( parent_group_name, repo_group_name) - expected_group_name_unicode = expected_group_name.decode('utf8') + expected_group_name_non_ascii = expected_group_name try: response = self.app.post( @@ -175,9 +177,9 @@ class TestAdminRepositoryGroups(object): u'Created repository group %s' % ( h.route_path('repo_group_home', repo_group_name=expected_group_name), - expected_group_name_unicode)) + expected_group_name_non_ascii)) finally: - RepoGroupModel().delete(expected_group_name_unicode) + RepoGroupModel().delete(expected_group_name_non_ascii) Session().commit() def test_user_with_creation_permissions_cannot_create_subgroups( diff --git a/rhodecode/apps/admin/tests/test_admin_settings.py b/rhodecode/apps/admin/tests/test_admin_settings.py --- a/rhodecode/apps/admin/tests/test_admin_settings.py +++ b/rhodecode/apps/admin/tests/test_admin_settings.py @@ -22,19 +22,20 @@ import pytest import rhodecode from rhodecode.apps._base import ADMIN_PREFIX -from rhodecode.lib.utils2 import md5 +from rhodecode.lib.hash_utils import md5_safe from rhodecode.model.db import RhodeCodeUi from rhodecode.model.meta import Session from rhodecode.model.settings import SettingsModel, IssueTrackerSettingsModel from rhodecode.tests import assert_session_flash -from rhodecode.tests.utils import AssertResponse UPDATE_DATA_QUALNAME = 'rhodecode.model.update.UpdateModel.get_update_data' def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { @@ -233,6 +234,7 @@ class TestAdminSettingsGlobal(object): route_path('admin_settings_global_update'), params=params) assert_session_flash(response, 'Updated application settings') + app_settings = SettingsModel().get_all_settings() del settings['csrf_token'] for key, value in settings.items(): @@ -413,8 +415,9 @@ class TestAdminSettingsVcs(object): @pytest.fixture() def disable_sql_cache(self, request): + # patch _do_orm_execute so it returns None similar like if we don't use a cached query patcher = mock.patch( - 'rhodecode.lib.caching_query.FromCache.process_query') + 'rhodecode.lib.caching_query.ORMCache._do_orm_execute', return_value=None) request.addfinalizer(patcher.stop) patcher.start() @@ -428,8 +431,7 @@ class TestAdminSettingsVcs(object): @pytest.fixture(scope='class', autouse=True) def cleanup_settings(self, request, baseapp): ui_id = RhodeCodeUi.ui_id - original_ids = list( - r.ui_id for r in RhodeCodeUi.query().values(ui_id)) + original_ids = [r.ui_id for r in RhodeCodeUi.query().with_entities(ui_id)] @request.addfinalizer def cleanup(): @@ -644,9 +646,9 @@ class TestAdminSettingsIssueTracker(obje } self.app.post(post_url, post_data, status=302) settings = SettingsModel().get_all_settings() - self.uid = md5(pattern) + self.uid = md5_safe(pattern) assert settings[self.PATTERN_KEY+self.uid] == pattern - self.another_uid = md5(another_pattern) + self.another_uid = md5_safe(another_pattern) assert settings[self.PATTERN_KEY+self.another_uid] == another_pattern @request.addfinalizer @@ -654,7 +656,7 @@ class TestAdminSettingsIssueTracker(obje defaults = SettingsModel().get_all_settings() entries = [name for name in defaults if ( - (self.uid in name) or (self.another_uid) in name)] + (self.uid in name) or (self.another_uid in name))] start = len(self.RC_PREFIX) for del_key in entries: # TODO: anderson: get_by_name needs name without prefix @@ -667,7 +669,7 @@ class TestAdminSettingsIssueTracker(obje self, autologin_user, backend, csrf_token, request): old_pattern = 'issuetracker_pat1' - old_uid = md5(old_pattern) + old_uid = md5_safe(old_pattern) post_url = route_path('admin_settings_issuetracker_update') post_data = { @@ -681,7 +683,7 @@ class TestAdminSettingsIssueTracker(obje self.app.post(post_url, post_data, status=302) new_pattern = 'issuetracker_pat1_edited' - self.new_uid = md5(new_pattern) + self.new_uid = md5_safe(new_pattern) post_url = route_path('admin_settings_issuetracker_update') post_data = { @@ -708,7 +710,7 @@ class TestAdminSettingsIssueTracker(obje self, autologin_user, csrf_token, request, settings_util): prefix = 'issuetracker' pattern = 'issuetracker_pat' - self.uid = md5(pattern) + self.uid = md5_safe(pattern) pattern_key = '_'.join([prefix, 'pat', self.uid]) rc_pattern_key = '_'.join(['rhodecode', pattern_key]) desc_key = '_'.join([prefix, 'desc', self.uid]) @@ -742,7 +744,7 @@ class TestAdminSettingsIssueTracker(obje self, autologin_user, backend, csrf_token, settings_util, xhr_header): old_pattern = 'issuetracker_pat_deleted' - old_uid = md5(old_pattern) + old_uid = md5_safe(old_pattern) post_url = route_path('admin_settings_issuetracker_update') post_data = { diff --git a/rhodecode/apps/admin/tests/test_admin_user_groups.py b/rhodecode/apps/admin/tests/test_admin_user_groups.py --- a/rhodecode/apps/admin/tests/test_admin_user_groups.py +++ b/rhodecode/apps/admin/tests/test_admin_user_groups.py @@ -30,7 +30,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { diff --git a/rhodecode/apps/admin/tests/test_admin_users.py b/rhodecode/apps/admin/tests/test_admin_users.py --- a/rhodecode/apps/admin/tests/test_admin_users.py +++ b/rhodecode/apps/admin/tests/test_admin_users.py @@ -34,7 +34,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { diff --git a/rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py b/rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py --- a/rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py +++ b/rhodecode/apps/admin/tests/test_admin_users_ssh_keys.py @@ -28,7 +28,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { diff --git a/rhodecode/apps/file_store/tests/test_upload_file.py b/rhodecode/apps/file_store/tests/test_upload_file.py --- a/rhodecode/apps/file_store/tests/test_upload_file.py +++ b/rhodecode/apps/file_store/tests/test_upload_file.py @@ -27,7 +27,9 @@ from rhodecode.apps.file_store import ut def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'upload_file': '/_file_store/upload', @@ -59,7 +61,7 @@ class TestFileStoreViews(TestController) status = 200 store = utils.get_file_storage({config_keys.store_path: store_path}) filesystem_file = os.path.join(str(tmpdir), fid) - with open(filesystem_file, 'wb') as f: + with open(filesystem_file, 'wt') as f: f.write(content) with open(filesystem_file, 'rb') as f: @@ -120,7 +122,7 @@ class TestFileStoreViews(TestController) self.log_user() response = self.app.post( route_path('upload_file'), - upload_files=[('store_file', 'myfile.txt', 'SOME CONTENT')], + upload_files=[('store_file', b'myfile.txt', b'SOME CONTENT')], params={'csrf_token': self.csrf_token}, status=200) @@ -134,7 +136,7 @@ class TestFileStoreViews(TestController) fid = 'example.txt' filesystem_file = os.path.join(str(tmpdir), fid) - with open(filesystem_file, 'wb') as f: + with open(filesystem_file, 'wt') as f: f.write(content) with open(filesystem_file, 'rb') as f: diff --git a/rhodecode/apps/gist/tests/test_admin_gists.py b/rhodecode/apps/gist/tests/test_admin_gists.py --- a/rhodecode/apps/gist/tests/test_admin_gists.py +++ b/rhodecode/apps/gist/tests/test_admin_gists.py @@ -30,7 +30,8 @@ from rhodecode.tests import ( def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { @@ -59,7 +60,7 @@ class GistUtility(object): self._gist_ids = [] def __call__( - self, f_name, content='some gist', lifetime=-1, + self, f_name: bytes, content: bytes = b'some gist', lifetime=-1, description='gist-desc', gist_type='public', acl_level=Gist.GIST_PUBLIC, owner=TEST_USER_ADMIN_LOGIN): gist_mapping = { @@ -94,14 +95,14 @@ class TestGistsController(TestController def test_index_empty(self, create_gist): self.log_user() response = self.app.get(route_path('gists_show')) - response.mustcontain('data: [],') + response.mustcontain('var gist_data = [];') def test_index(self, create_gist): self.log_user() - g1 = create_gist('gist1') - g2 = create_gist('gist2', lifetime=1400) - g3 = create_gist('gist3', description='gist3-desc') - g4 = create_gist('gist4', gist_type='private').gist_access_id + g1 = create_gist(b'gist1') + g2 = create_gist(b'gist2', lifetime=1400) + g3 = create_gist(b'gist3', description='gist3-desc') + g4 = create_gist(b'gist4', gist_type='private').gist_access_id response = self.app.get(route_path('gists_show')) response.mustcontain(g1.gist_access_id) @@ -111,13 +112,12 @@ class TestGistsController(TestController response.mustcontain(no=[g4]) # Expiration information should be visible - expires_tag = '%s' % h.age_component( - h.time_to_utcdatetime(g2.gist_expires)) + expires_tag = str(h.age_component(h.time_to_utcdatetime(g2.gist_expires))) response.mustcontain(expires_tag.replace('"', '\\"')) def test_index_private_gists(self, create_gist): self.log_user() - gist = create_gist('gist5', gist_type='private') + gist = create_gist(b'gist5', gist_type='private') response = self.app.get(route_path('gists_show', params=dict(private=1))) # and privates @@ -125,10 +125,10 @@ class TestGistsController(TestController def test_index_show_all(self, create_gist): self.log_user() - create_gist('gist1') - create_gist('gist2', lifetime=1400) - create_gist('gist3', description='gist3-desc') - create_gist('gist4', gist_type='private') + create_gist(b'gist1') + create_gist(b'gist2', lifetime=1400) + create_gist(b'gist3', description='gist3-desc') + create_gist(b'gist4', gist_type='private') response = self.app.get(route_path('gists_show', params=dict(all=1))) @@ -139,9 +139,9 @@ class TestGistsController(TestController def test_index_show_all_hidden_from_regular(self, create_gist): self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS) - create_gist('gist2', gist_type='private') - create_gist('gist3', gist_type='private') - create_gist('gist4', gist_type='private') + create_gist(b'gist2', gist_type='private') + create_gist(b'gist3', gist_type='private') + create_gist(b'gist4', gist_type='private') response = self.app.get(route_path('gists_show', params=dict(all=1))) @@ -181,7 +181,7 @@ class TestGistsController(TestController def test_access_expired_gist(self, create_gist): self.log_user() - gist = create_gist('never-see-me') + gist = create_gist(b'never-see-me') gist.gist_expires = 0 # 1970 Session().add(gist) Session().commit() @@ -269,7 +269,7 @@ class TestGistsController(TestController def test_delete(self, create_gist): self.log_user() - gist = create_gist('delete-me') + gist = create_gist(b'delete-me') response = self.app.post( route_path('gist_delete', gist_id=gist.gist_id), params={'csrf_token': self.csrf_token}) @@ -277,7 +277,7 @@ class TestGistsController(TestController def test_delete_normal_user_his_gist(self, create_gist): self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS) - gist = create_gist('delete-me', owner=TEST_USER_REGULAR_LOGIN) + gist = create_gist(b'delete-me', owner=TEST_USER_REGULAR_LOGIN) response = self.app.post( route_path('gist_delete', gist_id=gist.gist_id), @@ -286,14 +286,14 @@ class TestGistsController(TestController def test_delete_normal_user_not_his_own_gist(self, create_gist): self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS) - gist = create_gist('delete-me-2') + gist = create_gist(b'delete-me-2') self.app.post( route_path('gist_delete', gist_id=gist.gist_id), params={'csrf_token': self.csrf_token}, status=404) def test_show(self, create_gist): - gist = create_gist('gist-show-me') + gist = create_gist(b'gist-show-me') response = self.app.get(route_path('gist_show', gist_id=gist.gist_access_id)) response.mustcontain('added file: gist-show-me<') @@ -308,12 +308,12 @@ class TestGistsController(TestController def test_show_without_hg(self, create_gist): with mock.patch( 'rhodecode.lib.vcs.settings.ALIASES', ['git']): - gist = create_gist('gist-show-me-again') + gist = create_gist(b'gist-show-me-again') self.app.get( route_path('gist_show', gist_id=gist.gist_access_id), status=200) def test_show_acl_private(self, create_gist): - gist = create_gist('gist-show-me-only-when-im-logged-in', + gist = create_gist(b'gist-show-me-only-when-im-logged-in', acl_level=Gist.ACL_LEVEL_PRIVATE) self.app.get( route_path('gist_show', gist_id=gist.gist_access_id), status=404) @@ -331,7 +331,7 @@ class TestGistsController(TestController response.mustcontain('gist-desc') def test_show_as_raw(self, create_gist): - gist = create_gist('gist-show-me', content='GIST CONTENT') + gist = create_gist(b'gist-show-me', content=b'GIST CONTENT') response = self.app.get( route_path('gist_show_formatted', gist_id=gist.gist_access_id, revision='tip', @@ -339,7 +339,7 @@ class TestGistsController(TestController assert response.text == 'GIST CONTENT' def test_show_as_raw_individual_file(self, create_gist): - gist = create_gist('gist-show-me-raw', content='GIST BODY') + gist = create_gist(b'gist-show-me-raw', content=b'GIST BODY') response = self.app.get( route_path('gist_show_formatted_path', gist_id=gist.gist_access_id, format='raw', @@ -348,24 +348,24 @@ class TestGistsController(TestController def test_edit_page(self, create_gist): self.log_user() - gist = create_gist('gist-for-edit', content='GIST EDIT BODY') + gist = create_gist(b'gist-for-edit', content=b'GIST EDIT BODY') response = self.app.get(route_path('gist_edit', gist_id=gist.gist_access_id)) response.mustcontain('GIST EDIT BODY') def test_edit_page_non_logged_user(self, create_gist): - gist = create_gist('gist-for-edit', content='GIST EDIT BODY') + gist = create_gist(b'gist-for-edit', content=b'GIST EDIT BODY') self.app.get(route_path('gist_edit', gist_id=gist.gist_access_id), status=302) def test_edit_normal_user_his_gist(self, create_gist): self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS) - gist = create_gist('gist-for-edit', owner=TEST_USER_REGULAR_LOGIN) + gist = create_gist(b'gist-for-edit', owner=TEST_USER_REGULAR_LOGIN) self.app.get(route_path('gist_edit', gist_id=gist.gist_access_id, status=200)) def test_edit_normal_user_not_his_own_gist(self, create_gist): self.log_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS) - gist = create_gist('delete-me') + gist = create_gist(b'delete-me') self.app.get(route_path('gist_edit', gist_id=gist.gist_access_id), status=404) @@ -375,7 +375,7 @@ class TestGistsController(TestController password = 'test' user = user_util.create_user( firstname=xss_atack_string, password=password) - create_gist('gist', gist_type='public', owner=user.username) + create_gist(b'gist', gist_type='public', owner=user.username) response = self.app.get(route_path('gists_show')) response.mustcontain(xss_escaped_string) @@ -385,6 +385,6 @@ class TestGistsController(TestController password = 'test' user = user_util.create_user( lastname=xss_atack_string, password=password) - create_gist('gist', gist_type='public', owner=user.username) + create_gist(b'gist', gist_type='public', owner=user.username) response = self.app.get(route_path('gists_show')) response.mustcontain(xss_escaped_string) diff --git a/rhodecode/apps/home/tests/test_get_repo_list_data.py b/rhodecode/apps/home/tests/test_get_repo_list_data.py --- a/rhodecode/apps/home/tests/test_get_repo_list_data.py +++ b/rhodecode/apps/home/tests/test_get_repo_list_data.py @@ -18,18 +18,20 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import json - from . import assert_and_get_repo_list_content from rhodecode.tests import TestController from rhodecode.tests.fixture import Fixture from rhodecode.model.db import Repository +from rhodecode.lib.ext_json import json + fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repo_list_data': '/_repos', diff --git a/rhodecode/apps/home/tests/test_get_user_data.py b/rhodecode/apps/home/tests/test_get_user_data.py --- a/rhodecode/apps/home/tests/test_get_user_data.py +++ b/rhodecode/apps/home/tests/test_get_user_data.py @@ -17,19 +17,19 @@ # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ - -import json import pytest from rhodecode.tests import TestController from rhodecode.tests.fixture import Fixture - +from rhodecode.lib.ext_json import json fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'user_autocomplete_data': '/_users', diff --git a/rhodecode/apps/home/tests/test_get_user_group_data.py b/rhodecode/apps/home/tests/test_get_user_group_data.py --- a/rhodecode/apps/home/tests/test_get_user_group_data.py +++ b/rhodecode/apps/home/tests/test_get_user_group_data.py @@ -1,22 +1,4 @@ -# Copyright (C) 2016-2020 RhodeCode GmbH -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License, version 3 -# (only), as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# This program is dual-licensed. If you wish to learn more about the -# RhodeCode Enterprise Edition, including its added features, Support services, -# and proprietary license terms, please see https://rhodecode.com/licenses/ -# -*- coding: utf-8 -*- # Copyright (C) 2016-2020 RhodeCode GmbH # @@ -36,19 +18,39 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import json + +# Copyright (C) 2016-2020 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ import pytest from rhodecode.tests import TestController from rhodecode.tests.fixture import Fixture +from rhodecode.lib.ext_json import json fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'user_autocomplete_data': '/_users', diff --git a/rhodecode/apps/home/tests/test_home.py b/rhodecode/apps/home/tests/test_home.py --- a/rhodecode/apps/home/tests/test_home.py +++ b/rhodecode/apps/home/tests/test_home.py @@ -163,7 +163,7 @@ class TestHomeController(TestController) 'show_version', state, 'bool') Session().add(sett) Session().commit() - SettingsModel().invalidate_settings_cache() + SettingsModel().invalidate_settings_cache(hard=True) response = self.app.get(route_path('home')) if state is True: diff --git a/rhodecode/apps/journal/tests/test_journal.py b/rhodecode/apps/journal/tests/test_journal.py --- a/rhodecode/apps/journal/tests/test_journal.py +++ b/rhodecode/apps/journal/tests/test_journal.py @@ -27,7 +27,9 @@ from rhodecode.model.db import UserFollo def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'journal': ADMIN_PREFIX + '/journal', diff --git a/rhodecode/apps/login/tests/test_login.py b/rhodecode/apps/login/tests/test_login.py --- a/rhodecode/apps/login/tests/test_login.py +++ b/rhodecode/apps/login/tests/test_login.py @@ -38,7 +38,9 @@ whitelist_view = ['RepoCommitsView:repo_ def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { @@ -160,18 +162,28 @@ class TestLoginController(object): 'file:///etc/passwd', 'ftp://some.ftp.server', 'http://other.domain', - '/\r\nX-Forwarded-Host: http://example.org', ], ids=no_newline_id_generator) def test_login_bad_came_froms(self, url_came_from): _url = '{}?came_from={}'.format(route_path('login'), url_came_from) response = self.app.post( - _url, - {'username': 'test_admin', 'password': 'test12'}) + _url, {'username': 'test_admin', 'password': 'test12'}, status=302) assert response.status == '302 Found' response = response.follow() assert response.status == '200 OK' assert response.request.path == '/' + @pytest.mark.xfail(reason="newline params changed behaviour in python3") + @pytest.mark.parametrize("url_came_from", [ + '/\r\nX-Forwarded-Host: \rhttp://example.org', + ], ids=no_newline_id_generator) + def test_login_bad_came_froms_404(self, url_came_from): + _url = '{}?came_from={}'.format(route_path('login'), url_came_from) + response = self.app.post( + _url, {'username': 'test_admin', 'password': 'test12'}, status=302) + + response = response.follow() + assert response.status == '404 Not Found' + def test_login_short_password(self): response = self.app.post(route_path('login'), {'username': 'test_admin', @@ -184,7 +196,7 @@ class TestLoginController(object): response = self.app.post( route_path('login'), {'username': user_regular.username, - 'password': u'invalid-non-asci\xe4'.encode('utf8')}) + 'password': 'invalid-non-asci\xe4'.encode('utf8')}) response.mustcontain('invalid user name') response.mustcontain('invalid password') @@ -486,6 +498,10 @@ class TestLoginController(object): auth_token = user_admin.api_key with fixture.anon_access(False): + # webtest uses linter to check if response is bytes, + # and we use memoryview here as a wrapper, quick turn-off + self.app.lint = False + self.app.get( route_path('repo_commit_raw', repo_name=HG_REPO, commit_id='tip', @@ -511,6 +527,9 @@ class TestLoginController(object): assert auth_token with fixture.anon_access(False): + # webtest uses linter to check if response is bytes, + # and we use memoryview here as a wrapper, quick turn-off + self.app.lint = False self.app.get( route_path('repo_commit_raw', repo_name=HG_REPO, commit_id='tip', @@ -536,6 +555,10 @@ class TestLoginController(object): with mock.patch.dict('rhodecode.CONFIG', whitelist): with fixture.anon_access(False): + # webtest uses linter to check if response is bytes, + # and we use memoryview here as a wrapper, quick turn-off + self.app.lint = False + self.app.get( route_path('repo_commit_raw', repo_name=HG_REPO, commit_id='tip', @@ -552,6 +575,9 @@ class TestLoginController(object): TEST_USER_ADMIN_LOGIN, 'test') Session().commit() with fixture.anon_access(False): + # webtest uses linter to check if response is bytes, + # and we use memoryview here as a wrapper, quick turn-off + self.app.lint = False self.app.get( route_path('repo_commit_raw', repo_name=HG_REPO, commit_id='tip', @@ -572,6 +598,9 @@ class TestLoginController(object): Session().add(new_auth_token) Session().commit() with fixture.anon_access(False): + # webtest uses linter to check if response is bytes, + # and we use memoryview here as a wrapper, quick turn-off + self.app.lint = False self.app.get( route_path('repo_commit_raw', repo_name=HG_REPO, commit_id='tip', diff --git a/rhodecode/apps/login/tests/test_password_reset.py b/rhodecode/apps/login/tests/test_password_reset.py --- a/rhodecode/apps/login/tests/test_password_reset.py +++ b/rhodecode/apps/login/tests/test_password_reset.py @@ -30,7 +30,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { diff --git a/rhodecode/apps/my_account/tests/test_my_account_notifications.py b/rhodecode/apps/my_account/tests/test_my_account_notifications.py --- a/rhodecode/apps/my_account/tests/test_my_account_notifications.py +++ b/rhodecode/apps/my_account/tests/test_my_account_notifications.py @@ -34,7 +34,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { @@ -133,7 +135,8 @@ class TestNotificationsController(TestCo u2 = User.get(u2.user_id) # check DB - get_notif = lambda un: [x.notification for x in un] + def get_notif(un): + return [x.notification for x in un] assert get_notif(cur_user.notifications) == [notification] assert get_notif(u1.notifications) == [notification] assert get_notif(u2.notifications) == [notification] diff --git a/rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py b/rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py --- a/rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py +++ b/rhodecode/apps/my_account/tests/test_my_account_ssh_keys.py @@ -28,7 +28,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { diff --git a/rhodecode/apps/repo_group/tests/test_repo_groups_advanced.py b/rhodecode/apps/repo_group/tests/test_repo_groups_advanced.py --- a/rhodecode/apps/repo_group/tests/test_repo_groups_advanced.py +++ b/rhodecode/apps/repo_group/tests/test_repo_groups_advanced.py @@ -23,7 +23,9 @@ from rhodecode.tests import assert_sessi def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'edit_repo_group_advanced': diff --git a/rhodecode/apps/repo_group/tests/test_repo_groups_permissions.py b/rhodecode/apps/repo_group/tests/test_repo_groups_permissions.py --- a/rhodecode/apps/repo_group/tests/test_repo_groups_permissions.py +++ b/rhodecode/apps/repo_group/tests/test_repo_groups_permissions.py @@ -23,7 +23,9 @@ from rhodecode.tests.utils import permis def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'edit_repo_group_perms': diff --git a/rhodecode/apps/repo_group/tests/test_repo_groups_settings.py b/rhodecode/apps/repo_group/tests/test_repo_groups_settings.py --- a/rhodecode/apps/repo_group/tests/test_repo_groups_settings.py +++ b/rhodecode/apps/repo_group/tests/test_repo_groups_settings.py @@ -23,7 +23,9 @@ from rhodecode.tests import assert_sessi def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'edit_repo_group': '/{repo_group_name}/_edit', diff --git a/rhodecode/apps/repository/tests/test_pull_requests_list.py b/rhodecode/apps/repository/tests/test_pull_requests_list.py --- a/rhodecode/apps/repository/tests/test_pull_requests_list.py +++ b/rhodecode/apps/repository/tests/test_pull_requests_list.py @@ -22,7 +22,9 @@ from rhodecode.model.db import Repositor def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'pullrequest_show_all': '/{repo_name}/pull-request', diff --git a/rhodecode/apps/repository/tests/test_repo_bookmarks.py b/rhodecode/apps/repository/tests/test_repo_bookmarks.py --- a/rhodecode/apps/repository/tests/test_repo_bookmarks.py +++ b/rhodecode/apps/repository/tests/test_repo_bookmarks.py @@ -22,7 +22,9 @@ from rhodecode.model.db import Repositor def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'bookmarks_home': '/{repo_name}/bookmarks', diff --git a/rhodecode/apps/repository/tests/test_repo_branches.py b/rhodecode/apps/repository/tests/test_repo_branches.py --- a/rhodecode/apps/repository/tests/test_repo_branches.py +++ b/rhodecode/apps/repository/tests/test_repo_branches.py @@ -22,7 +22,9 @@ from rhodecode.model.db import Repositor def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'branches_home': '/{repo_name}/branches', diff --git a/rhodecode/apps/repository/tests/test_repo_changelog.py b/rhodecode/apps/repository/tests/test_repo_changelog.py --- a/rhodecode/apps/repository/tests/test_repo_changelog.py +++ b/rhodecode/apps/repository/tests/test_repo_changelog.py @@ -28,7 +28,9 @@ MATCH_HASH = re.compile(r' # Date 2014-01-07 12:21:40 # Node ID 2062ec7beeeaf9f44a1c25c41479565040b930b2 @@ -296,7 +306,7 @@ new file mode 10644 Added a symlink """ + diffs['hg'], - 'git': r"""From fd627b9e0dd80b47be81af07c4a98518244ed2f7 2014-01-07 12:22:20 + 'git': br"""From fd627b9e0dd80b47be81af07c4a98518244ed2f7 2014-01-07 12:22:20 From: Marcin Kuzminski Date: 2014-01-07 12:22:20 Subject: [PATCH] Added a symlink @@ -304,7 +314,7 @@ Subject: [PATCH] Added a symlink --- """ + diffs['git'], - 'svn': r"""# SVN changeset patch + 'svn': br"""# SVN changeset patch # User marcin # Date 2014-09-02 12:25:22.071142 # Revision 393 diff --git a/rhodecode/apps/repository/tests/test_repo_compare.py b/rhodecode/apps/repository/tests/test_repo_compare.py --- a/rhodecode/apps/repository/tests/test_repo_compare.py +++ b/rhodecode/apps/repository/tests/test_repo_compare.py @@ -27,7 +27,9 @@ from rhodecode.tests.utils import Assert def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repo_compare_select': '/{repo_name}/compare', @@ -65,30 +67,30 @@ class TestCompareView(object): # fork = backend.create_repo() + origin = backend.create_repo() # prepare fork commit0 = commit_change( - fork.repo_name, filename='file1', content='A', - message='A', vcs_type=backend.alias, parent=None, newfile=True) + fork.repo_name, filename=b'file1', content=b'A', + message='A - Initial Commit', vcs_type=backend.alias, parent=None, newfile=True) commit1 = commit_change( - fork.repo_name, filename='file1', content='B', + fork.repo_name, filename=b'file1', content=b'B', message='B, child of A', vcs_type=backend.alias, parent=commit0) commit_change( # commit 2 - fork.repo_name, filename='file1', content='C', + fork.repo_name, filename=b'file1', content=b'C', message='C, child of B', vcs_type=backend.alias, parent=commit1) commit3 = commit_change( - fork.repo_name, filename='file1', content='D', + fork.repo_name, filename=b'file1', content=b'D', message='D, child of A', vcs_type=backend.alias, parent=commit0) commit4 = commit_change( - fork.repo_name, filename='file1', content='E', + fork.repo_name, filename=b'file1', content=b'E', message='E, child of D', vcs_type=backend.alias, parent=commit3) # prepare origin repository, taking just the history up to D - origin = backend.create_repo() origin_repo = origin.scm_instance(cache=False) origin_repo.config.clear_section('hooks') @@ -98,7 +100,7 @@ class TestCompareView(object): # Verify test fixture setup # This does not work for git if backend.alias != 'git': - assert 5 == len(fork.scm_instance().commit_ids) + assert 5 == len(fork.scm_instance(cache=False).commit_ids) assert 2 == len(origin_repo.commit_ids) # Comparing the revisions @@ -108,7 +110,8 @@ class TestCompareView(object): source_ref_type="rev", source_ref=commit3.raw_id, target_ref_type="rev", target_ref=commit4.raw_id, params=dict(merge='1', target_repo=fork.repo_name) - )) + ), + status=200) compare_page = ComparePage(response) compare_page.contains_commits([commit4]) @@ -119,7 +122,7 @@ class TestCompareView(object): # commit something ! commit0 = commit_change( - repo1.repo_name, filename='file1', content='line1\n', + repo1.repo_name, filename=b'file1', content=b'line1\n', message='commit1', vcs_type=backend.alias, parent=None, newfile=True) @@ -128,11 +131,11 @@ class TestCompareView(object): # add two extra commit into fork commit1 = commit_change( - repo2.repo_name, filename='file1', content='line1\nline2\n', + repo2.repo_name, filename=b'file1', content=b'line1\nline2\n', message='commit2', vcs_type=backend.alias, parent=commit0) commit2 = commit_change( - repo2.repo_name, filename='file1', content='line1\nline2\nline3\n', + repo2.repo_name, filename=b'file1', content=b'line1\nline2\nline3\n', message='commit3', vcs_type=backend.alias, parent=commit1) commit_id1 = repo1.scm_instance().DEFAULT_BRANCH_NAME @@ -167,7 +170,7 @@ class TestCompareView(object): # commit something ! commit0 = commit_change( - repo1.repo_name, filename='file1', content='line1\n', + repo1.repo_name, filename=b'file1', content=b'line1\n', message='commit1', vcs_type=backend.alias, parent=None, newfile=True) @@ -176,17 +179,17 @@ class TestCompareView(object): # now commit something to origin repo commit_change( - repo1.repo_name, filename='file2', content='line1file2\n', + repo1.repo_name, filename=b'file2', content=b'line1file2\n', message='commit2', vcs_type=backend.alias, parent=commit0, newfile=True) # add two extra commit into fork commit1 = commit_change( - repo2.repo_name, filename='file1', content='line1\nline2\n', + repo2.repo_name, filename=b'file1', content=b'line1\nline2\n', message='commit2', vcs_type=backend.alias, parent=commit0) commit2 = commit_change( - repo2.repo_name, filename='file1', content='line1\nline2\nline3\n', + repo2.repo_name, filename=b'file1', content=b'line1\nline2\nline3\n', message='commit3', vcs_type=backend.alias, parent=commit1) commit_id1 = repo1.scm_instance().DEFAULT_BRANCH_NAME @@ -250,11 +253,11 @@ class TestCompareView(object): # commit something ! commit0 = commit_change( - repo1.repo_name, filename='file1', content='line1\n', + repo1.repo_name, filename=b'file1', content=b'line1\n', message='commit1', vcs_type=backend.alias, parent=None, newfile=True) commit1 = commit_change( - repo1.repo_name, filename='file1', content='line1\nline2\n', + repo1.repo_name, filename=b'file1', content=b'line1\nline2\n', message='commit2', vcs_type=backend.alias, parent=commit0) # fork this repo @@ -262,19 +265,16 @@ class TestCompareView(object): # now make commit3-6 commit2 = commit_change( - repo1.repo_name, filename='file1', content='line1\nline2\nline3\n', + repo1.repo_name, filename=b'file1', content=b'line1\nline2\nline3\n', message='commit3', vcs_type=backend.alias, parent=commit1) commit3 = commit_change( - repo1.repo_name, filename='file1', - content='line1\nline2\nline3\nline4\n', message='commit4', - vcs_type=backend.alias, parent=commit2) + repo1.repo_name, filename=b'file1',content=b'line1\nline2\nline3\nline4\n', + message='commit4', vcs_type=backend.alias, parent=commit2) commit4 = commit_change( - repo1.repo_name, filename='file1', - content='line1\nline2\nline3\nline4\nline5\n', message='commit5', - vcs_type=backend.alias, parent=commit3) + repo1.repo_name, filename=b'file1', content=b'line1\nline2\nline3\nline4\nline5\n', + message='commit5', vcs_type=backend.alias, parent=commit3) commit_change( # commit 5 - repo1.repo_name, filename='file1', - content='line1\nline2\nline3\nline4\nline5\nline6\n', + repo1.repo_name, filename=b'file1', content=b'line1\nline2\nline3\nline4\nline5\nline6\n', message='commit6', vcs_type=backend.alias, parent=commit4) response = self.app.get( @@ -313,11 +313,11 @@ class TestCompareView(object): # commit something ! commit0 = commit_change( - repo1.repo_name, filename='file1', content='line1\n', + repo1.repo_name, filename=b'file1', content=b'line1\n', message='commit1', vcs_type=backend.alias, parent=None, newfile=True) commit1 = commit_change( - repo1.repo_name, filename='file1', content='line1\nline2\n', + repo1.repo_name, filename=b'file1', content=b'line1\nline2\n', message='commit2', vcs_type=backend.alias, parent=commit0) # fork this repo @@ -325,19 +325,19 @@ class TestCompareView(object): # now make commit3-6 commit2 = commit_change( - repo1.repo_name, filename='file1', content='line1\nline2\nline3\n', + repo1.repo_name, filename=b'file1', content=b'line1\nline2\nline3\n', message='commit3', vcs_type=backend.alias, parent=commit1) commit3 = commit_change( - repo1.repo_name, filename='file1', - content='line1\nline2\nline3\nline4\n', message='commit4', + repo1.repo_name, filename=b'file1', + content=b'line1\nline2\nline3\nline4\n', message='commit4', vcs_type=backend.alias, parent=commit2) commit4 = commit_change( - repo1.repo_name, filename='file1', - content='line1\nline2\nline3\nline4\nline5\n', message='commit5', + repo1.repo_name, filename=b'file1', + content=b'line1\nline2\nline3\nline4\nline5\n', message='commit5', vcs_type=backend.alias, parent=commit3) commit5 = commit_change( - repo1.repo_name, filename='file1', - content='line1\nline2\nline3\nline4\nline5\nline6\n', + repo1.repo_name, filename=b'file1', + content=b'line1\nline2\nline3\nline4\nline5\nline6\n', message='commit6', vcs_type=backend.alias, parent=commit4) response = self.app.get( @@ -399,8 +399,8 @@ class TestCompareView(object): r1_name = repo1.repo_name commit0 = commit_change( - repo=r1_name, filename='file1', - content='line1', message='commit1', vcs_type=backend.alias, + repo=r1_name, filename=b'file1', + content=b'line1', message='commit1', vcs_type=backend.alias, newfile=True) assert repo1.scm_instance().commit_ids == [commit0.raw_id] @@ -412,20 +412,20 @@ class TestCompareView(object): r2_name = repo2.repo_name commit1 = commit_change( - repo=r2_name, filename='file1-fork', - content='file1-line1-from-fork', message='commit1-fork', + repo=r2_name, filename=b'file1-fork', + content=b'file1-line1-from-fork', message='commit1-fork', vcs_type=backend.alias, parent=repo2.scm_instance()[-1], newfile=True) commit2 = commit_change( - repo=r2_name, filename='file2-fork', - content='file2-line1-from-fork', message='commit2-fork', + repo=r2_name, filename=b'file2-fork', + content=b'file2-line1-from-fork', message='commit2-fork', vcs_type=backend.alias, parent=commit1, newfile=True) commit_change( # commit 3 - repo=r2_name, filename='file3-fork', - content='file3-line1-from-fork', message='commit3-fork', + repo=r2_name, filename=b'file3-fork', + content=b'file3-line1-from-fork', message='commit3-fork', vcs_type=backend.alias, parent=commit2, newfile=True) # compare ! @@ -446,8 +446,8 @@ class TestCompareView(object): response.mustcontain('No commits in this compare') commit0 = commit_change( - repo=r1_name, filename='file2', - content='line1-added-after-fork', message='commit2-parent', + repo=r1_name, filename=b'file2', + content=b'line1-added-after-fork', message='commit2-parent', vcs_type=backend.alias, parent=None, newfile=True) # compare ! @@ -487,11 +487,10 @@ class TestCompareView(object): def test_errors_when_comparing_unknown_source_repo(self, backend): repo = backend.repo - badrepo = 'badrepo' - response = self.app.get( + self.app.get( route_path('repo_compare', - repo_name=badrepo, + repo_name='badrepo', source_ref_type="rev", source_ref='tip', target_ref_type="rev", target_ref='tip', params=dict(merge='1', target_repo=repo.repo_name) diff --git a/rhodecode/apps/repository/tests/test_repo_compare_local.py b/rhodecode/apps/repository/tests/test_repo_compare_local.py --- a/rhodecode/apps/repository/tests/test_repo_compare_local.py +++ b/rhodecode/apps/repository/tests/test_repo_compare_local.py @@ -23,7 +23,9 @@ from .test_repo_compare import ComparePa def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repo_compare_select': '/{repo_name}/compare', @@ -55,8 +57,8 @@ class TestCompareView(object): # outgoing commits between tags commit_indexes = { - 'git': [113] + range(115, 121), - 'hg': [112] + range(115, 121), + 'git': [113] + list(range(115, 121)), + 'hg': [112] + list(range(115, 121)), } repo = backend.repo commits = (repo.get_commit(commit_idx=idx) diff --git a/rhodecode/apps/repository/tests/test_repo_compare_on_single_file.py b/rhodecode/apps/repository/tests/test_repo_compare_on_single_file.py --- a/rhodecode/apps/repository/tests/test_repo_compare_on_single_file.py +++ b/rhodecode/apps/repository/tests/test_repo_compare_on_single_file.py @@ -29,7 +29,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repo_compare_select': '/{repo_name}/compare', @@ -151,9 +153,9 @@ class TestSideBySideDiff(object): @pytest.mark.xfail(reason='GIT does not handle empty commit compare correct (missing 1 commit)') def test_diff_side_by_side_from_0_commit(self, app, backend, backend_stub): - f_path = 'test_sidebyside_file.py' - commit1_content = 'content-25d7e49c18b159446c\n' - commit2_content = 'content-603d6c72c46d953420\n' + f_path = b'test_sidebyside_file.py' + commit1_content = b'content-25d7e49c18b159446c\n' + commit2_content = b'content-603d6c72c46d953420\n' repo = backend.create_repo() commit1 = commit_change( @@ -185,9 +187,9 @@ class TestSideBySideDiff(object): @pytest.mark.xfail(reason='GIT does not handle empty commit compare correct (missing 1 commit)') def test_diff_side_by_side_from_0_commit_with_file_filter(self, app, backend, backend_stub): - f_path = 'test_sidebyside_file.py' - commit1_content = 'content-25d7e49c18b159446c\n' - commit2_content = 'content-603d6c72c46d953420\n' + f_path = b'test_sidebyside_file.py' + commit1_content = b'content-25d7e49c18b159446c\n' + commit2_content = b'content-603d6c72c46d953420\n' repo = backend.create_repo() commit1 = commit_change( @@ -222,7 +224,7 @@ class TestSideBySideDiff(object): {'message': 'First commit'}, {'message': 'Second commit'}, {'message': 'Commit with binary', - 'added': [nodes.FileNode('file.empty', content='')]}, + 'added': [nodes.FileNode(b'file.empty', content=b'')]}, ] f_path = 'file.empty' repo = backend.create_repo(commits=commits) diff --git a/rhodecode/apps/repository/tests/test_repo_feed.py b/rhodecode/apps/repository/tests/test_repo_feed.py --- a/rhodecode/apps/repository/tests/test_repo_feed.py +++ b/rhodecode/apps/repository/tests/test_repo_feed.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -24,7 +23,9 @@ from rhodecode.tests import TestControll def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'rss_feed_home': '/{repo_name}/feed-rss', diff --git a/rhodecode/apps/repository/tests/test_repo_files.py b/rhodecode/apps/repository/tests/test_repo_files.py --- a/rhodecode/apps/repository/tests/test_repo_files.py +++ b/rhodecode/apps/repository/tests/test_repo_files.py @@ -23,10 +23,11 @@ import mock import pytest from rhodecode.apps.repository.tests.test_repo_compare import ComparePage -from rhodecode.apps.repository.views.repo_files import RepoFilesView +from rhodecode.apps.repository.views.repo_files import RepoFilesView, get_archive_name, get_path_sha from rhodecode.lib import helpers as h from collections import OrderedDict from rhodecode.lib.ext_json import json +from rhodecode.lib.str_utils import safe_str from rhodecode.lib.vcs import nodes from rhodecode.lib.vcs.conf import settings @@ -46,7 +47,9 @@ def get_node_history(backend_type): def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repo_summary': '/{repo_name}', @@ -506,7 +509,7 @@ class TestRawFileHandling(object): def test_raw_svg_should_not_be_rendered(self, backend): backend.create_repo() - backend.ensure_file("xss.svg") + backend.ensure_file(b"xss.svg") response = self.app.get( route_path('repo_file_raw', repo_name=backend.repo_name, @@ -523,10 +526,10 @@ class TestRepositoryArchival(object): backend.enable_downloads() commit = backend.repo.get_commit(commit_idx=173) for a_type, content_type, extension in settings.ARCHIVE_SPECS: + path_sha = get_path_sha('/') + filename = get_archive_name(backend.repo_name, commit_sha=commit.short_id, ext=extension, path_sha=path_sha) - short = commit.short_id + extension fname = commit.raw_id + extension - filename = '%s-%s' % (backend.repo_name, short) response = self.app.get( route_path('repo_archivefile', repo_name=backend.repo_name, @@ -545,10 +548,10 @@ class TestRepositoryArchival(object): backend.enable_downloads() commit = backend.repo.get_commit(commit_idx=173) for a_type, content_type, extension in settings.ARCHIVE_SPECS: + path_sha = get_path_sha('/') + filename = get_archive_name(backend.repo_name, commit_sha=commit.short_id, ext=extension, path_sha=path_sha, with_hash=False) - short = 'plain' + extension fname = commit.raw_id + extension - filename = '%s-%s' % (backend.repo_name, short) response = self.app.get( route_path('repo_archivefile', repo_name=backend.repo_name, @@ -622,7 +625,7 @@ class TestFilesDiff(object): commits = [ {'message': 'First commit'}, {'message': 'Commit with binary', - 'added': [nodes.FileNode('file.bin', content='\0BINARY\0')]}, + 'added': [nodes.FileNode(b'file.bin', content='\0BINARY\0')]}, ] repo = backend.create_repo(commits=commits) @@ -899,7 +902,7 @@ class TestModifyFilesWithWebInterface(ob def test_edit_file_view_not_on_branch(self, backend): repo = backend.create_repo() - backend.ensure_file("vcs/nodes.py") + backend.ensure_file(b"vcs/nodes.py") response = self.app.get( route_path('repo_files_edit_file', @@ -912,7 +915,7 @@ class TestModifyFilesWithWebInterface(ob def test_edit_file_view_commit_changes(self, backend, csrf_token): repo = backend.create_repo() - backend.ensure_file("vcs/nodes.py", content="print 'hello'") + backend.ensure_file(b"vcs/nodes.py", content=b"print 'hello'") response = self.app.post( route_path('repo_files_update_file', @@ -934,7 +937,7 @@ class TestModifyFilesWithWebInterface(ob def test_edit_file_view_commit_changes_default_message(self, backend, csrf_token): repo = backend.create_repo() - backend.ensure_file("vcs/nodes.py", content="print 'hello'") + backend.ensure_file(b"vcs/nodes.py", content=b"print 'hello'") commit_id = ( backend.default_branch_name or @@ -967,7 +970,7 @@ class TestModifyFilesWithWebInterface(ob def test_delete_file_view_not_on_branch(self, backend): repo = backend.create_repo() - backend.ensure_file('vcs/nodes.py') + backend.ensure_file(b'vcs/nodes.py') response = self.app.get( route_path('repo_files_remove_file', @@ -980,7 +983,7 @@ class TestModifyFilesWithWebInterface(ob def test_delete_file_view_commit_changes(self, backend, csrf_token): repo = backend.create_repo() - backend.ensure_file("vcs/nodes.py") + backend.ensure_file(b"vcs/nodes.py") response = self.app.post( route_path('repo_files_delete_file', @@ -1015,12 +1018,18 @@ class TestFilesViewOtherCases(object): 'repo_files_add_file', repo_name=repo.repo_name, commit_id=0, f_path='') + add_new = f'add a new file' + + repo_file_upload_url = route_path( + 'repo_files_upload_file', + repo_name=repo.repo_name, + commit_id=0, f_path='') + upload_new = f'upload a new file' assert_session_flash( response, - 'There are no files yet. Click here to add a new file.' - .format(repo_file_add_url)) + 'There are no files yet. Click here to %s or %s.' % (add_new, upload_new) + ) def test_access_empty_repo_redirect_to_summary_with_alert_no_write_perms( self, backend_stub, autologin_regular_user): @@ -1041,12 +1050,12 @@ class TestFilesViewOtherCases(object): assert_session_flash(response, no_=repo_file_add_url) @pytest.mark.parametrize('file_node', [ - 'archive/file.zip', - 'diff/my-file.txt', - 'render.py', - 'render', - 'remove_file', - 'remove_file/to-delete.txt', + b'archive/file.zip', + b'diff/my-file.txt', + b'render.py', + b'render', + b'remove_file', + b'remove_file/to-delete.txt', ]) def test_file_names_equal_to_routes_parts(self, backend, file_node): backend.create_repo() @@ -1055,7 +1064,7 @@ class TestFilesViewOtherCases(object): self.app.get( route_path('repo_files', repo_name=backend.repo_name, - commit_id='tip', f_path=file_node), + commit_id='tip', f_path=safe_str(file_node)), status=200) diff --git a/rhodecode/apps/repository/tests/test_repo_forks.py b/rhodecode/apps/repository/tests/test_repo_forks.py --- a/rhodecode/apps/repository/tests/test_repo_forks.py +++ b/rhodecode/apps/repository/tests/test_repo_forks.py @@ -33,7 +33,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repo_summary': '/{repo_name}', diff --git a/rhodecode/apps/repository/tests/test_repo_issue_tracker.py b/rhodecode/apps/repository/tests/test_repo_issue_tracker.py --- a/rhodecode/apps/repository/tests/test_repo_issue_tracker.py +++ b/rhodecode/apps/repository/tests/test_repo_issue_tracker.py @@ -19,14 +19,16 @@ import pytest -from rhodecode.lib.utils2 import md5 +from rhodecode.lib.hash_utils import md5_safe from rhodecode.model.db import Repository from rhodecode.model.meta import Session from rhodecode.model.settings import SettingsModel, IssueTrackerSettingsModel def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repo_summary': '/{repo_name}', @@ -69,9 +71,9 @@ class TestRepoIssueTracker(object): self.app.post(post_url, post_data, status=302) self.settings_model = IssueTrackerSettingsModel(repo=backend.repo) settings = self.settings_model.get_repo_settings() - self.uid = md5(pattern) + self.uid = md5_safe(pattern) assert settings[self.uid]['pat'] == pattern - self.another_uid = md5(another_pattern) + self.another_uid = md5_safe(another_pattern) assert settings[self.another_uid]['pat'] == another_pattern # test pattern @@ -95,7 +97,7 @@ class TestRepoIssueTracker(object): entry_key = 'issuetracker_pat_' pattern = 'issuetracker_pat2' old_pattern = 'issuetracker_pat' - old_uid = md5(old_pattern) + old_uid = md5_safe(old_pattern) sett = SettingsModel(repo=backend.repo).create_or_update_setting( entry_key+old_uid, old_pattern, 'unicode') @@ -114,7 +116,7 @@ class TestRepoIssueTracker(object): self.app.post(post_url, post_data, status=302) self.settings_model = IssueTrackerSettingsModel(repo=backend.repo) settings = self.settings_model.get_repo_settings() - self.uid = md5(pattern) + self.uid = md5_safe(pattern) assert settings[self.uid]['pat'] == pattern with pytest.raises(KeyError): key = settings[old_uid] @@ -129,7 +131,7 @@ class TestRepoIssueTracker(object): repo_name = repo.repo_name entry_key = 'issuetracker_pat_' pattern = 'issuetracker_pat3' - uid = md5(pattern) + uid = md5_safe(pattern) settings_util.create_repo_rhodecode_setting( repo=backend.repo, name=entry_key+uid, value=entry_key, type_='unicode', cleanup=False) diff --git a/rhodecode/apps/repository/tests/test_repo_maintainance.py b/rhodecode/apps/repository/tests/test_repo_maintainance.py --- a/rhodecode/apps/repository/tests/test_repo_maintainance.py +++ b/rhodecode/apps/repository/tests/test_repo_maintainance.py @@ -32,7 +32,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'edit_repo_maintenance': '/{repo_name}/settings/maintenance', diff --git a/rhodecode/apps/repository/tests/test_repo_permissions.py b/rhodecode/apps/repository/tests/test_repo_permissions.py --- a/rhodecode/apps/repository/tests/test_repo_permissions.py +++ b/rhodecode/apps/repository/tests/test_repo_permissions.py @@ -23,7 +23,9 @@ from rhodecode.tests.utils import permis def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'edit_repo_perms': '/{repo_name}/settings/permissions' diff --git a/rhodecode/apps/repository/tests/test_repo_pullrequests.py b/rhodecode/apps/repository/tests/test_repo_pullrequests.py --- a/rhodecode/apps/repository/tests/test_repo_pullrequests.py +++ b/rhodecode/apps/repository/tests/test_repo_pullrequests.py @@ -36,7 +36,9 @@ from rhodecode.tests import ( def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repo_changelog': '/{repo_name}/changelog', @@ -119,21 +121,21 @@ class TestPullrequestsView(object): def test_show_versions_of_pr(self, backend, csrf_token): commits = [ {'message': 'initial-commit', - 'added': [FileNode('test-file.txt', 'LINE1\n')]}, + 'added': [FileNode(b'test-file.txt', b'LINE1\n')]}, {'message': 'commit-1', - 'changed': [FileNode('test-file.txt', 'LINE1\nLINE2\n')]}, + 'changed': [FileNode(b'test-file.txt', b'LINE1\nLINE2\n')]}, # Above is the initial version of PR that changes a single line # from now on we'll add 3x commit adding a nother line on each step {'message': 'commit-2', - 'changed': [FileNode('test-file.txt', 'LINE1\nLINE2\nLINE3\n')]}, + 'changed': [FileNode(b'test-file.txt', b'LINE1\nLINE2\nLINE3\n')]}, {'message': 'commit-3', - 'changed': [FileNode('test-file.txt', 'LINE1\nLINE2\nLINE3\nLINE4\n')]}, + 'changed': [FileNode(b'test-file.txt', b'LINE1\nLINE2\nLINE3\nLINE4\n')]}, {'message': 'commit-4', - 'changed': [FileNode('test-file.txt', 'LINE1\nLINE2\nLINE3\nLINE4\nLINE5\n')]}, + 'changed': [FileNode(b'test-file.txt', b'LINE1\nLINE2\nLINE3\nLINE4\nLINE5\n')]}, ] commit_ids = backend.create_master_repo(commits) @@ -404,8 +406,8 @@ class TestPullrequestsView(object): response = self.app.post( route_path('pullrequest_update', - repo_name=pull_request.target_repo.repo_name, - pull_request_id=pull_request_id), + repo_name=pull_request.target_repo.repo_name, + pull_request_id=pull_request_id), params={ 'edit_pull_request': 'true', 'title': 'New title', @@ -413,21 +415,21 @@ class TestPullrequestsView(object): 'csrf_token': csrf_token}) assert_session_flash( - response, u'Pull request title & description updated.', + response, 'Pull request title & description updated.', category='success') pull_request = PullRequest.get(pull_request_id) assert pull_request.title == 'New title' assert pull_request.description == 'New description' - def test_edit_title_description(self, pr_util, csrf_token): + def test_edit_title_description_special(self, pr_util, csrf_token): pull_request = pr_util.create_pull_request() pull_request_id = pull_request.pull_request_id response = self.app.post( route_path('pullrequest_update', - repo_name=pull_request.target_repo.repo_name, - pull_request_id=pull_request_id), + repo_name=pull_request.target_repo.repo_name, + pull_request_id=pull_request_id), params={ 'edit_pull_request': 'true', 'title': 'New title {} {2} {foo}', @@ -435,7 +437,7 @@ class TestPullrequestsView(object): 'csrf_token': csrf_token}) assert_session_flash( - response, u'Pull request title & description updated.', + response, 'Pull request title & description updated.', category='success') pull_request = PullRequest.get(pull_request_id) @@ -456,7 +458,7 @@ class TestPullrequestsView(object): 'description': 'New description', 'csrf_token': csrf_token}, status=200) assert_session_flash( - response, u'Cannot update closed pull requests.', + response, 'Cannot update closed pull requests.', category='error') def test_update_invalid_source_reference(self, pr_util, csrf_token): @@ -483,7 +485,7 @@ class TestPullrequestsView(object): from rhodecode.lib.vcs.backends.base import MergeFailureReason pull_request = pr_util.create_pull_request( approved=True, mergeable=True) - unicode_reference = u'branch:invalid-branch:invalid-commit-id' + unicode_reference = 'branch:invalid-branch:invalid-commit-id' pull_request.target_ref = unicode_reference Session().add(pull_request) Session().commit() @@ -687,7 +689,7 @@ class TestPullrequestsView(object): ChangesetComment.comment_id == comment_id).first().text assert test_text == text_form_db - def test_comment_and_comment_edit(self, pr_util, csrf_token, xhr_header): + def test_comment_and_comment_edit_special(self, pr_util, csrf_token, xhr_header): pull_request = pr_util.create_pull_request() target_scm = pull_request.target_repo.scm_instance() target_scm_name = target_scm.name @@ -867,13 +869,12 @@ class TestPullrequestsView(object): # notifications properly with the new PR commits = [ {'message': 'ancestor', - 'added': [FileNode('file_A', content='content_of_ancestor')]}, + 'added': [FileNode(b'file_A', content=b'content_of_ancestor')]}, {'message': 'change', - 'added': [FileNode('file_a', content='content_of_change')]}, + 'added': [FileNode(b'file_a', content=b'content_of_change')]}, {'message': 'change-child'}, {'message': 'ancestor-child', 'parents': ['ancestor'], - 'added': [ - FileNode('file_B', content='content_of_ancestor_child')]}, + 'added': [ FileNode(b'file_B', content=b'content_of_ancestor_child')]}, {'message': 'ancestor-child-2'}, ] commit_ids = backend.create_master_repo(commits) @@ -935,13 +936,13 @@ class TestPullrequestsView(object): def test_create_pull_request_stores_ancestor_commit_id(self, backend, csrf_token): commits = [ {'message': 'ancestor', - 'added': [FileNode('file_A', content='content_of_ancestor')]}, + 'added': [FileNode(b'file_A', content=b'content_of_ancestor')]}, {'message': 'change', - 'added': [FileNode('file_a', content='content_of_change')]}, + 'added': [FileNode(b'file_a', content=b'content_of_change')]}, {'message': 'change-child'}, {'message': 'ancestor-child', 'parents': ['ancestor'], 'added': [ - FileNode('file_B', content='content_of_ancestor_child')]}, + FileNode(b'file_B', content=b'content_of_ancestor_child')]}, {'message': 'ancestor-child-2'}, ] commit_ids = backend.create_master_repo(commits) @@ -1021,9 +1022,9 @@ class TestPullrequestsView(object): actions = [log.action for log in user_logs] pr_commit_ids = PullRequestModel()._get_commit_ids(pull_request) expected_actions = [ - u'repo.pull_request.close', - u'repo.pull_request.merge', - u'repo.pull_request.comment.create' + 'repo.pull_request.close', + 'repo.pull_request.merge', + 'repo.pull_request.comment.create' ] assert actions == expected_actions @@ -1046,8 +1047,8 @@ class TestPullrequestsView(object): response = self.app.post( route_path('pullrequest_merge', - repo_name=pull_request.target_repo.scm_instance().name, - pull_request_id=pull_request.pull_request_id), + repo_name=pull_request.target_repo.scm_instance().name, + pull_request_id=pull_request.pull_request_id), params={'csrf_token': csrf_token}).follow() assert response.status_int == 200 @@ -1121,8 +1122,8 @@ class TestPullrequestsView(object): branch=backend.default_branch_name, commit_id=commit_ids['ancestor']) pull_request.revisions = [commit_ids['change']] - pull_request.title = u"Test" - pull_request.description = u"Description" + pull_request.title = "Test" + pull_request.description = "Description" pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN) pull_request.pull_request_state = PullRequest.STATE_CREATED Session().add(pull_request) @@ -1175,8 +1176,8 @@ class TestPullrequestsView(object): branch=backend.default_branch_name, commit_id=commit_ids['ancestor']) pull_request.revisions = [commit_ids['change']] - pull_request.title = u"Test" - pull_request.description = u"Description" + pull_request.title = "Test" + pull_request.description = "Description" pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN) pull_request.pull_request_state = PullRequest.STATE_CREATED @@ -1242,8 +1243,8 @@ class TestPullrequestsView(object): commit_ids['feat-commit-1'], commit_ids['feat-commit-2'] ] - pull_request.title = u"Test" - pull_request.description = u"Description" + pull_request.title = "Test" + pull_request.description = "Description" pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN) pull_request.pull_request_state = PullRequest.STATE_CREATED Session().add(pull_request) @@ -1292,8 +1293,8 @@ class TestPullrequestsView(object): pull_request.target_ref = 'branch:{branch}:{commit_id}'.format( branch=backend.default_branch_name, commit_id=commit_ids['ancestor']) pull_request.revisions = [commit_ids['change']] - pull_request.title = u"Test" - pull_request.description = u"Description" + pull_request.title = "Test" + pull_request.description = "Description" pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN) pull_request.pull_request_state = PullRequest.STATE_CREATED Session().add(pull_request) @@ -1340,8 +1341,8 @@ class TestPullrequestsView(object): pull_request.target_ref = 'branch:{branch}:{commit_id}'.format( branch=backend_git.default_branch_name, commit_id=commit_ids['old-feature']) pull_request.revisions = [commit_ids['new-feature']] - pull_request.title = u"Test" - pull_request.description = u"Description" + pull_request.title = "Test" + pull_request.description = "Description" pull_request.author = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN) pull_request.pull_request_state = PullRequest.STATE_CREATED Session().add(pull_request) diff --git a/rhodecode/apps/repository/tests/test_repo_settings.py b/rhodecode/apps/repository/tests/test_repo_settings.py --- a/rhodecode/apps/repository/tests/test_repo_settings.py +++ b/rhodecode/apps/repository/tests/test_repo_settings.py @@ -32,7 +32,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'edit_repo': '/{repo_name}/settings', diff --git a/rhodecode/apps/repository/tests/test_repo_settings_advanced.py b/rhodecode/apps/repository/tests/test_repo_settings_advanced.py --- a/rhodecode/apps/repository/tests/test_repo_settings_advanced.py +++ b/rhodecode/apps/repository/tests/test_repo_settings_advanced.py @@ -19,7 +19,7 @@ import pytest -from rhodecode.lib.utils2 import safe_unicode, safe_str +from rhodecode.lib.str_utils import safe_str from rhodecode.model.db import Repository from rhodecode.model.repo import RepoModel from rhodecode.tests import ( @@ -31,7 +31,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repo_summary_explicit': '/{repo_name}/summary', diff --git a/rhodecode/apps/repository/tests/test_repo_summary.py b/rhodecode/apps/repository/tests/test_repo_summary.py --- a/rhodecode/apps/repository/tests/test_repo_summary.py +++ b/rhodecode/apps/repository/tests/test_repo_summary.py @@ -40,7 +40,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repo_summary': '/{repo_name}', @@ -276,8 +278,8 @@ class TestRepoLocation(object): response = self.app.get( route_path('repo_summary', repo_name=safe_str(repo_name)), status=302) - msg = 'The repository `%s` cannot be loaded in filesystem. ' \ - 'Please check if it exist, or is not damaged.' % repo_name + msg = f'The repository `{repo_name}` cannot be loaded in filesystem. ' \ + f'Please check if it exist, or is not damaged.' assert_session_flash(response, msg) @pytest.mark.parametrize("suffix", [u'', u'ąęł'], ids=['', 'non-ascii']) diff --git a/rhodecode/apps/repository/tests/test_repo_tags.py b/rhodecode/apps/repository/tests/test_repo_tags.py --- a/rhodecode/apps/repository/tests/test_repo_tags.py +++ b/rhodecode/apps/repository/tests/test_repo_tags.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -23,7 +22,9 @@ from rhodecode.model.db import Repositor def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'tags_home': '/{repo_name}/tags', diff --git a/rhodecode/apps/repository/tests/test_repo_vcs_settings.py b/rhodecode/apps/repository/tests/test_repo_vcs_settings.py --- a/rhodecode/apps/repository/tests/test_repo_vcs_settings.py +++ b/rhodecode/apps/repository/tests/test_repo_vcs_settings.py @@ -37,7 +37,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'repo_summary': '/{repo_name}', diff --git a/rhodecode/apps/repository/tests/test_vcs_settings.py b/rhodecode/apps/repository/tests/test_vcs_settings.py --- a/rhodecode/apps/repository/tests/test_vcs_settings.py +++ b/rhodecode/apps/repository/tests/test_vcs_settings.py @@ -27,7 +27,9 @@ from rhodecode.tests.utils import Assert def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error base_url = { 'edit_repo': '/{repo_name}/settings', diff --git a/rhodecode/apps/search/tests/test_search.py b/rhodecode/apps/search/tests/test_search.py --- a/rhodecode/apps/search/tests/test_search.py +++ b/rhodecode/apps/search/tests/test_search.py @@ -179,6 +179,7 @@ class TestSearchController(TestControlle def test_filters_are_not_applied_for_admin_user(self): self.log_user() with mock.patch('whoosh.searching.Searcher.search') as search_mock: + self.app.get(route_path('search'), {'q': 'test query', 'type': 'commit'}) assert search_mock.call_count == 1 diff --git a/rhodecode/apps/ssh_support/tests/conftest.py b/rhodecode/apps/ssh_support/tests/conftest.py --- a/rhodecode/apps/ssh_support/tests/conftest.py +++ b/rhodecode/apps/ssh_support/tests/conftest.py @@ -35,7 +35,7 @@ def dummy_conf_file(tmpdir): conf.set('app:main', 'ssh.executable.svn', '/usr/bin/svnserve') f_path = os.path.join(str(tmpdir), 'ssh_wrapper_test.ini') - with open(f_path, 'wb') as f: + with open(f_path, 'wt') as f: conf.write(f) return os.path.join(f_path) diff --git a/rhodecode/apps/ssh_support/tests/test_server_git.py b/rhodecode/apps/ssh_support/tests/test_server_git.py --- a/rhodecode/apps/ssh_support/tests/test_server_git.py +++ b/rhodecode/apps/ssh_support/tests/test_server_git.py @@ -18,7 +18,6 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import json import os import mock @@ -26,7 +25,7 @@ import pytest from rhodecode.apps.ssh_support.lib.backends.git import GitServer from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dummy_user - +from rhodecode.lib.ext_json import json class GitServerCreator(object): root = '/tmp/repo/path/' diff --git a/rhodecode/apps/svn_support/tests/test_mod_dav_svn_config.py b/rhodecode/apps/svn_support/tests/test_mod_dav_svn_config.py --- a/rhodecode/apps/svn_support/tests/test_mod_dav_svn_config.py +++ b/rhodecode/apps/svn_support/tests/test_mod_dav_svn_config.py @@ -39,7 +39,7 @@ class TestModDavSvnConfig(object): def get_repo_group_mocks(cls, count=1): repo_groups = [] for num in range(0, count): - full_path = u'/path/to/RepöGröúp-°µ {}'.format(num) + full_path = f'/path/to/RepöGröúp-°µ {num}' repo_group_mock = mock.MagicMock() repo_group_mock.full_path = full_path repo_group_mock.full_path_splitted = full_path.split('/') @@ -78,7 +78,7 @@ class TestModDavSvnConfig(object): def test_render_mod_dav_svn_config_with_alternative_template(self, tmpdir): repo_groups = self.get_repo_group_mocks(count=10) test_file_path = os.path.join(str(tmpdir), 'example.mako') - with open(test_file_path, 'wb') as f: + with open(test_file_path, 'wt') as f: f.write('TEST_EXAMPLE\n') generated_config = utils._render_mod_dav_svn_config( @@ -107,11 +107,11 @@ class TestModDavSvnConfig(object): # Assert that correct configuration directive is present. if list_parent_path: - assert not re.search('SVNListParentPath\s+Off', generated_config) - assert re.search('SVNListParentPath\s+On', generated_config) + assert not re.search(r'SVNListParentPath\s+Off', generated_config) + assert re.search(r'SVNListParentPath\s+On', generated_config) else: - assert re.search('SVNListParentPath\s+Off', generated_config) - assert not re.search('SVNListParentPath\s+On', generated_config) + assert re.search(r'SVNListParentPath\s+Off', generated_config) + assert not re.search(r'SVNListParentPath\s+On', generated_config) if use_ssl: assert 'RequestHeader edit Destination ^https: http: early' \ diff --git a/rhodecode/apps/user_group/tests/test_user_groups.py b/rhodecode/apps/user_group/tests/test_user_groups.py --- a/rhodecode/apps/user_group/tests/test_user_groups.py +++ b/rhodecode/apps/user_group/tests/test_user_groups.py @@ -29,7 +29,9 @@ fixture = Fixture() def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { diff --git a/rhodecode/apps/user_group/tests/test_user_groups_permissions.py b/rhodecode/apps/user_group/tests/test_user_groups_permissions.py --- a/rhodecode/apps/user_group/tests/test_user_groups_permissions.py +++ b/rhodecode/apps/user_group/tests/test_user_groups_permissions.py @@ -23,7 +23,9 @@ from rhodecode.tests.utils import permis def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { diff --git a/rhodecode/tests/__init__.py b/rhodecode/tests/__init__.py --- a/rhodecode/tests/__init__.py +++ b/rhodecode/tests/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -214,11 +213,13 @@ def assert_session_flash(response, msg=N msg = f'msg `{no_}` found in session flash.' pytest.fail(safe_str(msg)) else: + if msg not in message_text: fail_msg = f'msg `{msg}` not found in ' \ f'session flash: got `{message_text}` (type:{type(message_text)}) instead' pytest.fail(safe_str(fail_msg)) + if category: assert category == message.category diff --git a/rhodecode/tests/auth_external_test.py b/rhodecode/tests/auth_external_test.py --- a/rhodecode/tests/auth_external_test.py +++ b/rhodecode/tests/auth_external_test.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/config/__init__.py b/rhodecode/tests/config/__init__.py --- a/rhodecode/tests/config/__init__.py +++ b/rhodecode/tests/config/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/config/test_environment.py b/rhodecode/tests/config/test_environment.py --- a/rhodecode/tests/config/test_environment.py +++ b/rhodecode/tests/config/test_environment.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/config/test_routing_links.py b/rhodecode/tests/config/test_routing_links.py --- a/rhodecode/tests/config/test_routing_links.py +++ b/rhodecode/tests/config/test_routing_links.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/config/test_sanitize_settings.py b/rhodecode/tests/config/test_sanitize_settings.py --- a/rhodecode/tests/config/test_sanitize_settings.py +++ b/rhodecode/tests/config/test_sanitize_settings.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # @@ -118,7 +118,7 @@ class TestSanitizeVcsSettings(object): _string_funcs = [ ('vcs.svn.compatible_version', ''), ('vcs.hooks.protocol', 'http'), - ('vcs.hooks.host', '127.0.0.1'), + ('vcs.hooks.host', '*'), ('vcs.scm_app_implementation', 'http'), ('vcs.server', ''), ('vcs.server.protocol', 'http'), diff --git a/rhodecode/tests/config/test_utils.py b/rhodecode/tests/config/test_utils.py --- a/rhodecode/tests/config/test_utils.py +++ b/rhodecode/tests/config/test_utils.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2012-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/conftest.py b/rhodecode/tests/conftest.py new file mode 100644 --- /dev/null +++ b/rhodecode/tests/conftest.py @@ -0,0 +1,132 @@ +# Copyright (C) 2010-2020 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +""" +py.test config for test suite for making push/pull operations. + +.. important:: + + You must have git >= 1.8.5 for tests to work fine. With 68b939b git started + to redirect things to stderr instead of stdout. +""" + +import pytest +import logging + +from rhodecode.authentication import AuthenticationPluginRegistry +from rhodecode.model.db import Permission, User +from rhodecode.model.meta import Session +from rhodecode.model.settings import SettingsModel +from rhodecode.model.user import UserModel + + +log = logging.getLogger(__name__) + + +@pytest.fixture() +def enable_auth_plugins(request, baseapp, csrf_token): + """ + Return a factory object that when called, allows to control which + authentication plugins are enabled. + """ + + class AuthPluginManager(object): + + def cleanup(self): + self._enable_plugins(['egg:rhodecode-enterprise-ce#rhodecode']) + + def enable(self, plugins_list, override=None): + return self._enable_plugins(plugins_list, override) + + def _enable_plugins(self, plugins_list, override=None): + override = override or {} + params = { + 'auth_plugins': ','.join(plugins_list), + } + + # helper translate some names to others, to fix settings code + name_map = { + 'token': 'authtoken' + } + log.debug('enable_auth_plugins: enabling following auth-plugins: %s', plugins_list) + + for module in plugins_list: + plugin_name = module.partition('#')[-1] + if plugin_name in name_map: + plugin_name = name_map[plugin_name] + enabled_plugin = f'auth_{plugin_name}_enabled' + cache_ttl = f'auth_{plugin_name}_cache_ttl' + + # default params that are needed for each plugin, + # `enabled` and `cache_ttl` + params.update({ + enabled_plugin: True, + cache_ttl: 0 + }) + if override.get: + params.update(override.get(module, {})) + + validated_params = params + + for k, v in validated_params.items(): + setting = SettingsModel().create_or_update_setting(k, v) + Session().add(setting) + Session().commit() + + AuthenticationPluginRegistry.invalidate_auth_plugins_cache(hard=True) + + enabled_plugins = SettingsModel().get_auth_plugins() + assert plugins_list == enabled_plugins + + enabler = AuthPluginManager() + request.addfinalizer(enabler.cleanup) + + return enabler + + +@pytest.fixture() +def test_user_factory(request, baseapp): + + def user_factory(username='test_user', password='qweqwe', first_name='John', last_name='Testing', **kwargs): + usr = UserModel().create_or_update( + username=username, + password=password, + email=f'{username}@rhodecode.org', + firstname=first_name, lastname=last_name) + Session().commit() + + for k, v in kwargs.items(): + setattr(usr, k, v) + Session().add(usr) + + assert User.get_by_username(username) == usr + + @request.addfinalizer + def cleanup(): + if UserModel().get_user(usr.user_id) is None: + return + + perm = Permission.query().all() + for p in perm: + UserModel().revoke_perm(usr, p) + + UserModel().delete(usr.user_id) + Session().commit() + return usr + + return user_factory diff --git a/rhodecode/tests/conftest_common.py b/rhodecode/tests/conftest_common.py new file mode 100644 --- /dev/null +++ b/rhodecode/tests/conftest_common.py @@ -0,0 +1,167 @@ +# Copyright (C) 2010-2020 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import pytest +from rhodecode.lib import ext_json + + +def get_backends_from_metafunc(metafunc): + requested_backends = set(metafunc.config.getoption('--backends')) + backend_mark = metafunc.definition.get_closest_marker('backends') + if backend_mark: + # Supported backends by this test function, created from + # pytest.mark.backends + backends = backend_mark.args + elif hasattr(metafunc.cls, 'backend_alias'): + # Support class attribute "backend_alias", this is mainly + # for legacy reasons for tests not yet using pytest.mark.backends + backends = [metafunc.cls.backend_alias] + else: + backends = metafunc.config.getoption('--backends') + return requested_backends.intersection(backends) + + +def pytest_addoption(parser): + + def _parse_json(value): + return ext_json.str_json(value) if value else None + + def _split_comma(value): + return value.split(',') + + parser.addoption( + '--keep-tmp-path', action='store_true', + help="Keep the test temporary directories") + + parser.addoption( + '--backends', action='store', type=_split_comma, + default=['git', 'hg', 'svn'], + help="Select which backends to test for backend specific tests.") + parser.addoption( + '--dbs', action='store', type=_split_comma, + default=['sqlite'], + help="Select which database to test for database specific tests. " + "Possible options are sqlite,postgres,mysql") + parser.addoption( + '--appenlight', '--ae', action='store_true', + help="Track statistics in appenlight.") + parser.addoption( + '--appenlight-api-key', '--ae-key', + help="API key for Appenlight.") + parser.addoption( + '--appenlight-url', '--ae-url', + default="https://ae.rhodecode.com", + help="Appenlight service URL, defaults to https://ae.rhodecode.com") + parser.addoption( + '--sqlite-connection-string', action='store', + default='', help="Connection string for the dbs tests with SQLite") + parser.addoption( + '--postgres-connection-string', action='store', + default='', help="Connection string for the dbs tests with Postgres") + parser.addoption( + '--mysql-connection-string', action='store', + default='', help="Connection string for the dbs tests with MySQL") + parser.addoption( + '--repeat', type=int, default=100, + help="Number of repetitions in performance tests.") + + parser.addoption( + '--test-loglevel', dest='test_loglevel', + help="Set default Logging level for tests, critical(default), error, warn , info, debug") + group = parser.getgroup('pylons') + group.addoption( + '--with-pylons', dest='pyramid_config', + help="Set up a Pylons environment with the specified config file.") + group.addoption( + '--ini-config-override', action='store', type=_parse_json, + default=None, dest='pyramid_config_override', help=( + "Overrides the .ini file settings. Should be specified in JSON" + " format, e.g. '{\"section\": {\"parameter\": \"value\", ...}}'" + ) + ) + parser.addini( + 'pyramid_config', + "Set up a Pyramid environment with the specified config file.") + + vcsgroup = parser.getgroup('vcs') + vcsgroup.addoption( + '--without-vcsserver', dest='with_vcsserver', action='store_false', + help="Do not start the VCSServer in a background process.") + vcsgroup.addoption( + '--with-vcsserver-http', dest='vcsserver_config_http', + help="Start the HTTP VCSServer with the specified config file.") + vcsgroup.addoption( + '--vcsserver-protocol', dest='vcsserver_protocol', + help="Start the VCSServer with HTTP protocol support.") + vcsgroup.addoption( + '--vcsserver-config-override', action='store', type=_parse_json, + default=None, dest='vcsserver_config_override', help=( + "Overrides the .ini file settings for the VCSServer. " + "Should be specified in JSON " + "format, e.g. '{\"section\": {\"parameter\": \"value\", ...}}'" + ) + ) + vcsgroup.addoption( + '--vcsserver-port', action='store', type=int, + default=None, help=( + "Allows to set the port of the vcsserver. Useful when testing " + "against an already running server and random ports cause " + "trouble.")) + parser.addini( + 'vcsserver_config_http', + "Start the HTTP VCSServer with the specified config file.") + parser.addini( + 'vcsserver_protocol', + "Start the VCSServer with HTTP protocol support.") + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + """ + Adding the remote traceback if the exception has this information. + + VCSServer attaches this information as the attribute `_vcs_server_traceback` + to the exception instance. + """ + outcome = yield + report = outcome.get_result() + + if call.excinfo: + exc = call.excinfo.value + vcsserver_traceback = getattr(exc, '_vcs_server_traceback', None) + + if vcsserver_traceback and report.outcome == 'failed': + section = f'VCSServer remote traceback {report.when}' + report.sections.append((section, vcsserver_traceback)) + + +def pytest_generate_tests(metafunc): + + # Support test generation based on --backend parameter + if 'backend_alias' in metafunc.fixturenames: + backends = get_backends_from_metafunc(metafunc) + scope = None + if not backends: + pytest.skip("Not enabled for any of selected backends") + + metafunc.parametrize('backend_alias', backends, scope=scope) + + backend_mark = metafunc.definition.get_closest_marker('backends') + if backend_mark: + backends = get_backends_from_metafunc(metafunc) + if not backends: + pytest.skip("Not enabled for any of selected backends") diff --git a/rhodecode/tests/database/conftest.py b/rhodecode/tests/database/conftest.py --- a/rhodecode/tests/database/conftest.py +++ b/rhodecode/tests/database/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -20,13 +19,13 @@ from subprocess import Popen, PIPE import os -import shutil import sys import tempfile import pytest from sqlalchemy.engine import url +from rhodecode.lib.str_utils import safe_str, safe_bytes from rhodecode.tests.fixture import TestINI @@ -145,17 +144,19 @@ class DBBackend(object): _env.update(env) self.p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, env=_env) self.stdout, self.stderr = self.p.communicate() - sys.stdout.write('COMMAND:'+command+'\n') - sys.stdout.write(self.stdout) + stdout_str = safe_str(self.stdout) + sys.stdout.write(f'COMMAND:{command}\n') + sys.stdout.write(stdout_str) return self.stdout, self.stderr def assert_returncode_success(self): + from rich import print as pprint if not self.p.returncode == 0: - print(self.stderr) - raise AssertionError('non 0 retcode:{}'.format(self.p.returncode)) + pprint(safe_str(self.stderr)) + raise AssertionError(f'non 0 retcode:{self.p.returncode}') def assert_correct_output(self, stdout, version): - assert 'UPGRADE FOR STEP {} COMPLETED'.format(version) in stdout + assert b'UPGRADE FOR STEP %b COMPLETED' % safe_bytes(version) in stdout def setup_rhodecode_db(self, ini_params=None, env=None): if not ini_params: @@ -233,11 +234,11 @@ class SQLiteDBBackend(DBBackend): def import_dump(self, dumpname): dump = os.path.join(self.fixture_store, dumpname) target = os.path.join(self._basetemp, '{0.db_name}.sqlite'.format(self)) - return self.execute('cp -v {} {}'.format(dump, target)) + return self.execute(f'cp -v {dump} {target}') def teardown_db(self): - return self.execute("rm -rf {}.sqlite".format( - os.path.join(self._basetemp, self.db_name))) + target_db = os.path.join(self._basetemp, self.db_name) + return self.execute(f"rm -rf {target_db}.sqlite") class MySQLDBBackend(DBBackend): @@ -273,21 +274,15 @@ class PostgresDBBackend(DBBackend): def setup_db(self): # dump schema for tests # pg_dump -U postgres -h localhost $TEST_DB_NAME - self._db_url = [{'app:main': { - 'sqlalchemy.db1.url': - self.connection_string}}] - return self.execute("PGPASSWORD={} psql -U {} -h localhost " - "-c 'create database '{}';'".format( - self.password, self.user, self.db_name)) + self._db_url = [{'app:main': {'sqlalchemy.db1.url': self.connection_string}}] + cmd = f"PGPASSWORD={self.password} psql -U {self.user} -h localhost -c 'create database '{self.db_name}';'" + return self.execute(cmd) def teardown_db(self): - return self.execute("PGPASSWORD={} psql -U {} -h localhost " - "-c 'drop database if exists '{}';'".format( - self.password, self.user, self.db_name)) + cmd = f"PGPASSWORD={self.password} psql -U {self.user} -h localhost -c 'drop database if exists '{self.db_name}';'" + return self.execute(cmd) def import_dump(self, dumpname): dump = os.path.join(self.fixture_store, dumpname) - return self.execute( - "PGPASSWORD={} psql -U {} -h localhost -d {} -1 " - "-f {}".format( - self.password, self.user, self.db_name, dump)) + cmd = f"PGPASSWORD={self.password} psql -U {self.user} -h localhost -d {self.db_name} -1 -f {dump}" + return self.execute(cmd) diff --git a/rhodecode/tests/database/test_creation.py b/rhodecode/tests/database/test_creation.py --- a/rhodecode/tests/database/test_creation.py +++ b/rhodecode/tests/database/test_creation.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/database/test_migration.py b/rhodecode/tests/database/test_migration.py --- a/rhodecode/tests/database/test_migration.py +++ b/rhodecode/tests/database/test_migration.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/events/__init__.py b/rhodecode/tests/events/__init__.py --- a/rhodecode/tests/events/__init__.py +++ b/rhodecode/tests/events/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/events/test_pullrequest.py b/rhodecode/tests/events/test_pullrequest.py --- a/rhodecode/tests/events/test_pullrequest.py +++ b/rhodecode/tests/events/test_pullrequest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/events/test_repo.py b/rhodecode/tests/events/test_repo.py --- a/rhodecode/tests/events/test_repo.py +++ b/rhodecode/tests/events/test_repo.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/fixture.py b/rhodecode/tests/fixture.py --- a/rhodecode/tests/fixture.py +++ b/rhodecode/tests/fixture.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -148,7 +147,7 @@ class Fixture(object): plugin = self._get_plugin() plugin.create_or_update_setting('auth_restriction', auth_restriction) Session().commit() - SettingsModel().invalidate_settings_cache() + SettingsModel().invalidate_settings_cache(hard=True) def __exit__(self, exc_type, exc_val, exc_tb): @@ -156,7 +155,7 @@ class Fixture(object): plugin.create_or_update_setting( 'auth_restriction', RhodeCodeAuthPlugin.AUTH_RESTRICTION_NONE) Session().commit() - SettingsModel().invalidate_settings_cache() + SettingsModel().invalidate_settings_cache(hard=True) return context() @@ -181,14 +180,14 @@ class Fixture(object): plugin = self._get_plugin() plugin.create_or_update_setting('scope_restriction', scope_restriction) Session().commit() - SettingsModel().invalidate_settings_cache() + SettingsModel().invalidate_settings_cache(hard=True) def __exit__(self, exc_type, exc_val, exc_tb): plugin = self._get_plugin() plugin.create_or_update_setting( 'scope_restriction', RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_ALL) Session().commit() - SettingsModel().invalidate_settings_cache() + SettingsModel().invalidate_settings_cache(hard=True) return context() @@ -399,7 +398,7 @@ class Fixture(object): 'gist_type': GistModel.cls.GIST_PUBLIC, 'lifetime': -1, 'acl_level': Gist.ACL_LEVEL_PUBLIC, - 'gist_mapping': {'filename1.txt': {'content': 'hello world'},} + 'gist_mapping': {b'filename1.txt': {'content': b'hello world'},} } form_data.update(kwargs) gist = GistModel().create( @@ -420,7 +419,7 @@ class Fixture(object): Session().commit() def load_resource(self, resource_name, strip=False): - with open(os.path.join(FIXTURES, resource_name)) as f: + with open(os.path.join(FIXTURES, resource_name), 'rb') as f: source = f.read() if strip: source = source.strip() diff --git a/rhodecode/tests/fixture_mods/fixture_pyramid.py b/rhodecode/tests/fixture_mods/fixture_pyramid.py --- a/rhodecode/tests/fixture_mods/fixture_pyramid.py +++ b/rhodecode/tests/fixture_mods/fixture_pyramid.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -30,7 +29,7 @@ def vcsserver(request, vcsserver_port, v """ Session scope VCSServer. - Tests wich need the VCSServer have to rely on this fixture in order + Tests which need the VCSServer have to rely on this fixture in order to ensure it will be running. For specific needs, the fixture vcsserver_factory can be used. It allows to @@ -58,7 +57,7 @@ def vcsserver_factory(tmpdir_factory): """ def factory(request, overrides=(), vcsserver_port=None, - log_file=None): + log_file=None, workers='2'): if vcsserver_port is None: vcsserver_port = get_available_port() @@ -74,7 +73,7 @@ def vcsserver_factory(tmpdir_factory): basetemp=tmpdir_factory.getbasetemp().strpath, prefix='test_vcs_') - server = RcVCSServer(config_file, log_file) + server = RcVCSServer(config_file, log_file, workers) server.start() @request.addfinalizer @@ -100,7 +99,8 @@ def ini_config(request, tmpdir_factory, overrides = [ {'server:main': {'port': rcserver_port}}, {'app:main': { - 'vcs.server': 'localhost:%s' % vcsserver_port, + 'cache_dir': '%(here)s/rc_data', + 'vcs.server': f'localhost:{vcsserver_port}', # johbo: We will always start the VCSServer on our own based on the # fixtures of the test cases. For the test run it must always be # off in the INI file. @@ -109,7 +109,7 @@ def ini_config(request, tmpdir_factory, 'vcs.server.protocol': 'http', 'vcs.scm_app_implementation': 'http', 'vcs.hooks.protocol': 'http', - 'vcs.hooks.host': '127.0.0.1', + 'vcs.hooks.host': '*', }}, {'handler_console': { diff --git a/rhodecode/tests/fixture_mods/fixture_utils.py b/rhodecode/tests/fixture_mods/fixture_utils.py --- a/rhodecode/tests/fixture_mods/fixture_utils.py +++ b/rhodecode/tests/fixture_mods/fixture_utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -57,6 +56,7 @@ from rhodecode.model.integration import from rhodecode.integrations import integration_type_registry from rhodecode.integrations.types.base import IntegrationTypeBase from rhodecode.lib.utils import repo2db_mapper +from rhodecode.lib.str_utils import safe_bytes from rhodecode.lib.hash_utils import sha1_safe from rhodecode.lib.vcs.backends import get_backend from rhodecode.lib.vcs.nodes import FileNode @@ -540,7 +540,7 @@ class Backend(object): def create_repo( self, commits=None, number_of_commits=0, heads=None, - name_suffix=u'', bare=False, **kwargs): + name_suffix='', bare=False, **kwargs): """ Create a repository and record it for later cleanup. @@ -585,14 +585,14 @@ class Backend(object): self._cleanup_repos.append(self.repo_name) return repo - def new_repo_name(self, suffix=u''): + def new_repo_name(self, suffix=''): self.repo_name = self._next_repo_name() + suffix self._cleanup_repos.append(self.repo_name) return self.repo_name def _next_repo_name(self): return u"%s_%s" % ( - self.invalid_repo_name.sub(u'_', self._test_name), len(self._cleanup_repos)) + self.invalid_repo_name.sub('_', self._test_name), len(self._cleanup_repos)) def ensure_file(self, filename, content='Test content\n'): assert self._cleanup_repos, "Avoid writing into vcs_test repos" @@ -634,14 +634,98 @@ class Backend(object): repo.set_refs(ref_name, refs[ref_name]) -def vcsbackend_base(request, backend_alias, tests_tmp_path, baseapp, test_repo): +class VcsBackend(object): + """ + Represents the test configuration for one supported vcs backend. + """ + + invalid_repo_name = re.compile(r'[^0-9a-zA-Z]+') + + def __init__(self, alias, repo_path, test_name, test_repo_container): + self.alias = alias + self._repo_path = repo_path + self._cleanup_repos = [] + self._test_name = test_name + self._test_repo_container = test_repo_container + + def __getitem__(self, key): + return self._test_repo_container(key, self.alias).scm_instance() + + def __repr__(self): + return f'{self.__class__.__name__}(alias={self.alias}, repo={self._repo_path})' + + @property + def repo(self): + """ + Returns the "current" repository. This is the vcs_test repo of the last + repo which has been created. + """ + Repository = get_backend(self.alias) + return Repository(self._repo_path) + + @property + def backend(self): + """ + Returns the backend implementation class. + """ + return get_backend(self.alias) + + def create_repo(self, commits=None, number_of_commits=0, _clone_repo=None, + bare=False): + repo_name = self._next_repo_name() + self._repo_path = get_new_dir(repo_name) + repo_class = get_backend(self.alias) + src_url = None + if _clone_repo: + src_url = _clone_repo.path + repo = repo_class(self._repo_path, create=True, src_url=src_url, bare=bare) + self._cleanup_repos.append(repo) + + commits = commits or [ + {'message': 'Commit %s of %s' % (x, repo_name)} + for x in range(number_of_commits)] + _add_commits_to_repo(repo, commits) + return repo + + def clone_repo(self, repo): + return self.create_repo(_clone_repo=repo) + + def cleanup(self): + for repo in self._cleanup_repos: + shutil.rmtree(repo.path) + + def new_repo_path(self): + repo_name = self._next_repo_name() + self._repo_path = get_new_dir(repo_name) + return self._repo_path + + def _next_repo_name(self): + + return "{}_{}".format( + self.invalid_repo_name.sub('_', self._test_name), + len(self._cleanup_repos) + ) + + def add_file(self, repo, filename, content='Test content\n'): + imc = repo.in_memory_commit + imc.add(FileNode(safe_bytes(filename), content=safe_bytes(content))) + imc.commit( + message='Automatic commit from vcsbackend fixture', + author='Automatic ') + + def ensure_file(self, filename, content='Test content\n'): + assert self._cleanup_repos, "Avoid writing into vcs_test repos" + self.add_file(self.repo, filename, content) + + +def vcsbackend_base(request, backend_alias, tests_tmp_path, baseapp, test_repo) -> VcsBackend: if backend_alias not in request.config.getoption('--backends'): pytest.skip("Backend %s not selected." % (backend_alias, )) utils.check_xfail_backends(request.node, backend_alias) utils.check_skip_backends(request.node, backend_alias) - repo_name = 'vcs_test_%s' % (backend_alias, ) + repo_name = f'vcs_test_{backend_alias}' repo_path = os.path.join(tests_tmp_path, repo_name) backend = VcsBackend( alias=backend_alias, @@ -691,85 +775,6 @@ def vcsbackend_stub(vcsbackend_git): return vcsbackend_git -class VcsBackend(object): - """ - Represents the test configuration for one supported vcs backend. - """ - - invalid_repo_name = re.compile(r'[^0-9a-zA-Z]+') - - def __init__(self, alias, repo_path, test_name, test_repo_container): - self.alias = alias - self._repo_path = repo_path - self._cleanup_repos = [] - self._test_name = test_name - self._test_repo_container = test_repo_container - - def __getitem__(self, key): - return self._test_repo_container(key, self.alias).scm_instance() - - @property - def repo(self): - """ - Returns the "current" repository. This is the vcs_test repo of the last - repo which has been created. - """ - Repository = get_backend(self.alias) - return Repository(self._repo_path) - - @property - def backend(self): - """ - Returns the backend implementation class. - """ - return get_backend(self.alias) - - def create_repo(self, commits=None, number_of_commits=0, _clone_repo=None, - bare=False): - repo_name = self._next_repo_name() - self._repo_path = get_new_dir(repo_name) - repo_class = get_backend(self.alias) - src_url = None - if _clone_repo: - src_url = _clone_repo.path - repo = repo_class(self._repo_path, create=True, src_url=src_url, bare=bare) - self._cleanup_repos.append(repo) - - commits = commits or [ - {'message': 'Commit %s of %s' % (x, repo_name)} - for x in range(number_of_commits)] - _add_commits_to_repo(repo, commits) - return repo - - def clone_repo(self, repo): - return self.create_repo(_clone_repo=repo) - - def cleanup(self): - for repo in self._cleanup_repos: - shutil.rmtree(repo.path) - - def new_repo_path(self): - repo_name = self._next_repo_name() - self._repo_path = get_new_dir(repo_name) - return self._repo_path - - def _next_repo_name(self): - return "%s_%s" % ( - self.invalid_repo_name.sub('_', self._test_name), - len(self._cleanup_repos)) - - def add_file(self, repo, filename, content='Test content\n'): - imc = repo.in_memory_commit - imc.add(FileNode(filename, content=content)) - imc.commit( - message=u'Automatic commit from vcsbackend fixture', - author=u'Automatic ') - - def ensure_file(self, filename, content='Test content\n'): - assert self._cleanup_repos, "Avoid writing into vcs_test repos" - self.add_file(self.repo, filename, content) - - def _add_commits_to_repo(vcs_repo, commits): commit_ids = {} if not commits: @@ -782,11 +787,11 @@ def _add_commits_to_repo(vcs_repo, commi message = str(commit.get('message', 'Commit %s' % idx)) for node in commit.get('added', []): - imc.add(FileNode(node.path, content=node.content)) + imc.add(FileNode(safe_bytes(node.path), content=node.content)) for node in commit.get('changed', []): - imc.change(FileNode(node.path, content=node.content)) + imc.change(FileNode(safe_bytes(node.path), content=node.content)) for node in commit.get('removed', []): - imc.remove(FileNode(node.path)) + imc.remove(FileNode(safe_bytes(node.path))) parents = [ vcs_repo.get_commit(commit_id=commit_ids[p]) @@ -794,7 +799,7 @@ def _add_commits_to_repo(vcs_repo, commi operations = ('added', 'changed', 'removed') if not any((commit.get(o) for o in operations)): - imc.add(FileNode('file_%s' % idx, content=message)) + imc.add(FileNode(b'file_%b' % safe_bytes(str(idx)), content=safe_bytes(message))) commit = imc.commit( message=message, @@ -877,7 +882,7 @@ class PRTestUtility(object): def create_pull_request( self, commits=None, target_head=None, source_head=None, revisions=None, approved=False, author=None, mergeable=False, - enable_notifications=True, name_suffix=u'', reviewers=None, observers=None, + enable_notifications=True, name_suffix='', reviewers=None, observers=None, title=u"Test", description=u"Description"): self.set_mergeable(mergeable) if not enable_notifications: @@ -1002,7 +1007,7 @@ class PRTestUtility(object): return comment def create_inline_comment( - self, linked_to=None, line_no=u'n1', file_path='file_1'): + self, linked_to=None, line_no='n1', file_path='file_1'): comment = CommentsModel().create( text=u"Test comment", repo=self.target_repository.repo_name, diff --git a/rhodecode/tests/fixtures/diff_with_diff_data.diff b/rhodecode/tests/fixtures/diff_with_diff_data.diff --- a/rhodecode/tests/fixtures/diff_with_diff_data.diff +++ b/rhodecode/tests/fixtures/diff_with_diff_data.diff @@ -245,8 +245,8 @@ index e34033e29fa9b3d3366b723beab129cee7 + 'author': 'Joe Doe ', + 'date': datetime.datetime(2010, 1, 1, 20), + 'added': [ -+ FileNode('foobar', content='foobar'), -+ FileNode('foobar2', content='foobar2'), ++ FileNode(b'foobar', content='foobar'), ++ FileNode(b'foobar2', content='foobar2'), + ], + }, + { @@ -254,10 +254,10 @@ index e34033e29fa9b3d3366b723beab129cee7 + 'author': 'Jane Doe ', + 'date': datetime.datetime(2010, 1, 1, 21), + 'added': [ -+ FileNode('foobar3', content='foobar3'), ++ FileNode(b'foobar3', content='foobar3'), + ], + 'changed': [ -+ FileNode('foobar', 'FOOBAR'), ++ FileNode(b'foobar', 'FOOBAR'), + ], + }, + { @@ -265,9 +265,9 @@ index e34033e29fa9b3d3366b723beab129cee7 + 'author': 'Jane Doe ', + 'date': datetime.datetime(2010, 1, 1, 22), + 'changed': [ -+ FileNode('foobar3', content='FOOBAR\nFOOBAR\nFOOBAR\n'), ++ FileNode(b'foobar3', content='FOOBAR\nFOOBAR\nFOOBAR\n'), + ], -+ 'removed': [FileNode('foobar')], ++ 'removed': [FileNode(b'foobar')], + }, + ] + return commits diff --git a/rhodecode/tests/functional/__init__.py b/rhodecode/tests/functional/__init__.py --- a/rhodecode/tests/functional/__init__.py +++ b/rhodecode/tests/functional/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/functional/test_bad_request_data.py b/rhodecode/tests/functional/test_bad_request_data.py --- a/rhodecode/tests/functional/test_bad_request_data.py +++ b/rhodecode/tests/functional/test_bad_request_data.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/functional/test_delegated_admin.py b/rhodecode/tests/functional/test_delegated_admin.py --- a/rhodecode/tests/functional/test_delegated_admin.py +++ b/rhodecode/tests/functional/test_delegated_admin.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # @@ -25,7 +25,9 @@ from rhodecode.tests.fixture import Fixt def route_path(name, params=None, **kwargs): - import urllib.request, urllib.parse, urllib.error + import urllib.request + import urllib.parse + import urllib.error from rhodecode.apps._base import ADMIN_PREFIX base_url = { diff --git a/rhodecode/tests/functional/test_sessions.py b/rhodecode/tests/functional/test_sessions.py --- a/rhodecode/tests/functional/test_sessions.py +++ b/rhodecode/tests/functional/test_sessions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/integrations/conftest.py b/rhodecode/tests/integrations/conftest.py --- a/rhodecode/tests/integrations/conftest.py +++ b/rhodecode/tests/integrations/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/integrations/test_integration.py b/rhodecode/tests/integrations/test_integration.py --- a/rhodecode/tests/integrations/test_integration.py +++ b/rhodecode/tests/integrations/test_integration.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/integrations/test_slack.py b/rhodecode/tests/integrations/test_slack.py --- a/rhodecode/tests/integrations/test_slack.py +++ b/rhodecode/tests/integrations/test_slack.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/integrations/test_webhook.py b/rhodecode/tests/integrations/test_webhook.py --- a/rhodecode/tests/integrations/test_webhook.py +++ b/rhodecode/tests/integrations/test_webhook.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/__init__.py b/rhodecode/tests/lib/__init__.py --- a/rhodecode/tests/lib/__init__.py +++ b/rhodecode/tests/lib/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/auth_modules/test_auth_modules.py b/rhodecode/tests/lib/auth_modules/test_auth_modules.py --- a/rhodecode/tests/lib/auth_modules/test_auth_modules.py +++ b/rhodecode/tests/lib/auth_modules/test_auth_modules.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/middleware/__init__.py b/rhodecode/tests/lib/middleware/__init__.py --- a/rhodecode/tests/lib/middleware/__init__.py +++ b/rhodecode/tests/lib/middleware/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/middleware/mock_scm_app.py b/rhodecode/tests/lib/middleware/mock_scm_app.py --- a/rhodecode/tests/lib/middleware/mock_scm_app.py +++ b/rhodecode/tests/lib/middleware/mock_scm_app.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/middleware/test_appenlight.py b/rhodecode/tests/lib/middleware/test_appenlight.py --- a/rhodecode/tests/lib/middleware/test_appenlight.py +++ b/rhodecode/tests/lib/middleware/test_appenlight.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/middleware/test_simplegit.py b/rhodecode/tests/lib/middleware/test_simplegit.py --- a/rhodecode/tests/lib/middleware/test_simplegit.py +++ b/rhodecode/tests/lib/middleware/test_simplegit.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/middleware/test_simplehg.py b/rhodecode/tests/lib/middleware/test_simplehg.py --- a/rhodecode/tests/lib/middleware/test_simplehg.py +++ b/rhodecode/tests/lib/middleware/test_simplehg.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/middleware/test_simplesvn.py b/rhodecode/tests/lib/middleware/test_simplesvn.py --- a/rhodecode/tests/lib/middleware/test_simplesvn.py +++ b/rhodecode/tests/lib/middleware/test_simplesvn.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/middleware/test_simplevcs.py b/rhodecode/tests/lib/middleware/test_simplevcs.py --- a/rhodecode/tests/lib/middleware/test_simplevcs.py +++ b/rhodecode/tests/lib/middleware/test_simplevcs.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -18,11 +17,10 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import base64 - import mock import pytest +from rhodecode.lib.str_utils import base64_to_str from rhodecode.lib.utils2 import AttributeDict from rhodecode.tests.utils import CustomTestApp @@ -32,7 +30,7 @@ from rhodecode.lib.middleware import sim from rhodecode.lib.middleware.https_fixup import HttpsFixup from rhodecode.lib.middleware.utils import scm_app_http from rhodecode.model.db import User, _hash_key -from rhodecode.model.meta import Session +from rhodecode.model.meta import Session, cache as db_cache from rhodecode.tests import ( HG_REPO, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS) from rhodecode.tests.lib.middleware import mock_scm_app @@ -75,10 +73,13 @@ class StubVCSController(simplevcs.Simple @pytest.fixture() def vcscontroller(baseapp, config_stub, request_stub): + from rhodecode.config.middleware import ce_auth_resources + config_stub.testing_securitypolicy() config_stub.include('rhodecode.authentication') - config_stub.include('rhodecode.authentication.plugins.auth_rhodecode') - config_stub.include('rhodecode.authentication.plugins.auth_token') + + for resource in ce_auth_resources: + config_stub.include(resource) controller = StubVCSController( baseapp.config.get_settings(), request_stub.registry) @@ -98,27 +99,45 @@ def _remove_default_user_from_query_cach user = User.get_default_user(cache=True) query = Session().query(User).filter(User.username == user.username) query = query.options( - FromCache("sql_cache_short", "get_user_%s" % _hash_key(user.username))) - query.invalidate() + FromCache("sql_cache_short", f"get_user_{_hash_key(user.username)}")) + + db_cache.invalidate( + query, {}, + FromCache("sql_cache_short", f"get_user_{_hash_key(user.username)}")) + Session().expire(user) def test_handles_exceptions_during_permissions_checks( - vcscontroller, disable_anonymous_user): - user_and_pass = '%s:%s' % (TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS) - auth_password = base64.encodestring(user_and_pass).strip() + vcscontroller, disable_anonymous_user, enable_auth_plugins, test_user_factory): + + test_password = 'qweqwe' + test_user = test_user_factory(password=test_password, extern_type='headers', extern_name='headers') + test_username = test_user.username + + enable_auth_plugins.enable([ + 'egg:rhodecode-enterprise-ce#headers', + 'egg:rhodecode-enterprise-ce#token', + 'egg:rhodecode-enterprise-ce#rhodecode'], + override={ + 'egg:rhodecode-enterprise-ce#headers': {'auth_headers_header': 'REMOTE_USER'} + }) + + user_and_pass = f'{test_username}:{test_password}' + auth_password = base64_to_str(user_and_pass) + extra_environ = { 'AUTH_TYPE': 'Basic', - 'HTTP_AUTHORIZATION': 'Basic %s' % auth_password, - 'REMOTE_USER': TEST_USER_ADMIN_LOGIN, + 'HTTP_AUTHORIZATION': f'Basic {auth_password}', + 'REMOTE_USER': test_username, } - # Verify that things are hooked up correctly + # Verify that things are hooked up correctly, we pass user with headers bound auth, and headers filled in vcscontroller.get('/', status=200, extra_environ=extra_environ) # Simulate trouble during permission checks with mock.patch('rhodecode.model.db.User.get_by_username', - side_effect=Exception) as get_user: + side_effect=Exception('permission_error_test')) as get_user: # Verify that a correct 500 is returned and check that the expected # code path was hit. vcscontroller.get('/', status=500, extra_environ=extra_environ) @@ -230,7 +249,7 @@ class TestShadowRepoExposure(object): controller.is_shadow_repo = True controller._action = 'pull' controller._is_shadow_repo_dir = True - controller.stub_response_body = 'dummy body value' + controller.stub_response_body = (b'dummy body value',) controller._get_default_cache_ttl = mock.Mock( return_value=(False, 0)) @@ -242,10 +261,10 @@ class TestShadowRepoExposure(object): } response = controller(environ_stub, mock.Mock()) - response_body = ''.join(response) + response_body = b''.join(response) # Assert that we got the response from the wsgi app. - assert response_body == controller.stub_response_body + assert response_body == b''.join(controller.stub_response_body) def test_pull_on_shadow_repo_that_is_missing(self, baseapp, request_stub): """ @@ -258,7 +277,7 @@ class TestShadowRepoExposure(object): controller.is_shadow_repo = True controller._action = 'pull' controller._is_shadow_repo_dir = False - controller.stub_response_body = 'dummy body value' + controller.stub_response_body = (b'dummy body value',) environ_stub = { 'HTTP_HOST': 'test.example.com', 'HTTP_ACCEPT': 'application/mercurial', @@ -267,10 +286,10 @@ class TestShadowRepoExposure(object): } response = controller(environ_stub, mock.Mock()) - response_body = ''.join(response) + response_body = b''.join(response) # Assert that we got the response from the wsgi app. - assert '404 Not Found' in response_body + assert b'404 Not Found' in response_body def test_push_on_shadow_repo_raises(self, baseapp, request_stub): """ @@ -281,7 +300,7 @@ class TestShadowRepoExposure(object): controller._check_ssl = mock.Mock() controller.is_shadow_repo = True controller._action = 'push' - controller.stub_response_body = 'dummy body value' + controller.stub_response_body = (b'dummy body value',) environ_stub = { 'HTTP_HOST': 'test.example.com', 'HTTP_ACCEPT': 'application/mercurial', @@ -290,11 +309,11 @@ class TestShadowRepoExposure(object): } response = controller(environ_stub, mock.Mock()) - response_body = ''.join(response) + response_body = b''.join(response) assert response_body != controller.stub_response_body # Assert that a 406 error is returned. - assert '406 Not Acceptable' in response_body + assert b'406 Not Acceptable' in response_body def test_set_repo_names_no_shadow(self, baseapp, request_stub): """ diff --git a/rhodecode/tests/lib/middleware/test_vcs.py b/rhodecode/tests/lib/middleware/test_vcs.py --- a/rhodecode/tests/lib/middleware/test_vcs.py +++ b/rhodecode/tests/lib/middleware/test_vcs.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/middleware/test_vcs_unavailable.py b/rhodecode/tests/lib/middleware/test_vcs_unavailable.py --- a/rhodecode/tests/lib/middleware/test_vcs_unavailable.py +++ b/rhodecode/tests/lib/middleware/test_vcs_unavailable.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/middleware/utils/__init__.py b/rhodecode/tests/lib/middleware/utils/__init__.py --- a/rhodecode/tests/lib/middleware/utils/__init__.py +++ b/rhodecode/tests/lib/middleware/utils/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/middleware/utils/test_scm_app_http.py b/rhodecode/tests/lib/middleware/utils/test_scm_app_http.py --- a/rhodecode/tests/lib/middleware/utils/test_scm_app_http.py +++ b/rhodecode/tests/lib/middleware/utils/test_scm_app_http.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # @@ -55,20 +55,20 @@ def data(): def test_reuse_app_no_data(repeat, vcsserver_http_echo_app): app = vcs_http_app(vcsserver_http_echo_app) - for x in range(repeat / 10): + for x in range(repeat // 10): response = app.post('/') assert response.status_code == 200 def test_reuse_app_with_data(data, repeat, vcsserver_http_echo_app): app = vcs_http_app(vcsserver_http_echo_app) - for x in range(repeat / 10): + for x in range(repeat // 10): response = app.post('/', params=data) assert response.status_code == 200 def test_create_app_per_request_no_data(repeat, vcsserver_http_echo_app): - for x in range(repeat / 10): + for x in range(repeat // 10): app = vcs_http_app(vcsserver_http_echo_app) response = app.post('/') assert response.status_code == 200 @@ -76,7 +76,7 @@ def test_create_app_per_request_no_data( def test_create_app_per_request_with_data( data, repeat, vcsserver_http_echo_app): - for x in range(repeat / 10): + for x in range(repeat // 10): app = vcs_http_app(vcsserver_http_echo_app) response = app.post('/', params=data) assert response.status_code == 200 diff --git a/rhodecode/tests/lib/middleware/utils/test_scm_app_http_chunking.py b/rhodecode/tests/lib/middleware/utils/test_scm_app_http_chunking.py --- a/rhodecode/tests/lib/middleware/utils/test_scm_app_http_chunking.py +++ b/rhodecode/tests/lib/middleware/utils/test_scm_app_http_chunking.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/middleware/utils/test_wsgi_app_caller_client.py b/rhodecode/tests/lib/middleware/utils/test_wsgi_app_caller_client.py --- a/rhodecode/tests/lib/middleware/utils/test_wsgi_app_caller_client.py +++ b/rhodecode/tests/lib/middleware/utils/test_wsgi_app_caller_client.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -82,7 +81,7 @@ def test_remote_app_caller(): ('a1', 'a2', 'a3', 'a4', None)) # Note: RemoteAppCaller is expected to return a tuple like the # following one - return (['content'], '200 OK', [('Content-Type', 'text/plain')]) + return ([b'content'], '200 OK', [('Content-Type', 'text/plain')]) wrapper_app = wsgi_app_caller_client.RemoteAppCaller( RemoteAppCallerMock(), 'a1', 'a2', arg3='a3', arg4='a4') diff --git a/rhodecode/tests/lib/test_action_parser.py b/rhodecode/tests/lib/test_action_parser.py --- a/rhodecode/tests/lib/test_action_parser.py +++ b/rhodecode/tests/lib/test_action_parser.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_audit_logger.py b/rhodecode/tests/lib/test_audit_logger.py --- a/rhodecode/tests/lib/test_audit_logger.py +++ b/rhodecode/tests/lib/test_audit_logger.py @@ -1,5 +1,4 @@ import collections -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_auth.py b/rhodecode/tests/lib/test_auth.py --- a/rhodecode/tests/lib/test_auth.py +++ b/rhodecode/tests/lib/test_auth.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -19,13 +18,13 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ import os -from hashlib import sha1 import pytest from mock import patch from rhodecode.lib import auth -from rhodecode.lib.utils2 import md5 +from rhodecode.lib.str_utils import safe_bytes +from rhodecode.lib.hash_utils import md5_safe, sha1 from rhodecode.model.auth_token import AuthTokenModel from rhodecode.model.db import Session, User from rhodecode.model.repo import RepoModel @@ -638,7 +637,7 @@ def test_auth_user_get_cookie_store_for_ expected_data = { 'username': user.username, 'user_id': user.user_id, - 'password': md5(user.password), + 'password': md5_safe(user.password), 'is_authenticated': False } assert auth_user.get_cookie_store() == expected_data @@ -650,7 +649,7 @@ def test_auth_user_get_cookie_store_for_ expected_data = { 'username': User.DEFAULT_USER, 'user_id': default_user.user_id, - 'password': md5(default_user.password), + 'password': md5_safe(default_user.password), 'is_authenticated': True } assert auth_user.get_cookie_store() == expected_data @@ -678,10 +677,10 @@ def get_permissions(user, **kwargs): class TestGenerateAuthToken(object): def test_salt_is_used_when_specified(self): - salt = 'abcde' + salt = b'abcde' user_name = 'test_user' result = auth.generate_auth_token(user_name, salt) - expected_result = sha1(user_name + salt).hexdigest() + expected_result = sha1(safe_bytes(user_name) + salt) assert result == expected_result def test_salt_is_geneated_when_not_specified(self): @@ -690,7 +689,8 @@ class TestGenerateAuthToken(object): with patch.object(auth, 'os') as os_mock: os_mock.urandom.return_value = random_salt result = auth.generate_auth_token(user_name) - expected_result = sha1(user_name + random_salt).hexdigest() + + expected_result = sha1(safe_bytes(user_name) + random_salt) assert result == expected_result diff --git a/rhodecode/tests/lib/test_auth_crypto_backend.py b/rhodecode/tests/lib/test_auth_crypto_backend.py --- a/rhodecode/tests/lib/test_auth_crypto_backend.py +++ b/rhodecode/tests/lib/test_auth_crypto_backend.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_base.py b/rhodecode/tests/lib/test_base.py --- a/rhodecode/tests/lib/test_base.py +++ b/rhodecode/tests/lib/test_base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_caches.py b/rhodecode/tests/lib/test_caches.py --- a/rhodecode/tests/lib/test_caches.py +++ b/rhodecode/tests/lib/test_caches.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # @@ -67,12 +67,12 @@ class TestCaches(object): def test_cache_keygen(self, example_input, example_namespace): def func_wrapped(): return 1 - func = rc_cache.utils.key_generator(None, example_namespace, func_wrapped) + func = rc_cache.utils.custom_key_generator(None, example_namespace, func_wrapped) key = func(*example_input) assert key def test_store_value_in_cache(self): - cache_region = rc_cache.get_or_create_region('cache_perms') + cache_region = rc_cache.get_or_create_region('cache_perms', 'test_cache') # make sure we empty the cache now cache_region.delete_multi(cache_region.backend.list_keys()) @@ -88,7 +88,7 @@ class TestCaches(object): assert len(set(cache_region.backend.list_keys())) == 10 def test_store_and_get_value_from_region(self): - cache_region = rc_cache.get_or_create_region('cache_perms') + cache_region = rc_cache.get_or_create_region('cache_perms', 'test_cache') # make sure we empty the cache now for key in cache_region.backend.list_keys(): cache_region.delete(key) diff --git a/rhodecode/tests/lib/test_codeblocks.py b/rhodecode/tests/lib/test_codeblocks.py --- a/rhodecode/tests/lib/test_codeblocks.py +++ b/rhodecode/tests/lib/test_codeblocks.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # @@ -40,33 +40,34 @@ class TestTokenizeString(object): def test_tokenize_as_python(self): lexer = get_lexer_by_name('python') tokens = list(tokenize_string(self.python_code, lexer)) + expected_tokens = [ + ('w', '\n'), + ('', ' '), + ('kn', 'import'), + ('', ' '), + ('nn', 'this'), + ('w', '\n'), + ('w', '\n'), + ('', ' '), + ('n', 'var'), + ('', ' '), + ('o', '='), + ('', ' '), + ('mi', '6'), + ('w', '\n'), + ('', ' '), + ('nb', 'print'), + ('p', '('), + ('s2', '"'), + ('s2', 'this'), + ('s2', '"'), + ('p', ')'), + ('w', '\n'), + ('w', '\n'), + ('', ' ') + ] - assert tokens == [ - ('', u'\n'), - ('', u' '), - ('kn', u'import'), - ('', u' '), - ('nn', u'this'), - ('', u'\n'), - ('', u'\n'), - ('', u' '), - ('n', u'var'), - ('', u' '), - ('o', u'='), - ('', u' '), - ('mi', u'6'), - ('', u'\n'), - ('', u' '), - ('k', u'print'), - ('p', u'('), - ('s2', u'"'), - ('s2', u'this'), - ('s2', u'"'), - ('p', u')'), - ('', u'\n'), - ('', u'\n'), - ('', u' ') - ] + assert tokens == expected_tokens def test_tokenize_as_text(self): lexer = get_lexer_by_name('text') @@ -74,7 +75,7 @@ class TestTokenizeString(object): assert tokens == [ ('', - u'\n import this\n\n var = 6\n print("this")\n\n ') + '\n import this\n\n var = 6\n print("this")\n\n ') ] @@ -86,9 +87,9 @@ class TestSplitTokenStream(object): lines = list(split_token_stream(tokens, content)) assert lines == [ - [('type1', u'some')], - [('type1', u'text'), ('type2', u'more')], - [('type2', u'')], + [('type1', 'some')], + [('type1', 'text'), ('type2', 'more')], + [('type2', '')], ] def test_split_token_stream_single(self): @@ -126,7 +127,7 @@ class TestSplitTokenStream(object): def test_no_tokens_by_content(self): tokens = [] - content = u'\ufeff' + content = '\ufeff' lines = list(split_token_stream(tokens, content)) assert lines == [ [('', content)], @@ -134,15 +135,15 @@ class TestSplitTokenStream(object): def test_no_tokens_by_valid_content(self): from pygments.lexers.css import CssLexer - content = u'\ufeff table.dataTable' + content = '\ufeff table.dataTable' tokens = tokenize_string(content, CssLexer()) lines = list(split_token_stream(tokens, content)) assert lines == [ - [('', u' '), - ('nt', u'table'), - ('p', u'.'), - ('nc', u'dataTable')], + [('w', ' '), + ('nt', 'table'), + ('p', '.'), + ('nc', 'dataTable')], ] @@ -248,59 +249,59 @@ class TestRenderTokenStream(object): '', ), ( - [('', '', u'')], + [('', '', '')], '', ), ( - [('', '', u'text')], + [('', '', 'text')], 'text', ), ( - [('A', '', u'')], + [('A', '', '')], '', ), ( - [('A', '', u'hello')], + [('A', '', 'hello')], 'hello', ), ( - [('A', '', u'hel'), ('A', '', u'lo')], + [('A', '', 'hel'), ('A', '', 'lo')], 'hello', ), ( - [('A', '', u'two\n'), ('A', '', u'lines')], + [('A', '', 'two\n'), ('A', '', 'lines')], 'two\nlines', ), ( - [('A', '', u'\nthree\n'), ('A', '', u'lines')], + [('A', '', '\nthree\n'), ('A', '', 'lines')], '\nthree\nlines', ), ( - [('', '', u'\n'), ('A', '', u'line')], + [('', '', '\n'), ('A', '', 'line')], '\nline', ), ( - [('', 'ins', u'\n'), ('A', '', u'line')], + [('', 'ins', '\n'), ('A', '', 'line')], '\nline', ), ( - [('A', '', u'hel'), ('A', 'ins', u'lo')], + [('A', '', 'hel'), ('A', 'ins', 'lo')], 'hello', ), ( - [('A', '', u'hel'), ('A', 'ins', u'l'), ('A', 'ins', u'o')], + [('A', '', 'hel'), ('A', 'ins', 'l'), ('A', 'ins', 'o')], 'hello', ), ( - [('A', '', u'hel'), ('A', 'ins', u'l'), ('A', 'del', u'o')], + [('A', '', 'hel'), ('A', 'ins', 'l'), ('A', 'del', 'o')], 'hello', ), ( - [('A', '', u'hel'), ('B', '', u'lo')], + [('A', '', 'hel'), ('B', '', 'lo')], 'hello', ), ( - [('A', '', u'hel'), ('B', 'ins', u'lo')], + [('A', '', 'hel'), ('B', 'ins', 'lo')], 'hello', ), ], ids=no_newline_id_generator) @@ -310,23 +311,23 @@ class TestRenderTokenStream(object): @pytest.mark.parametrize('tokenstream,output', [ ( - [('A', u'hel'), ('A', u'lo')], + [('A', 'hel'), ('A', 'lo')], 'hello', ), ( - [('A', u'hel'), ('A', u'l'), ('A', u'o')], + [('A', 'hel'), ('A', 'l'), ('A', 'o')], 'hello', ), ( - [('A', u'hel'), ('A', u'l'), ('A', u'o')], + [('A', 'hel'), ('A', 'l'), ('A', 'o')], 'hello', ), ( - [('A', u'hel'), ('B', u'lo')], + [('A', 'hel'), ('B', 'lo')], 'hello', ), ( - [('A', u'hel'), ('B', u'lo')], + [('A', 'hel'), ('B', 'lo')], 'hello', ), ]) diff --git a/rhodecode/tests/lib/test_colored_formatter.py b/rhodecode/tests/lib/test_colored_formatter.py --- a/rhodecode/tests/lib/test_colored_formatter.py +++ b/rhodecode/tests/lib/test_colored_formatter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_datelib.py b/rhodecode/tests/lib/test_datelib.py --- a/rhodecode/tests/lib/test_datelib.py +++ b/rhodecode/tests/lib/test_datelib.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_db_manage.py b/rhodecode/tests/lib/test_db_manage.py --- a/rhodecode/tests/lib/test_db_manage.py +++ b/rhodecode/tests/lib/test_db_manage.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_dbmigrate.py b/rhodecode/tests/lib/test_dbmigrate.py --- a/rhodecode/tests/lib/test_dbmigrate.py +++ b/rhodecode/tests/lib/test_dbmigrate.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_diffs.py b/rhodecode/tests/lib/test_diffs.py --- a/rhodecode/tests/lib/test_diffs.py +++ b/rhodecode/tests/lib/test_diffs.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -28,6 +27,7 @@ from rhodecode.lib.diffs import ( DiffProcessor, NEW_FILENODE, DEL_FILENODE, MOD_FILENODE, RENAMED_FILENODE, CHMOD_FILENODE, BIN_FILENODE, COPIED_FILENODE) +from rhodecode.lib.str_utils import safe_bytes from rhodecode.lib.utils2 import AttributeDict from rhodecode.lib.vcs.backends.git import GitCommit from rhodecode.tests.fixture import Fixture, no_newline_id_generator @@ -38,134 +38,6 @@ from rhodecode.lib.vcs.backends.svn.repo fixture = Fixture() -def test_diffprocessor_as_html_with_comments(): - raw_diff = textwrap.dedent(''' - diff --git a/setup.py b/setup.py - index 5b36422..cfd698e 100755 - --- a/setup.py - +++ b/setup.py - @@ -2,7 +2,7 @@ - #!/usr/bin/python - # Setup file for X - # Copyright (C) No one - - - +x - try: - from setuptools import setup, Extension - except ImportError: - ''') - diff = GitDiff(raw_diff) - processor = DiffProcessor(diff) - processor.prepare() - - # Note that the cell with the context in line 5 (in the html) has the - # no-comment class, which will prevent the add comment icon to be displayed. - expected_html = textwrap.dedent(''' - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
...... - 
@@ -2,7 +2,7 @@
-
-
22 - 
#!/usr/bin/python
-
-
33 - 
# Setup file for X
-
-
44 - 
# Copyright (C) No one
-
-
5 - 
-
-
5 - 
x
-
-
66 - 
try:
-
-
77 - 
    from setuptools import setup, Extension
-
-
88 - 
except ImportError:
-
-
- ''').strip() - html = processor.as_html(enable_comments=True).replace('\t', ' ') - - assert html == expected_html - - class TestMixedFilenameEncodings(object): @pytest.fixture(scope="class") @@ -176,7 +48,7 @@ class TestMixedFilenameEncodings(object) @pytest.fixture() def processor(self, raw_diff): diff = MercurialDiff(raw_diff) - processor = DiffProcessor(diff) + processor = DiffProcessor(diff, diff_format='newdiff') return processor def test_filenames_are_decoded_to_unicode(self, processor): @@ -207,6 +79,8 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '100755', 'ops': {NEW_FILENODE: 'new file 100755', BIN_FILENODE: 'binary diff hidden'}}), ]), @@ -216,6 +90,8 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '', 'ops': {MOD_FILENODE: 'modified file', BIN_FILENODE: 'binary diff hidden'}}), ]), @@ -225,6 +101,9 @@ DIFF_FIXTURES = [ {'added': 3, 'deleted': 0, 'binary': False, + 'old_mode': '100755', + 'new_mode': '100644', + 'renamed': ('README.rst', 'README'), 'ops': {MOD_FILENODE: 'modified file', RENAMED_FILENODE: 'file renamed from README.rst to README', CHMOD_FILENODE: 'modified file chmod 100755 => 100644'}}), @@ -235,6 +114,8 @@ DIFF_FIXTURES = [ {'added': 2, 'deleted': 1, 'binary': False, + 'old_mode': '', + 'new_mode': '', 'ops': {MOD_FILENODE: 'modified file'}}), ]), ('hg', @@ -243,6 +124,9 @@ DIFF_FIXTURES = [ {'added': 3, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '', + 'renamed': ('README', 'README.rst'), 'ops': {MOD_FILENODE: 'modified file', RENAMED_FILENODE: 'file renamed from README to README.rst'}}), ]), @@ -252,6 +136,8 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '', 'ops': {DEL_FILENODE: 'deleted file', BIN_FILENODE: 'binary diff hidden'}}), ]), @@ -261,6 +147,8 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '100644', + 'new_mode': '100755', 'ops': {CHMOD_FILENODE: 'modified file chmod 100644 => 100755', BIN_FILENODE: 'binary diff hidden'}}), ]), @@ -270,6 +158,8 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '100755', + 'new_mode': '100644', 'ops': {CHMOD_FILENODE: 'modified file chmod 100755 => 100644'}}), ]), ('hg', @@ -278,6 +168,9 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '', + 'renamed': ('file', 'file_renamed'), 'ops': {RENAMED_FILENODE: 'file renamed from file to file_renamed'}}), ]), ('hg', @@ -286,6 +179,9 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '100644', + 'new_mode': '100755', + 'renamed': ('README.rst', 'README'), 'ops': {CHMOD_FILENODE: 'modified file chmod 100644 => 100755', RENAMED_FILENODE: 'file renamed from README.rst to README'}}), ]), @@ -295,43 +191,59 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'new_mode': '100644', + 'old_mode': '', 'ops': {NEW_FILENODE: 'new file 100644', BIN_FILENODE: 'binary diff hidden'}}), ('js/jquery/hashgrid.js', 'A', {'added': 340, 'deleted': 0, 'binary': False, + 'new_mode': '100755', + 'old_mode': '', 'ops': {NEW_FILENODE: 'new file 100755'}}), ('index.html', 'M', {'added': 3, 'deleted': 2, 'binary': False, + 'new_mode': '', + 'old_mode': '', 'ops': {MOD_FILENODE: 'modified file'}}), ('less/docs.less', 'M', {'added': 34, 'deleted': 0, 'binary': False, + 'new_mode': '', + 'old_mode': '', 'ops': {MOD_FILENODE: 'modified file'}}), ('less/scaffolding.less', 'M', {'added': 1, 'deleted': 3, 'binary': False, + 'new_mode': '', + 'old_mode': '', 'ops': {MOD_FILENODE: 'modified file'}}), ('readme.markdown', 'M', {'added': 1, 'deleted': 10, 'binary': False, + 'new_mode': '', + 'old_mode': '', 'ops': {MOD_FILENODE: 'modified file'}}), ('img/baseline-20px.png', 'D', {'added': 0, 'deleted': 0, 'binary': True, + 'new_mode': '', + 'old_mode': '', 'ops': {DEL_FILENODE: 'deleted file', BIN_FILENODE: 'binary diff hidden'}}), ('js/global.js', 'D', {'added': 0, 'deleted': 75, 'binary': False, + 'new_mode': '', + 'old_mode': '', 'ops': {DEL_FILENODE: 'deleted file'}}) ]), ('git', @@ -340,6 +252,8 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '100644', + 'new_mode': '100755', 'ops': {CHMOD_FILENODE: 'modified file chmod 100644 => 100755'}}) ]), ('git', @@ -348,6 +262,8 @@ DIFF_FIXTURES = [ {'added': 1, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {MOD_FILENODE: 'modified file'}}) ]), ('git', @@ -356,6 +272,9 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '', + 'renamed': ('work-horus.xls', 'file.xls'), 'ops': { RENAMED_FILENODE: 'file renamed from work-horus.xls to file.xls'}}) ]), @@ -365,6 +284,8 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '', 'ops': {MOD_FILENODE: 'modified file', BIN_FILENODE: 'binary diff hidden'}}) ]), @@ -374,43 +295,59 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '100644', 'ops': {NEW_FILENODE: 'new file 100644', BIN_FILENODE: 'binary diff hidden'}}), ('js/jquery/hashgrid.js', 'A', {'added': 340, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '100755', 'ops': {NEW_FILENODE: 'new file 100755'}}), ('index.html', 'M', {'added': 3, 'deleted': 2, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {MOD_FILENODE: 'modified file'}}), ('less/docs.less', 'M', {'added': 34, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {MOD_FILENODE: 'modified file'}}), ('less/scaffolding.less', 'M', {'added': 1, 'deleted': 3, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {MOD_FILENODE: 'modified file'}}), ('readme.markdown', 'M', {'added': 1, 'deleted': 10, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {MOD_FILENODE: 'modified file'}}), ('img/baseline-20px.png', 'D', {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '', 'ops': {DEL_FILENODE: 'deleted file', BIN_FILENODE: 'binary diff hidden'}}), ('js/global.js', 'D', {'added': 0, 'deleted': 75, 'binary': False, + 'old_mode': '', + 'new_mode': '', 'ops': {DEL_FILENODE: 'deleted file'}}), ]), ('hg', @@ -419,26 +356,36 @@ DIFF_FIXTURES = [ {'added': 18, 'deleted': 2, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {MOD_FILENODE: 'modified file'}}), ('vcs/backends/git/repository.py', 'M', {'added': 46, 'deleted': 15, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {MOD_FILENODE: 'modified file'}}), ('vcs/backends/hg.py', 'M', {'added': 22, 'deleted': 3, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {MOD_FILENODE: 'modified file'}}), ('vcs/tests/test_git.py', 'M', {'added': 5, 'deleted': 5, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {MOD_FILENODE: 'modified file'}}), ('vcs/tests/test_repository.py', 'M', {'added': 174, 'deleted': 2, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {MOD_FILENODE: 'modified file'}}), ]), ('hg', @@ -447,6 +394,9 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '', + 'copied': ('file1', 'file2'), 'ops': {COPIED_FILENODE: 'file copied from file1 to file2'}}), ]), ('hg', @@ -455,6 +405,9 @@ DIFF_FIXTURES = [ {'added': 1, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '', + 'copied': ('file2', 'file3'), 'ops': {COPIED_FILENODE: 'file copied from file2 to file3', MOD_FILENODE: 'modified file'}}), ]), @@ -464,6 +417,9 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '100644', + 'new_mode': '100755', + 'copied': ('file3', 'file4'), 'ops': {COPIED_FILENODE: 'file copied from file3 to file4', CHMOD_FILENODE: 'modified file chmod 100644 => 100755'}}), ]), @@ -473,6 +429,9 @@ DIFF_FIXTURES = [ {'added': 2, 'deleted': 1, 'binary': False, + 'old_mode': '100755', + 'new_mode': '100644', + 'copied': ('file4', 'file5'), 'ops': {COPIED_FILENODE: 'file copied from file4 to file5', CHMOD_FILENODE: 'modified file chmod 100755 => 100644', MOD_FILENODE: 'modified file'}})]), @@ -484,6 +443,9 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '', + 'renamed': ('file_with_ spaces.txt', 'file_with_ two spaces.txt'), 'ops': { RENAMED_FILENODE: ( 'file renamed from file_with_ spaces.txt to file_with_ ' @@ -495,10 +457,12 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '', + 'renamed': ('file_ with update.txt', 'file_changed _.txt'), 'ops': { RENAMED_FILENODE: ( - 'file renamed from file_ with update.txt to file_changed' - ' _.txt')} + 'file renamed from file_ with update.txt to file_changed _.txt')} }), ]), ('hg', 'hg_diff_copy_file_with_spaces.diff', @@ -506,6 +470,9 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '', + 'copied': ('file_changed_without_spaces.txt', 'file_copied_ with spaces.txt'), 'ops': { COPIED_FILENODE: ( 'file copied from file_changed_without_spaces.txt to' @@ -520,6 +487,8 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '100644', 'ops': {NEW_FILENODE: 'new file 100644', BIN_FILENODE: 'binary diff hidden'} }), @@ -530,6 +499,8 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': True, + 'old_mode': '', + 'new_mode': '100644', 'ops': {NEW_FILENODE: 'new file 100644', } }), ]), @@ -540,6 +511,8 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '10644', 'ops': {NEW_FILENODE: 'new file 10644', #TODO(Marcink): depends on binary detection on svn patches # BIN_FILENODE: 'binary diff hidden' @@ -553,6 +526,8 @@ DIFF_FIXTURES = [ {'added': 0, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '', 'ops': {MOD_FILENODE: 'modified file', #TODO(Marcink): depends on binary detection on svn patches # BIN_FILENODE: 'binary diff hidden' @@ -562,48 +537,64 @@ DIFF_FIXTURES = [ {'added': 89, 'deleted': 34, 'binary': False, + 'old_mode': '', + 'new_mode': '', 'ops': {MOD_FILENODE: 'modified file'} }), ('trunk/doc/source/en/tsvn_ch04.xml', 'M', {'added': 66, 'deleted': 21, 'binary': False, + 'old_mode': '', + 'new_mode': '', 'ops': {MOD_FILENODE: 'modified file'} }), ('trunk/src/Changelog.txt', 'M', {'added': 2, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '', 'ops': {MOD_FILENODE: 'modified file'} }), ('trunk/src/Resources/TortoiseProcENG.rc', 'M', {'added': 19, 'deleted': 13, 'binary': False, + 'old_mode': '', + 'new_mode': '', 'ops': {MOD_FILENODE: 'modified file'} }), ('trunk/src/TortoiseProc/SetOverlayPage.cpp', 'M', {'added': 16, 'deleted': 1, 'binary': False, + 'old_mode': '', + 'new_mode': '', 'ops': {MOD_FILENODE: 'modified file'} }), ('trunk/src/TortoiseProc/SetOverlayPage.h', 'M', {'added': 3, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '', 'ops': {MOD_FILENODE: 'modified file'} }), ('trunk/src/TortoiseProc/resource.h', 'M', {'added': 2, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '', 'ops': {MOD_FILENODE: 'modified file'} }), ('trunk/src/TortoiseShell/ShellCache.h', 'M', {'added': 50, 'deleted': 1, 'binary': False, + 'old_mode': '', + 'new_mode': '', 'ops': {MOD_FILENODE: 'modified file'} }), ]), @@ -621,6 +612,8 @@ DIFF_FIXTURES_WITH_CONTENT = [ 'added': 1, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {NEW_FILENODE: 'new file 100644', } }, '@@ -0,0 +1 @@\n+test_content b\n' # diff @@ -637,6 +630,8 @@ DIFF_FIXTURES_WITH_CONTENT = [ 'added': 1, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {NEW_FILENODE: 'new file 100644', } }, '@@ -0,0 +1 @@\n+test_content b\n' # diff @@ -648,6 +643,8 @@ DIFF_FIXTURES_WITH_CONTENT = [ 'added': 1, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {NEW_FILENODE: 'new file 100644', } }, '@@ -0,0 +1 @@\n+test_content c\n' # diff @@ -664,6 +661,8 @@ DIFF_FIXTURES_WITH_CONTENT = [ 'added': 1, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {NEW_FILENODE: 'new file 100644', } }, '@@ -0,0 +1 @@\n+test_content b\n\n' # diff @@ -675,6 +674,8 @@ DIFF_FIXTURES_WITH_CONTENT = [ 'added': 1, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {NEW_FILENODE: 'new file 100644', } }, '@@ -0,0 +1 @@\n+test_content c\n' # diff @@ -691,6 +692,8 @@ DIFF_FIXTURES_WITH_CONTENT = [ 'added': 1, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {NEW_FILENODE: 'new file 100644', } }, '@@ -0,0 +1,1 @@\n+file\n' # diff @@ -702,6 +705,8 @@ DIFF_FIXTURES_WITH_CONTENT = [ 'added': 1, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {NEW_FILENODE: 'new file 100644', } }, '@@ -0,0 +1,1 @@\n+another line\n' # diff @@ -713,6 +718,8 @@ DIFF_FIXTURES_WITH_CONTENT = [ 'added': 1, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {NEW_FILENODE: 'new file 100644', } }, '@@ -0,0 +1,1 @@\n+newline\n' # diff @@ -724,6 +731,8 @@ DIFF_FIXTURES_WITH_CONTENT = [ 'added': 1, 'deleted': 0, 'binary': False, + 'old_mode': '', + 'new_mode': '100644', 'ops': {NEW_FILENODE: 'new file 100644', } }, '@@ -0,0 +1,1 @@\n+fil4\n\\ No newline at end of file' # diff @@ -741,28 +750,46 @@ diff_class = { } -@pytest.fixture(params=DIFF_FIXTURES) -def diff_fixture(request): - vcs, diff_fixture, expected = request.param - diff_txt = fixture.load_resource(diff_fixture) - diff = diff_class[vcs](diff_txt) - return diff, expected +@pytest.mark.parametrize('vcs_type, diff_file, expected_data', DIFF_FIXTURES) +def test_diff_lib(vcs_type, diff_file, expected_data): + diff_txt = fixture.load_resource(diff_file) + diff = diff_class[vcs_type](diff_txt) - -def test_diff_lib(diff_fixture): - diff, expected_data = diff_fixture - diff_proc = DiffProcessor(diff) + diff_proc = DiffProcessor(diff, diff_format='newdiff') diff_proc_d = diff_proc.prepare() - data = [(x['filename'], x['operation'], x['stats']) for x in diff_proc_d] + data = [(x['filename'], x['operation'], x['stats']) + for x in diff_proc_d] assert expected_data == data -@pytest.fixture(params=DIFF_FIXTURES_WITH_CONTENT) -def diff_fixture_w_content(request): - vcs, diff_fixture, expected = request.param - diff_txt = fixture.load_resource(diff_fixture) - diff = diff_class[vcs](diff_txt) - return diff, expected +@pytest.mark.parametrize('vcs_type, diff_file, expected_data', DIFF_FIXTURES_WITH_CONTENT) +def test_diff_lib_newlines(vcs_type, diff_file, expected_data): + diff_txt = fixture.load_resource(diff_file) + diff = diff_class[vcs_type](diff_txt) + + diff_proc = DiffProcessor(diff, diff_format='newdiff') + diff_proc_d = diff_proc.prepare() + data = [(x['filename'], x['operation'], x['stats'], x['raw_diff']) + for x in diff_proc_d] + assert expected_data == data + + +@pytest.mark.parametrize('input_str', [ + b'', + b'\n', + b'\n\n', + b'First\n+second', + b'First\n+second\n', + + b'\n\n\n Multi \n\n\n', + b'\n\n\n Multi beginning', + b'Multi end \n\n\n', + b'Multi end', + b'@@ -0,0 +1 @@\n+test_content \n\n b\n' +], ids=no_newline_id_generator) +def test_splitlines(input_str): + result = DiffProcessor.diff_splitter(input_str) + assert list(result) == input_str.splitlines(True) def test_diff_over_limit(request): @@ -772,9 +799,9 @@ def test_diff_over_limit(request): raw_diff = fixture.load_resource('large_diff.diff') vcs_diff = GitDiff(raw_diff) - diff_processor = DiffProcessor( - vcs_diff, format='newdiff', diff_limit=diff_limit, file_limit=file_limit, - show_full_diff=False) + diff_processor = DiffProcessor(vcs_diff, diff_format='newdiff', + diff_limit=diff_limit, file_limit=file_limit, + show_full_diff=False) _parsed = diff_processor.prepare() @@ -796,30 +823,3 @@ def test_diff_over_limit(request): assert diffset.files[1].patch['filename'] == 'README.md' assert diffset.files[1].limited_diff is False - - -def test_diff_lib_newlines(diff_fixture_w_content): - diff, expected_data = diff_fixture_w_content - diff_proc = DiffProcessor(diff) - diff_proc_d = diff_proc.prepare() - data = [(x['filename'], x['operation'], x['stats'], x['raw_diff']) - for x in diff_proc_d] - assert expected_data == data - - -@pytest.mark.parametrize('input_str', [ - '', - '\n', - '\n\n', - 'First\n+second', - 'First\n+second\n', - - '\n\n\n Multi \n\n\n', - '\n\n\n Multi beginning', - 'Multi end \n\n\n', - 'Multi end', - '@@ -0,0 +1 @@\n+test_content \n\n b\n' -], ids=no_newline_id_generator) -def test_splitlines(input_str): - result = DiffProcessor.diff_splitter(input_str) - assert list(result) == input_str.splitlines(True) diff --git a/rhodecode/tests/lib/test_diffs_context.py b/rhodecode/tests/lib/test_diffs_context.py --- a/rhodecode/tests/lib/test_diffs_context.py +++ b/rhodecode/tests/lib/test_diffs_context.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -27,6 +26,7 @@ import textwrap import pytest from rhodecode.lib import diffs +from rhodecode.lib.str_utils import safe_bytes from rhodecode.lib.vcs.backends.git.diff import GitDiff @@ -40,13 +40,13 @@ def test_context_of_an_old_line_number(d context = diff_processor.get_context_of_line( path='file.txt', diff_line=diffs.DiffLineNumber(old=7, new=None)) expected_context = [ - ('unmod', 'line04\n'), - ('unmod', 'line05\n'), - ('unmod', 'line06\n'), - ('unmod', 'line07\n'), - ('add', 'line07a Add after line07\n'), - ('unmod', 'line08\n'), - ('unmod', 'line09\n'), + ('unmod', b'line04\n'), + ('unmod', b'line05\n'), + ('unmod', b'line06\n'), + ('unmod', b'line07\n'), + ('add', b'line07a Add after line07\n'), + ('unmod', b'line08\n'), + ('unmod', b'line09\n'), ] assert context == expected_context @@ -55,13 +55,13 @@ def test_context_of_a_new_line_number(di context = diff_processor.get_context_of_line( path='file.txt', diff_line=diffs.DiffLineNumber(old=None, new=8)) expected_context = [ - ('unmod', 'line05\n'), - ('unmod', 'line06\n'), - ('unmod', 'line07\n'), - ('add', 'line07a Add after line07\n'), - ('unmod', 'line08\n'), - ('unmod', 'line09\n'), - ('unmod', 'line10\n'), + ('unmod', b'line05\n'), + ('unmod', b'line06\n'), + ('unmod', b'line07\n'), + ('add', b'line07a Add after line07\n'), + ('unmod', b'line08\n'), + ('unmod', b'line09\n'), + ('unmod', b'line10\n'), ] assert context == expected_context @@ -72,11 +72,11 @@ def test_context_of_an_invisible_line_be context = diff_processor.get_context_of_line( path='file.txt', diff_line=diffs.DiffLineNumber(old=None, new=3)) expected_context = [ - ('unmod', 'line02\n'), - ('unmod', 'line03\n'), - ('unmod', 'line04\n'), - ('unmod', 'line05\n'), - ('unmod', 'line06\n'), + ('unmod', b'line02\n'), + ('unmod', b'line03\n'), + ('unmod', b'line04\n'), + ('unmod', b'line05\n'), + ('unmod', b'line06\n'), ] assert context == expected_context @@ -87,11 +87,11 @@ def test_context_of_an_invisible_line_en context = diff_processor.get_context_of_line( path='file.txt', diff_line=diffs.DiffLineNumber(old=12, new=None)) expected_context = [ - ('unmod', 'line09\n'), - ('unmod', 'line10\n'), - ('unmod', 'line11\n'), - ('unmod', 'line12\n'), - ('unmod', 'line13\n'), + ('unmod', b'line09\n'), + ('unmod', b'line10\n'), + ('unmod', b'line11\n'), + ('unmod', b'line12\n'), + ('unmod', b'line13\n'), ] assert context == expected_context @@ -101,11 +101,11 @@ def test_context_of_an_incomplete_hunk_i context = diff_processor.get_context_of_line( path='file.txt', diff_line=diffs.DiffLineNumber(old=None, new=2)) expected_context = [ - ('unmod', 'line01\n'), - ('add', 'line01a Add line after line01\n'), - ('unmod', 'line02\n'), - ('unmod', 'line03\n'), - ('unmod', 'line04\n'), + ('unmod', b'line01\n'), + ('add', b'line01a Add line after line01\n'), + ('unmod', b'line02\n'), + ('unmod', b'line03\n'), + ('unmod', b'line04\n'), ] assert context == expected_context @@ -115,11 +115,11 @@ def test_context_of_an_incomplete_hunk_i context = diff_processor.get_context_of_line( path='file.txt', diff_line=diffs.DiffLineNumber(old=None, new=80)) expected_context = [ - ('unmod', 'line36\n'), - ('unmod', 'line37\n'), - ('unmod', 'line38\n'), - ('add', 'line38a Add line after line38\n'), - ('unmod', 'line39\n'), + ('unmod', b'line36\n'), + ('unmod', b'line37\n'), + ('unmod', b'line38\n'), + ('add', b'line38a Add line after line38\n'), + ('unmod', b'line39\n'), ] assert context == expected_context @@ -131,7 +131,7 @@ def test_context_of_an_incomplete_hunk_i def test_appends_newline_for_each_context_line(diff_processor): context = diff_processor.get_context_of_line( path='file_b', diff_line=diffs.DiffLineNumber(old=None, new=1)) - assert context == [('add', 'test_content\n')] + assert context == [('add', b'test_content\n')] def test_context_of_a_missing_line_raises(diff_processor): @@ -151,13 +151,13 @@ def test_context_of_a_missing_file_raise def test_find_context_with_full_context(diff_processor): context_of_line_7 = [ - ('unmod', 'line05\n'), - ('unmod', 'line06\n'), - ('unmod', 'line07\n'), - ('add', 'line07a Add after line07\n'), - ('unmod', 'line08\n'), - ('unmod', 'line09\n'), - ('unmod', 'line10\n'), + ('unmod', b'line05\n'), + ('unmod', b'line06\n'), + ('unmod', b'line07\n'), + ('add', b'line07a Add after line07\n'), + ('unmod', b'line08\n'), + ('unmod', b'line09\n'), + ('unmod', b'line10\n'), ] found_line = diff_processor.find_context( 'file.txt', context_of_line_7, offset=3) @@ -167,13 +167,13 @@ def test_find_context_with_full_context( @pytest.mark.parametrize('diff_fixture', ['change-duplicated.diff']) def test_find_context_multiple_times(diff_processor): context = [ - ('unmod', 'line04\n'), - ('unmod', 'line05\n'), - ('unmod', 'line06\n'), - ('add', 'line06a add line\n'), - ('unmod', 'line07\n'), - ('unmod', 'line08\n'), - ('unmod', 'line09\n'), + ('unmod', b'line04\n'), + ('unmod', b'line05\n'), + ('unmod', b'line06\n'), + ('add', b'line06a add line\n'), + ('unmod', b'line07\n'), + ('unmod', b'line08\n'), + ('unmod', b'line09\n'), ] found_line = diff_processor.find_context('file.txt', context, offset=3) assert found_line == [ @@ -185,13 +185,13 @@ def test_find_context_multiple_times(dif @pytest.mark.parametrize('offset', [20, -20, -1, 7]) def test_find_context_offset_param_raises(diff_processor, offset): context_of_line_7 = [ - ('unmod', 'line04\n'), - ('unmod', 'line05\n'), - ('unmod', 'line06\n'), - ('unmod', 'line07\n'), - ('add', 'line07a Add after line07\n'), - ('unmod', 'line08\n'), - ('unmod', 'line09\n'), + ('unmod', b'line04\n'), + ('unmod', b'line05\n'), + ('unmod', b'line06\n'), + ('unmod', b'line07\n'), + ('add', b'line07a Add after line07\n'), + ('unmod', b'line08\n'), + ('unmod', b'line09\n'), ] with pytest.raises(ValueError): diff_processor.find_context( @@ -200,10 +200,10 @@ def test_find_context_offset_param_raise def test_find_context_beginning_of_chunk(diff_processor): context_of_first_line = [ - ('unmod', 'line02\n'), - ('unmod', 'line03\n'), - ('unmod', 'line04\n'), - ('unmod', 'line05\n'), + ('unmod', b'line02\n'), + ('unmod', b'line03\n'), + ('unmod', b'line04\n'), + ('unmod', b'line05\n'), ] found_line = diff_processor.find_context( 'file.txt', context_of_first_line, offset=0) @@ -213,13 +213,13 @@ def test_find_context_beginning_of_chunk @pytest.mark.parametrize('diff_fixture', ['change-in-beginning.diff']) def test_find_context_beginning_of_file(diff_processor): context_of_first_line = [ - ('add', 'line01a Add line after line01\n'), - ('unmod', 'line02\n'), - ('unmod', 'line03\n'), - ('unmod', 'line04\n'), - ('unmod', 'line05\n'), - ('unmod', 'line06\n'), - ('unmod', 'line07\n'), + ('add', b'line01a Add line after line01\n'), + ('unmod', b'line02\n'), + ('unmod', b'line03\n'), + ('unmod', b'line04\n'), + ('unmod', b'line05\n'), + ('unmod', b'line06\n'), + ('unmod', b'line07\n'), ] found_line = diff_processor.find_context( 'file.txt', context_of_first_line, offset=3) @@ -228,10 +228,10 @@ def test_find_context_beginning_of_file( def test_find_context_end_of_chunk(diff_processor): context_of_last_line = [ - ('unmod', 'line10\n'), - ('unmod', 'line11\n'), - ('unmod', 'line12\n'), - ('unmod', 'line13\n'), + ('unmod', b'line10\n'), + ('unmod', b'line11\n'), + ('unmod', b'line12\n'), + ('unmod', b'line13\n'), ] found_line = diff_processor.find_context( 'file.txt', context_of_last_line, offset=3) @@ -242,7 +242,7 @@ def test_find_context_end_of_chunk(diff_ def diff_processor(request, diff_fixture): raw_diff = diffs_store[diff_fixture] diff = GitDiff(raw_diff) - processor = diffs.DiffProcessor(diff) + processor = diffs.DiffProcessor(diff, diff_format='newdiff') processor.prepare() return processor @@ -252,7 +252,7 @@ def diff_fixture(): return 'default.diff' -diff_default = textwrap.dedent(""" +diff_default: bytes = safe_bytes(textwrap.dedent(""" diff --git a/file.txt b/file.txt index 76e4f2e..6f8738f 100644 --- a/file.txt @@ -271,10 +271,10 @@ diff_default = textwrap.dedent(""" line11 line12 line13 -""") +""")) -diff_beginning = textwrap.dedent(""" +diff_beginning: bytes = safe_bytes(textwrap.dedent(""" diff --git a/file.txt b/file.txt index 76e4f2e..47d39f4 100644 --- a/file.txt @@ -288,10 +288,10 @@ diff_beginning = textwrap.dedent(""" line05 line06 line07 -""") +""")) -diff_end = textwrap.dedent(""" +diff_end: bytes = safe_bytes(textwrap.dedent(""" diff --git a/file.txt b/file.txt index 76e4f2e..b1304db 100644 --- a/file.txt @@ -305,10 +305,10 @@ diff_end = textwrap.dedent(""" line38 +line38a Add line after line38 line39 -""") +""")) -diff_duplicated_change = textwrap.dedent(""" +diff_duplicated_change: bytes = safe_bytes(textwrap.dedent(""" diff --git a/file.txt b/file.txt index 76e4f2e..55c2781 100644 --- a/file.txt @@ -341,10 +341,10 @@ diff_duplicated_change = textwrap.dedent line10 line11 line12 -""") +""")) -diff_single_line = textwrap.dedent(""" +diff_single_line: bytes = safe_bytes(textwrap.dedent(""" diff --git a/file_b b/file_b new file mode 100644 index 00000000..915e94ff @@ -352,10 +352,10 @@ diff_single_line = textwrap.dedent(""" +++ b/file_b @@ -0,0 +1 @@ +test_content -""") +""")) -diff_single_line_two_files = textwrap.dedent(""" +diff_single_line_two_files: bytes = safe_bytes(textwrap.dedent(""" diff --git a/file_b b/file_b new file mode 100644 index 00000000..915e94ff @@ -370,7 +370,7 @@ diff_single_line_two_files = textwrap.de +++ b/file_c @@ -0,0 +1 @@ +test_content -""") +""")) diffs_store = { diff --git a/rhodecode/tests/lib/test_encrypt.py b/rhodecode/tests/lib/test_encrypt.py --- a/rhodecode/tests/lib/test_encrypt.py +++ b/rhodecode/tests/lib/test_encrypt.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_ext_json.py b/rhodecode/tests/lib/test_ext_json.py --- a/rhodecode/tests/lib/test_ext_json.py +++ b/rhodecode/tests/lib/test_ext_json.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_graphmod.py b/rhodecode/tests/lib/test_graphmod.py --- a/rhodecode/tests/lib/test_graphmod.py +++ b/rhodecode/tests/lib/test_graphmod.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_helpers.py b/rhodecode/tests/lib/test_helpers.py --- a/rhodecode/tests/lib/test_helpers.py +++ b/rhodecode/tests/lib/test_helpers.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_hooks_base.py b/rhodecode/tests/lib/test_hooks_base.py --- a/rhodecode/tests/lib/test_hooks_base.py +++ b/rhodecode/tests/lib/test_hooks_base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -83,23 +82,35 @@ def hook_extras(user_regular, repo_stub) return extras +class ExtensionMock(mock.Mock): + + def __repr__(self): + return f'ExtensionMock({self._mock_name})' + + @property + def output(self): + return 'MOCK' + + @property + def status(self): + return 0 + + @pytest.mark.parametrize('func, extension, event', [ (hooks_base.pre_push, 'pre_push_extension', 'RepoPrePushEvent'), - (hooks_base.post_push, 'post_pull_extension', 'RepoPushEvent'), + (hooks_base.post_push, 'post_push_extension', 'RepoPushEvent'), (hooks_base.pre_pull, 'pre_pull_extension', 'RepoPrePullEvent'), - (hooks_base.post_pull, 'post_push_extension', 'RepoPullEvent'), + (hooks_base.post_pull, 'post_pull_extension', 'RepoPullEvent'), ]) def test_hooks_propagate(func, extension, event, hook_extras): """ Tests that our hook code propagates to rhodecode extensions and triggers the appropriate event. """ - class ExtensionMock(mock.Mock): - @property - def output(self): - return 'MOCK' extension_mock = ExtensionMock() + extension_mock._mock_name = extension + events_mock = mock.Mock() patches = { 'Repository': mock.Mock(), @@ -115,15 +126,15 @@ def test_hooks_propagate(func, extension func(hook_extras) # Assert that extensions are called and event was fired. - extension_mock.called_once() + extension_mock.assert_called_once() assert_called_with_mock(events_mock.trigger, event) @pytest.mark.parametrize('func, extension, event', [ (hooks_base.pre_push, 'pre_push_extension', 'RepoPrePushEvent'), - (hooks_base.post_push, 'post_pull_extension', 'RepoPushEvent'), + (hooks_base.post_push, 'post_push_extension', 'RepoPushEvent'), (hooks_base.pre_pull, 'pre_pull_extension', 'RepoPrePullEvent'), - (hooks_base.post_pull, 'post_push_extension', 'RepoPullEvent'), + (hooks_base.post_pull, 'post_pull_extension', 'RepoPullEvent'), ]) def test_hooks_propagates_not_on_shadow(func, extension, event, hook_extras): """ @@ -131,7 +142,10 @@ def test_hooks_propagates_not_on_shadow( internal hooks code but not external ones like rhodecode extensions or trigger an event. """ - extension_mock = mock.Mock() + + extension_mock = ExtensionMock() + extension_mock._mock_name = extension + events_mock = mock.Mock() patches = { 'Repository': mock.Mock(), diff --git a/rhodecode/tests/lib/test_hooks_daemon.py b/rhodecode/tests/lib/test_hooks_daemon.py --- a/rhodecode/tests/lib/test_hooks_daemon.py +++ b/rhodecode/tests/lib/test_hooks_daemon.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -18,15 +17,19 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import json import logging import io import mock +import msgpack import pytest from rhodecode.lib import hooks_daemon +from rhodecode.lib.str_utils import safe_bytes from rhodecode.tests.utils import assert_message_in_log +from rhodecode.lib.ext_json import json + +test_proto = hooks_daemon.HooksHttpHandler.MSGPACK_HOOKS_PROTO class TestDummyHooksCallbackDaemon(object): @@ -73,7 +76,10 @@ class TestHooksHttpHandler(object): hooks_daemon.Hooks, data['method'], create=True, return_value=1) with hooks_patcher as hooks_mock: - MockServer(hooks_daemon.HooksHttpHandler, request) + handler = hooks_daemon.HooksHttpHandler + handler.DEFAULT_HOOKS_PROTO = test_proto + handler.wbufsize = 10240 + MockServer(handler, request) hooks_mock.assert_called_once_with(data['extras']) @@ -84,33 +90,51 @@ class TestHooksHttpHandler(object): 'first': 'one', 'second': 2 } + extras = {} + + # patching our _read to return test method and proto used read_patcher = mock.patch.object( hooks_daemon.HooksHttpHandler, '_read_request', - return_value=(rpc_method, {})) + return_value=(test_proto, rpc_method, extras)) + + # patch Hooks instance to return hook_result data on 'test' call hooks_patcher = mock.patch.object( hooks_daemon.Hooks, rpc_method, create=True, return_value=hook_result) with read_patcher, hooks_patcher: - server = MockServer(hooks_daemon.HooksHttpHandler, request) + handler = hooks_daemon.HooksHttpHandler + handler.DEFAULT_HOOKS_PROTO = test_proto + handler.wbufsize = 10240 + server = MockServer(handler, request) - expected_result = json.dumps(hook_result) - assert server.request.output_stream.buflist[-1] == expected_result + expected_result = hooks_daemon.HooksHttpHandler.serialize_data(hook_result) + + server.request.output_stream.seek(0) + assert server.request.output_stream.readlines()[-1] == expected_result def test_exception_is_returned_in_response(self): request = self._generate_post_request({}) rpc_method = 'test' + read_patcher = mock.patch.object( hooks_daemon.HooksHttpHandler, '_read_request', - return_value=(rpc_method, {})) + return_value=(test_proto, rpc_method, {})) + hooks_patcher = mock.patch.object( hooks_daemon.Hooks, rpc_method, create=True, side_effect=Exception('Test exception')) with read_patcher, hooks_patcher: - server = MockServer(hooks_daemon.HooksHttpHandler, request) + handler = hooks_daemon.HooksHttpHandler + handler.DEFAULT_HOOKS_PROTO = test_proto + handler.wbufsize = 10240 + server = MockServer(handler, request) - org_exc = json.loads(server.request.output_stream.buflist[-1]) + server.request.output_stream.seek(0) + data = server.request.output_stream.readlines() + msgpack_data = b''.join(data[5:]) + org_exc = hooks_daemon.HooksHttpHandler.deserialize_data(msgpack_data) expected_result = { 'exception': 'Exception', 'exception_traceback': org_exc['exception_traceback'], @@ -125,17 +149,22 @@ class TestHooksHttpHandler(object): fake_date = '1/Nov/2015 00:00:00' date_patcher = mock.patch.object( handler, 'log_date_time_string', return_value=fake_date) + with date_patcher, caplog.at_level(logging.DEBUG): handler.log_message('Some message %d, %s', 123, 'string') - expected_message = "HOOKS: {} - - [{}] Some message 123, string".format(ip_port, fake_date) + expected_message = f"HOOKS: {ip_port} - - [{fake_date}] Some message 123, string" assert_message_in_log( caplog.records, expected_message, levelno=logging.DEBUG, module='hooks_daemon') - def _generate_post_request(self, data): - payload = json.dumps(data) - return 'POST / HTTP/1.0\nContent-Length: {}\n\n{}'.format( + def _generate_post_request(self, data, proto=test_proto): + if proto == hooks_daemon.HooksHttpHandler.MSGPACK_HOOKS_PROTO: + payload = msgpack.packb(data) + else: + payload = json.dumps(data) + + return b'POST / HTTP/1.0\nContent-Length: %d\n\n%b' % ( len(payload), payload) @@ -190,9 +219,9 @@ class TestHttpHooksCallbackDaemon(object assert daemon._daemon == tcp_server _, port = tcp_server.server_address - expected_uri = '{}:{}'.format('127.0.0.1', port) - msg = 'HOOKS: {} Preparing HTTP callback daemon registering ' \ - 'hook object: rhodecode.lib.hooks_daemon.HooksHttpHandler'.format(expected_uri) + + msg = f"HOOKS: 127.0.0.1:{port} Preparing HTTP callback daemon registering " \ + f"hook object: " assert_message_in_log( caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon') @@ -205,8 +234,8 @@ class TestHttpHooksCallbackDaemon(object expected_uri = '{}:{}'.format('127.0.0.1', port) assert daemon.hooks_uri == expected_uri - msg = 'HOOKS: {} Preparing HTTP callback daemon registering ' \ - 'hook object: rhodecode.lib.hooks_daemon.HooksHttpHandler'.format(expected_uri) + msg = f"HOOKS: 127.0.0.1:{port} Preparing HTTP callback daemon registering " \ + f"hook object: " assert_message_in_log( caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon') @@ -318,16 +347,19 @@ class TestPrepareHooksDaemon(object): class MockRequest(object): + def __init__(self, request): self.request = request - self.input_stream = io.StringIO(b'{}'.format(self.request)) - self.output_stream = io.StringIO() + self.input_stream = io.BytesIO(safe_bytes(self.request)) + self.output_stream = io.BytesIO() # make it un-closable for testing invesitagion + self.output_stream.close = lambda: None def makefile(self, mode, *args, **kwargs): return self.output_stream if mode == 'wb' else self.input_stream class MockServer(object): + def __init__(self, handler_cls, request): ip_port = ('0.0.0.0', 8888) self.request = MockRequest(request) @@ -339,4 +371,5 @@ class MockServer(object): def tcp_server(): server = mock.Mock() server.server_address = ('127.0.0.1', 8881) + server.wbufsize = 1024 return server diff --git a/rhodecode/tests/lib/test_jsonalchemy.py b/rhodecode/tests/lib/test_jsonalchemy.py --- a/rhodecode/tests/lib/test_jsonalchemy.py +++ b/rhodecode/tests/lib/test_jsonalchemy.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # @@ -81,7 +81,7 @@ def test_mutation_types_with_nullable(en obj_nulls = DummyModel(name='nulls') obj_stuff = DummyModel( - name='stuff', json_list=[1,2,3], json_dict={'a': 5}, json_obj=9) + name='stuff', json_list=[1, 2, 3], json_dict={'a': 5}, json_obj=9) session.add(obj_nulls) session.add(obj_stuff) @@ -90,7 +90,7 @@ def test_mutation_types_with_nullable(en assert engine.execute( "select * from some_table where name = 'nulls';").first() == ( - (u'nulls', None, None, None) + ('nulls', None, None, None) ) ret_nulls = session.query(DummyModel).get('nulls') assert ret_nulls.json_list == [] @@ -99,7 +99,7 @@ def test_mutation_types_with_nullable(en assert engine.execute( "select * from some_table where name = 'stuff';").first() == ( - (u'stuff', u'[1, 2, 3]', u'{"a": 5}', u'9') + ('stuff', '[1,2,3]', '{"a":5}', '9') ) ret_stuff = session.query(DummyModel).get('stuff') assert ret_stuff.json_list == [1, 2, 3] diff --git a/rhodecode/tests/lib/test_libs.py b/rhodecode/tests/lib/test_libs.py --- a/rhodecode/tests/lib/test_libs.py +++ b/rhodecode/tests/lib/test_libs.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -142,7 +141,8 @@ def test_age(age_args, expected, kw, bas from rhodecode.lib.utils2 import age from dateutil import relativedelta n = datetime.datetime(year=2012, month=5, day=17) - delt = lambda *args, **kwargs: relativedelta.relativedelta(*args, **kwargs) + def delt(*args, **kwargs): + return relativedelta.relativedelta(*args, **kwargs) def translate(elem): return elem.interpolate() @@ -174,7 +174,8 @@ def test_age_in_future(age_args, expecte from rhodecode.lib.utils2 import age from dateutil import relativedelta n = datetime.datetime(year=2012, month=5, day=17) - delt = lambda *args, **kwargs: relativedelta.relativedelta(*args, **kwargs) + def delt(*args, **kwargs): + return relativedelta.relativedelta(*args, **kwargs) def translate(elem): return elem.interpolate() diff --git a/rhodecode/tests/lib/test_mako_emails.py b/rhodecode/tests/lib/test_mako_emails.py --- a/rhodecode/tests/lib/test_mako_emails.py +++ b/rhodecode/tests/lib/test_mako_emails.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_markup_renderer.py b/rhodecode/tests/lib/test_markup_renderer.py --- a/rhodecode/tests/lib/test_markup_renderer.py +++ b/rhodecode/tests/lib/test_markup_renderer.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -69,7 +68,7 @@ def test_markdown_inline_html(): xss_md = '\n'.join(['> onload="javascript:alert()" href="https://rhodecode.com">link']) rendered_html = MarkupRenderer.markdown(xss_md) - assert 'link' in rendered_html + assert 'link' in rendered_html def test_markdown_bleach_renders_correct(): @@ -301,7 +300,10 @@ console.log(s); ```python s = "Python syntax highlighting" -print s +print(s) + +class Orm(object): + pass ``` ``` @@ -317,7 +319,10 @@ alert(s); ```python s = "Python syntax highlighting" -print s +print(s) + +class Orm(object): + pass ``` ``` diff --git a/rhodecode/tests/lib/test_search.py b/rhodecode/tests/lib/test_search.py --- a/rhodecode/tests/lib/test_search.py +++ b/rhodecode/tests/lib/test_search.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_search_utils.py b/rhodecode/tests/lib/test_search_utils.py --- a/rhodecode/tests/lib/test_search_utils.py +++ b/rhodecode/tests/lib/test_search_utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/lib/test_system_info.py b/rhodecode/tests/lib/test_system_info.py --- a/rhodecode/tests/lib/test_system_info.py +++ b/rhodecode/tests/lib/test_system_info.py @@ -1,4 +1,4 @@ -import py.test + from rhodecode.lib.system_info import get_system_info @@ -6,10 +6,3 @@ from rhodecode.lib.system_info import ge def test_system_info(app): info = get_system_info({}) assert info['load']['value']['15_min'] != 'NOT AVAILABLE' - - -def test_system_info_without_psutil(monkeypatch, app): - import rhodecode.lib.system_info - monkeypatch.setattr(rhodecode.lib.system_info, 'psutil', None) - info = get_system_info({}) - assert info['load']['value']['15_min'] == 'NOT AVAILABLE' diff --git a/rhodecode/tests/lib/test_utils.py b/rhodecode/tests/lib/test_utils.py --- a/rhodecode/tests/lib/test_utils.py +++ b/rhodecode/tests/lib/test_utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -18,7 +17,6 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import json import multiprocessing import os @@ -28,16 +26,16 @@ import pytest from rhodecode.lib import caching_query from rhodecode.lib import utils -from rhodecode.lib.utils2 import md5 +from rhodecode.lib.str_utils import safe_bytes from rhodecode.model import settings from rhodecode.model import db from rhodecode.model import meta from rhodecode.model.repo import RepoModel from rhodecode.model.repo_group import RepoGroupModel -from rhodecode.model.scm import ScmModel from rhodecode.model.settings import UiSetting, SettingsModel from rhodecode.tests.fixture import Fixture - +from rhodecode_tools.lib.hash_utils import md5_safe +from rhodecode.lib.ext_json import json fixture = Fixture() @@ -67,11 +65,17 @@ def disable_hooks(request, hooks): # Invalidate cache ui_settings = session.query(db.RhodeCodeUi).options( caching_query.FromCache('sql_cache_short', 'get_hg_ui_settings')) - ui_settings.invalidate() + + meta.cache.invalidate( + ui_settings, {}, + caching_query.FromCache('sql_cache_short', 'get_hg_ui_settings')) ui_settings = session.query(db.RhodeCodeUi).options( caching_query.FromCache('sql_cache_short', 'get_hook_settings')) - ui_settings.invalidate() + + meta.cache.invalidate( + ui_settings, {}, + caching_query.FromCache('sql_cache_short', 'get_hook_settings')) @request.addfinalizer def rollback(): @@ -110,7 +114,7 @@ def test_make_db_config_hg_hooks(baseapp config = utils.make_db_config() hooks = extract_hooks(config) - assert set(hooks.iterkeys()).intersection(HG_HOOKS) == set(expected_hooks) + assert set(hooks.keys()).intersection(HG_HOOKS) == set(expected_hooks) @pytest.mark.parametrize('disabled_hooks,expected_hooks', [ @@ -180,11 +184,19 @@ def _stub_git_repo(repo_path): repo_path.ensure('.git', dir=True) -@pytest.mark.parametrize('str_class', [str, bytes], ids=['str', 'bytes']) -def test_get_dirpaths_returns_all_paths(tmpdir, str_class): +def test_get_dirpaths_returns_all_paths_on_str(tmpdir): tmpdir.ensure('test-file') - dirpaths = utils._get_dirpaths(str_class(tmpdir)) - assert dirpaths == ['test-file'] + tmpdir.ensure('test-file-1') + tmp_path = str(tmpdir) + dirpaths = utils.get_dirpaths(tmp_path) + assert list(sorted(dirpaths)) == ['test-file', 'test-file-1'] + + +def test_get_dirpaths_returns_all_paths_on_bytes(tmpdir): + tmpdir.ensure('test-file-bytes') + tmp_path = str(tmpdir) + dirpaths = utils.get_dirpaths(safe_bytes(tmp_path)) + assert list(sorted(dirpaths)) == [b'test-file-bytes'] def test_get_dirpaths_returns_all_paths_bytes( @@ -192,7 +204,7 @@ def test_get_dirpaths_returns_all_paths_ if platform_encodes_filenames: pytest.skip("This platform seems to encode filenames.") tmpdir.ensure('repo-a-umlaut-\xe4') - dirpaths = utils._get_dirpaths(str(tmpdir)) + dirpaths = utils.get_dirpaths(str(tmpdir)) assert dirpaths == ['repo-a-umlaut-\xe4'] @@ -201,8 +213,8 @@ def test_get_dirpaths_skips_paths_it_can if platform_encodes_filenames: pytest.skip("This platform seems to encode filenames.") path_with_latin1 = 'repo-a-umlaut-\xe4' - tmpdir.ensure(path_with_latin1) - dirpaths = utils._get_dirpaths(unicode(tmpdir)) + tmp_path = str(tmpdir.ensure(path_with_latin1)) + dirpaths = utils.get_dirpaths(tmp_path) assert dirpaths == [] @@ -266,7 +278,8 @@ def test_repo2db_mapper_installs_hooks_f class TestPasswordChanged(object): - def setup(self): + + def setup_method(self): self.session = { 'rhodecode_user': { 'password': '0cc175b9c0f1b6a831c399e269772661' @@ -282,7 +295,7 @@ class TestPasswordChanged(object): assert result is False def test_returns_false_if_password_was_not_changed(self): - self.session['rhodecode_user']['password'] = md5( + self.session['rhodecode_user']['password'] = md5_safe( self.auth_user.password) result = utils.password_changed(self.auth_user, self.session) assert result is False @@ -406,7 +419,7 @@ class TestConfigDataFromDb(object): class TestIsDirWritable(object): def test_returns_false_when_not_writable(self): - with mock.patch('__builtin__.open', side_effect=OSError): + with mock.patch('builtins.open', side_effect=OSError): assert not utils._is_dir_writable('/stub-path') def test_returns_true_when_writable(self, tmpdir): diff --git a/rhodecode/tests/load/api.py b/rhodecode/tests/load/api.py --- a/rhodecode/tests/load/api.py +++ b/rhodecode/tests/load/api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -18,10 +17,12 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import json + import requests +from rhodecode.lib.ext_json import sjson as json + class ApiError(Exception): """Error when accessing the API.""" diff --git a/rhodecode/tests/load/http_performance.py b/rhodecode/tests/load/http_performance.py --- a/rhodecode/tests/load/http_performance.py +++ b/rhodecode/tests/load/http_performance.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/load/profile-mem.py b/rhodecode/tests/load/profile-mem.py --- a/rhodecode/tests/load/profile-mem.py +++ b/rhodecode/tests/load/profile-mem.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/load/profile.py b/rhodecode/tests/load/profile.py --- a/rhodecode/tests/load/profile.py +++ b/rhodecode/tests/load/profile.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -35,7 +34,9 @@ import subprocess import sys import time import traceback -import urllib.request, urllib.parse, urllib.error +import urllib.request +import urllib.parse +import urllib.error PROFILING_INTERVAL = 5 RC_WEBSITE = "http://localhost:5001/" diff --git a/rhodecode/tests/load/time_urls.py b/rhodecode/tests/load/time_urls.py --- a/rhodecode/tests/load/time_urls.py +++ b/rhodecode/tests/load/time_urls.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/load/vcs_performance.py b/rhodecode/tests/load/vcs_performance.py --- a/rhodecode/tests/load/vcs_performance.py +++ b/rhodecode/tests/load/vcs_performance.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/__init__.py b/rhodecode/tests/models/__init__.py --- a/rhodecode/tests/models/__init__.py +++ b/rhodecode/tests/models/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/common.py b/rhodecode/tests/models/common.py --- a/rhodecode/tests/models/common.py +++ b/rhodecode/tests/models/common.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/schemas/test_gist_schema.py b/rhodecode/tests/models/schemas/test_gist_schema.py --- a/rhodecode/tests/models/schemas/test_gist_schema.py +++ b/rhodecode/tests/models/schemas/test_gist_schema.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # @@ -60,9 +60,9 @@ class TestGistSchema(object): lifetime_options=[1, 2, 3] ) nodes = [{ - 'filename': 'foobar', - 'filename_org': 'foobar', - 'content': 'content', + 'filename': b'foobar', + 'filename_org': b'foobar', + 'content': b'content', 'mimetype': 'xx' }] schema_data = schema.deserialize(dict( @@ -80,9 +80,9 @@ class TestGistSchema(object): convert_nodes=True ) nodes = [{ - 'filename': 'foobar', + 'filename': b'foobar', 'filename_org': None, - 'content': 'content', + 'content': b'content', 'mimetype': 'xx' }] schema_data = schema.deserialize(dict( diff --git a/rhodecode/tests/models/schemas/test_integration_schema.py b/rhodecode/tests/models/schemas/test_integration_schema.py --- a/rhodecode/tests/models/schemas/test_integration_schema.py +++ b/rhodecode/tests/models/schemas/test_integration_schema.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/schemas/test_repo_group_schema.py b/rhodecode/tests/models/schemas/test_repo_group_schema.py --- a/rhodecode/tests/models/schemas/test_repo_group_schema.py +++ b/rhodecode/tests/models/schemas/test_repo_group_schema.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/schemas/test_repo_schema.py b/rhodecode/tests/models/schemas/test_repo_schema.py --- a/rhodecode/tests/models/schemas/test_repo_schema.py +++ b/rhodecode/tests/models/schemas/test_repo_schema.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/schemas/test_schema_types.py b/rhodecode/tests/models/schemas/test_schema_types.py --- a/rhodecode/tests/models/schemas/test_schema_types.py +++ b/rhodecode/tests/models/schemas/test_schema_types.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/schemas/test_user_schema.py b/rhodecode/tests/models/schemas/test_user_schema.py --- a/rhodecode/tests/models/schemas/test_user_schema.py +++ b/rhodecode/tests/models/schemas/test_user_schema.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/schemas/test_user_usergroup_types.py b/rhodecode/tests/models/schemas/test_user_usergroup_types.py --- a/rhodecode/tests/models/schemas/test_user_usergroup_types.py +++ b/rhodecode/tests/models/schemas/test_user_usergroup_types.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/settings/__init__.py b/rhodecode/tests/models/settings/__init__.py --- a/rhodecode/tests/models/settings/__init__.py +++ b/rhodecode/tests/models/settings/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/settings/test_issue_tracker.py b/rhodecode/tests/models/settings/test_issue_tracker.py --- a/rhodecode/tests/models/settings/test_issue_tracker.py +++ b/rhodecode/tests/models/settings/test_issue_tracker.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/settings/test_settings.py b/rhodecode/tests/models/settings/test_settings.py --- a/rhodecode/tests/models/settings/test_settings.py +++ b/rhodecode/tests/models/settings/test_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/settings/test_vcs_settings.py b/rhodecode/tests/models/settings/test_vcs_settings.py --- a/rhodecode/tests/models/settings/test_vcs_settings.py +++ b/rhodecode/tests/models/settings/test_vcs_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/test_changeset_status.py b/rhodecode/tests/models/test_changeset_status.py --- a/rhodecode/tests/models/test_changeset_status.py +++ b/rhodecode/tests/models/test_changeset_status.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/test_changeset_status_group_voting.py b/rhodecode/tests/models/test_changeset_status_group_voting.py --- a/rhodecode/tests/models/test_changeset_status_group_voting.py +++ b/rhodecode/tests/models/test_changeset_status_group_voting.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/test_comment.py b/rhodecode/tests/models/test_comment.py --- a/rhodecode/tests/models/test_comment.py +++ b/rhodecode/tests/models/test_comment.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -116,7 +115,8 @@ def assert_inline_comments_order(query): """ Sorting by ID will make sure that the latest comments are at the bottom. """ - order_by = query._order_by + + order_by = query._order_by_clauses assert order_by assert len(order_by) == 1 assert str(order_by[0]) == 'changeset_comments.comment_id ASC' diff --git a/rhodecode/tests/models/test_db.py b/rhodecode/tests/models/test_db.py --- a/rhodecode/tests/models/test_db.py +++ b/rhodecode/tests/models/test_db.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -33,15 +32,14 @@ class TestModelReprImplementation(object def test_repr_without_id(self, DBModel, klass, id_attr): instance = DBModel() - expected_repr = '' % (klass, id(instance)) + expected_repr = f'' assert repr(instance) == expected_repr def test_repr_with_id(self, DBModel, klass, id_attr): test_id = random.randint(1, 10) instance = DBModel() setattr(instance, id_attr, test_id) - expected_repr = ( - '' % (klass, test_id)) + expected_repr = f'' assert repr(instance) == expected_repr diff --git a/rhodecode/tests/models/test_gist.py b/rhodecode/tests/models/test_gist.py --- a/rhodecode/tests/models/test_gist.py +++ b/rhodecode/tests/models/test_gist.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -35,8 +34,8 @@ class TestGistModel(object): return_value=repo.scm_instance()) with create_repo_patch as create_repo_mock: gist_mapping = { - 'filename.txt': { - 'content': 'Test content' + b'filename.txt': { + 'content': b'Test content' } } model.create('Test description', owner, gist_mapping) diff --git a/rhodecode/tests/models/test_notifications.py b/rhodecode/tests/models/test_notifications.py --- a/rhodecode/tests/models/test_notifications.py +++ b/rhodecode/tests/models/test_notifications.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -41,20 +40,20 @@ class TestNotifications(object): def create_users(self, request, app): Session.remove() self.u1 = UserModel().create_or_update( - username=u'u1', password=u'qweqwe', - email=u'u1@rhodecode.org', firstname=u'u1', lastname=u'u1') + username='u1', password='qweqwe', + email='u1@rhodecode.org', firstname='u1', lastname='u1') Session().commit() self.u1 = self.u1.user_id self.u2 = UserModel().create_or_update( - username=u'u2', password=u'qweqwe', - email=u'u2@rhodecode.org', firstname=u'u2', lastname=u'u2') + username='u2', password='qweqwe', + email='u2@rhodecode.org', firstname='u2', lastname='u2') Session().commit() self.u2 = self.u2.user_id self.u3 = UserModel().create_or_update( - username=u'u3', password=u'qweqwe', - email=u'u3@rhodecode.org', firstname=u'u3', lastname=u'u3') + username='u3', password='qweqwe', + email='u3@rhodecode.org', firstname='u3', lastname='u3') Session().commit() self.u3 = self.u3.user_id self.destroy_users.add('u1') @@ -73,8 +72,8 @@ class TestNotifications(object): def test_create_notification(self): usrs = [self.u1, self.u2] notification = NotificationModel().create( - created_by=self.u1, notification_subject=u'subj', - notification_body=u'hi there', recipients=usrs) + created_by=self.u1, notification_subject='subj', + notification_body='hi there', recipients=usrs) Session().commit() u1 = User.get(self.u1) u2 = User.get(self.u2) @@ -93,32 +92,31 @@ class TestNotifications(object): def test_create_notification_fails_for_invalid_recipients(self): with pytest.raises(Exception): NotificationModel().create( - created_by=self.u1, notification_subject=u'subj', - notification_body=u'hi there', recipients=['bad_user_id']) + created_by=self.u1, notification_subject='subj', + notification_body='hi there', recipients=['bad_user_id']) with pytest.raises(Exception): NotificationModel().create( - created_by=self.u1, notification_subject=u'subj', - notification_body=u'hi there', recipients=[]) + created_by=self.u1, notification_subject='subj', + notification_body='hi there', recipients=[]) def test_user_notifications(self): notification1 = NotificationModel().create( - created_by=self.u1, notification_subject=u'subj', - notification_body=u'hi there1', recipients=[self.u3]) + created_by=self.u1, notification_subject='subj', + notification_body='hi there1', recipients=[self.u3]) Session().commit() notification2 = NotificationModel().create( - created_by=self.u1, notification_subject=u'subj', - notification_body=u'hi there2', recipients=[self.u3]) + created_by=self.u1, notification_subject='subj', + notification_body='hi there2', recipients=[self.u3]) Session().commit() u3 = Session().query(User).get(self.u3) - assert sorted([x.notification for x in u3.notifications]) ==\ - sorted([notification2, notification1]) + assert [x.notification for x in u3.notifications] == [notification2, notification1] def test_delete_notifications(self): notification = NotificationModel().create( - created_by=self.u1, notification_subject=u'title', - notification_body=u'hi there3', + created_by=self.u1, notification_subject='title', + notification_body='hi there3', recipients=[self.u3, self.u1, self.u2]) Session().commit() notifications = Notification.query().all() @@ -136,8 +134,8 @@ class TestNotifications(object): def test_delete_association(self): notification = NotificationModel().create( - created_by=self.u1, notification_subject=u'title', - notification_body=u'hi there3', + created_by=self.u1, notification_subject='title', + notification_body='hi there3', recipients=[self.u3, self.u1, self.u2]) Session().commit() @@ -180,8 +178,8 @@ class TestNotifications(object): def test_notification_counter(self): NotificationModel().create( - created_by=self.u1, notification_subject=u'title', - notification_body=u'hi there_delete', recipients=[self.u3, self.u1]) + created_by=self.u1, notification_subject='title', + notification_body='hi there_delete', recipients=[self.u3, self.u1]) Session().commit() # creator has it's own notification marked as read @@ -190,8 +188,8 @@ class TestNotifications(object): assert NotificationModel().get_unread_cnt_for_user(self.u3) == 1 NotificationModel().create( - created_by=self.u1, notification_subject=u'title', - notification_body=u'hi there3', + created_by=self.u1, notification_subject='title', + notification_body='hi there3', recipients=[self.u3, self.u1, self.u2]) Session().commit() # creator has it's own notification marked as read diff --git a/rhodecode/tests/models/test_permissions.py b/rhodecode/tests/models/test_permissions.py --- a/rhodecode/tests/models/test_permissions.py +++ b/rhodecode/tests/models/test_permissions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/test_pullrequest.py b/rhodecode/tests/models/test_pullrequest.py --- a/rhodecode/tests/models/test_pullrequest.py +++ b/rhodecode/tests/models/test_pullrequest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -23,7 +22,6 @@ import pytest import textwrap import rhodecode -from rhodecode.lib.utils2 import safe_unicode from rhodecode.lib.vcs.backends import get_backend from rhodecode.lib.vcs.backends.base import ( MergeResponse, MergeFailureReason, Reference) @@ -34,7 +32,7 @@ from rhodecode.model.db import PullReque from rhodecode.model.pull_request import PullRequestModel from rhodecode.model.user import UserModel from rhodecode.tests import TEST_USER_ADMIN_LOGIN - +from rhodecode.lib.str_utils import safe_str pytestmark = [ pytest.mark.backends("git", "hg"), @@ -299,10 +297,10 @@ class TestPullRequestModel(object): u'Merge pull request !{pr_id} from {source_repo} {source_ref_name}' u'\n\n {pr_title}'.format( pr_id=pull_request.pull_request_id, - source_repo=safe_unicode( + source_repo=safe_str( pull_request.source_repo.scm_instance().name), source_ref_name=pull_request.source_ref_parts.name, - pr_title=safe_unicode(pull_request.title) + pr_title=safe_str(pull_request.title) ) ) self.merge_mock.assert_called_with( @@ -343,10 +341,10 @@ class TestPullRequestModel(object): u'Merge pull request !{pr_id} from {source_repo} {source_ref_name}' u'\n\n {pr_title}'.format( pr_id=pull_request.pull_request_id, - source_repo=safe_unicode( + source_repo=safe_str( pull_request.source_repo.scm_instance().name), source_ref_name=pull_request.source_ref_parts.name, - pr_title=safe_unicode(pull_request.title) + pr_title=safe_str(pull_request.title) ) ) self.merge_mock.assert_called_with( @@ -382,10 +380,10 @@ class TestPullRequestModel(object): u'Merge pull request !{pr_id} from {source_repo} {source_ref_name}' u'\n\n {pr_title}'.format( pr_id=pull_request.pull_request_id, - source_repo=safe_unicode( + source_repo=safe_str( pull_request.source_repo.scm_instance().name), source_ref_name=pull_request.source_ref_parts.name, - pr_title=safe_unicode(pull_request.title) + pr_title=safe_str(pull_request.title) ) ) self.merge_mock.assert_called_with( @@ -423,7 +421,7 @@ class TestPullRequestModel(object): diff = PullRequestModel()._get_diff_from_pr_or_version( source_repo, source_ref_id, target_ref_id, hide_whitespace_changes=False, diff_context=6) - assert 'file_1' in diff.raw + assert b'file_1' in diff.raw.tobytes() def test_generate_title_returns_unicode(self): title = PullRequestModel().generate_pullrequest_title( @@ -431,7 +429,7 @@ class TestPullRequestModel(object): source_ref='source-ref-dummy', target='target-dummy', ) - assert type(title) == unicode + assert type(title) == str @pytest.mark.parametrize('title, has_wip', [ ('hello', False), @@ -607,8 +605,8 @@ class TestUpdateCommentHandling(object): def test_comment_stays_unflagged_on_unchanged_diff(self, pr_util): commits = [ {'message': 'a'}, - {'message': 'b', 'added': [FileNode('file_b', 'test_content\n')]}, - {'message': 'c', 'added': [FileNode('file_c', 'test_content\n')]}, + {'message': 'b', 'added': [FileNode(b'file_b', b'test_content\n')]}, + {'message': 'c', 'added': [FileNode(b'file_c', b'test_content\n')]}, ] pull_request = pr_util.create_pull_request( commits=commits, target_head='a', source_head='b', revisions=['b']) @@ -618,13 +616,12 @@ class TestUpdateCommentHandling(object): assert_inline_comments(pull_request, visible=1, outdated=0) def test_comment_stays_unflagged_on_change_above(self, pr_util): - original_content = ''.join( - ['line {}\n'.format(x) for x in range(1, 11)]) - updated_content = 'new_line_at_top\n' + original_content + original_content = b''.join((b'line %d\n' % x for x in range(1, 11))) + updated_content = b'new_line_at_top\n' + original_content commits = [ {'message': 'a'}, - {'message': 'b', 'added': [FileNode('file_b', original_content)]}, - {'message': 'c', 'changed': [FileNode('file_b', updated_content)]}, + {'message': 'b', 'added': [FileNode(b'file_b', original_content)]}, + {'message': 'c', 'changed': [FileNode(b'file_b', updated_content)]}, ] pull_request = pr_util.create_pull_request( commits=commits, target_head='a', source_head='b', revisions=['b']) @@ -638,12 +635,12 @@ class TestUpdateCommentHandling(object): assert comment.line_no == u'n9' def test_comment_stays_unflagged_on_change_below(self, pr_util): - original_content = ''.join(['line {}\n'.format(x) for x in range(10)]) - updated_content = original_content + 'new_line_at_end\n' + original_content = b''.join([b'line %d\n' % x for x in range(10)]) + updated_content = original_content + b'new_line_at_end\n' commits = [ {'message': 'a'}, - {'message': 'b', 'added': [FileNode('file_b', original_content)]}, - {'message': 'c', 'changed': [FileNode('file_b', updated_content)]}, + {'message': 'b', 'added': [FileNode(b'file_b', original_content)]}, + {'message': 'c', 'changed': [FileNode(b'file_b', updated_content)]}, ] pull_request = pr_util.create_pull_request( commits=commits, target_head='a', source_head='b', revisions=['b']) @@ -654,17 +651,17 @@ class TestUpdateCommentHandling(object): @pytest.mark.parametrize('line_no', ['n4', 'o4', 'n10', 'o9']) def test_comment_flagged_on_change_around_context(self, pr_util, line_no): - base_lines = ['line {}\n'.format(x) for x in range(1, 13)] + base_lines = [b'line %d\n' % x for x in range(1, 13)] change_lines = list(base_lines) - change_lines.insert(6, 'line 6a added\n') + change_lines.insert(6, b'line 6a added\n') # Changes on the last line of sight update_lines = list(change_lines) - update_lines[0] = 'line 1 changed\n' - update_lines[-1] = 'line 12 changed\n' + update_lines[0] = b'line 1 changed\n' + update_lines[-1] = b'line 12 changed\n' def file_b(lines): - return FileNode('file_b', ''.join(lines)) + return FileNode(b'file_b', b''.join(lines)) commits = [ {'message': 'a', 'added': [file_b(base_lines)]}, @@ -681,14 +678,14 @@ class TestUpdateCommentHandling(object): assert_inline_comments(pull_request, visible=0, outdated=1) @pytest.mark.parametrize("change, content", [ - ('changed', 'changed\n'), - ('removed', ''), - ], ids=['changed', 'removed']) + ('changed', b'changed\n'), + ('removed', b''), + ], ids=['changed', b'removed']) def test_comment_flagged_on_change(self, pr_util, change, content): commits = [ {'message': 'a'}, - {'message': 'b', 'added': [FileNode('file_b', 'test_content\n')]}, - {'message': 'c', change: [FileNode('file_b', content)]}, + {'message': 'b', 'added': [FileNode(b'file_b', b'test_content\n')]}, + {'message': 'c', change: [FileNode(b'file_b', content)]}, ] pull_request = pr_util.create_pull_request( commits=commits, target_head='a', source_head='b', revisions=['b']) @@ -706,9 +703,9 @@ class TestUpdateChangedFiles(object): commits = [ {'message': 'a'}, {'message': 'b', - 'added': [FileNode('file_b', 'test_content b\n')]}, + 'added': [FileNode(b'file_b', b'test_content b\n')]}, {'message': 'c', - 'added': [FileNode('file_c', 'test_content c\n')]}, + 'added': [FileNode(b'file_c', b'test_content c\n')]}, ] # open a PR from a to b, adding file_b pull_request = pr_util.create_pull_request( @@ -728,11 +725,11 @@ class TestUpdateChangedFiles(object): commits = [ {'message': 'a'}, {'message': 'b', - 'added': [FileNode('file_b', 'test_content b\n')]}, + 'added': [FileNode(b'file_b', b'test_content b\n')]}, {'message': 'c', - 'changed': [FileNode('file_b', 'test_content b modified\n')]}, + 'changed': [FileNode(b'file_b', b'test_content b modified\n')]}, {'message': 'd', - 'changed': [FileNode('file_b', 'test_content b\n')]}, + 'changed': [FileNode(b'file_b', b'test_content b\n')]}, ] # open a PR from a to b, adding file_b pull_request = pr_util.create_pull_request( @@ -762,13 +759,13 @@ class TestUpdateChangedFiles(object): commits = [ {'message': 'a'}, {'message': 'b', 'added': [ - FileNode('file_a', 'test_content a\n'), - FileNode('file_b', 'test_content b\n'), - FileNode('file_c', 'test_content c\n')]}, + FileNode(b'file_a', b'test_content a\n'), + FileNode(b'file_b', b'test_content b\n'), + FileNode(b'file_c', b'test_content c\n')]}, {'message': 'c', 'changed': [ - FileNode('file_a', 'test_content a changed\n'), - FileNode('file_b', 'test_content b changed\n'), - FileNode('file_c', 'test_content c changed\n')]}, + FileNode(b'file_a', b'test_content a changed\n'), + FileNode(b'file_b', b'test_content b changed\n'), + FileNode(b'file_c', b'test_content c changed\n')]}, ] # open a PR from a to b, changing 3 files pull_request = pr_util.create_pull_request( @@ -787,13 +784,13 @@ class TestUpdateChangedFiles(object): commits = [ {'message': 'a'}, {'message': 'b', 'added': [ - FileNode('file_a', 'test_content a\n'), - FileNode('file_b', 'test_content b\n'), - FileNode('file_c', 'test_content c\n')]}, + FileNode(b'file_a', b'test_content a\n'), + FileNode(b'file_b', b'test_content b\n'), + FileNode(b'file_c', b'test_content c\n')]}, {'message': 'c', 'removed': [ - FileNode('file_a', 'test_content a changed\n'), - FileNode('file_b', 'test_content b changed\n'), - FileNode('file_c', 'test_content c changed\n')]}, + FileNode(b'file_a', b'test_content a changed\n'), + FileNode(b'file_b', b'test_content b changed\n'), + FileNode(b'file_c', b'test_content c changed\n')]}, ] # open a PR from a to b, removing 3 files pull_request = pr_util.create_pull_request( diff --git a/rhodecode/tests/models/test_pullrequest_git.py b/rhodecode/tests/models/test_pullrequest_git.py --- a/rhodecode/tests/models/test_pullrequest_git.py +++ b/rhodecode/tests/models/test_pullrequest_git.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -47,8 +46,8 @@ class TestGetDiffForPrOrVersion(object): def _prepare_pull_request(self, pr_util): commits = [ {'message': 'a'}, - {'message': 'b', 'added': [FileNode('file_b', 'test_content\n')]}, - {'message': 'c', 'added': [FileNode('file_c', 'test_content\n')]}, + {'message': 'b', 'added': [FileNode(b'file_b', 'test_content\n')]}, + {'message': 'c', 'added': [FileNode(b'file_c', 'test_content\n')]}, ] pull_request = pr_util.create_pull_request( commits=commits, target_head='a', source_head='c', @@ -62,7 +61,7 @@ class TestGetDiffForPrOrVersion(object): diff = PullRequestModel()._get_diff_from_pr_or_version( source_repo, source_ref_id, target_ref_id, hide_whitespace_changes=False, diff_context=6) - assert 'file_b' in diff.raw + assert b'file_b' in diff.raw.tobytes() def assert_commit_cannot_be_accessed( self, removed_commit_id, pull_request): diff --git a/rhodecode/tests/models/test_repo_groups.py b/rhodecode/tests/models/test_repo_groups.py --- a/rhodecode/tests/models/test_repo_groups.py +++ b/rhodecode/tests/models/test_repo_groups.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/test_repo_readme.py b/rhodecode/tests/models/test_repo_readme.py --- a/rhodecode/tests/models/test_repo_readme.py +++ b/rhodecode/tests/models/test_repo_readme.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -40,40 +39,39 @@ class CommitUtility: def __init__(self, vcsbackend): self.vcsbackend = vcsbackend - def commit_with_files(self, filenames): + def commit_with_files(self, filenames: list[bytes]): commits = [ {'message': 'Adding all requested files', 'added': [ - nodes.FileNode(filename, content='') - for filename in filenames + nodes.FileNode(filename, content=b'') for filename in filenames ]}] repo = self.vcsbackend.create_repo(commits=commits) return repo.get_commit() def test_no_matching_file_returns_none(commit_util): - commit = commit_util.commit_with_files(['LIESMICH']) + commit = commit_util.commit_with_files([b'LIESMICH']) finder = ReadmeFinder(default_renderer='rst') filenode = finder.search(commit) assert filenode is None def test_matching_file_returns_the_file_name(commit_util): - commit = commit_util.commit_with_files(['README']) + commit = commit_util.commit_with_files([b'README']) finder = ReadmeFinder(default_renderer='rst') filenode = finder.search(commit) assert filenode.path == 'README' def test_matching_file_with_extension(commit_util): - commit = commit_util.commit_with_files(['README.rst']) + commit = commit_util.commit_with_files([b'README.rst']) finder = ReadmeFinder(default_renderer='rst') filenode = finder.search(commit) assert filenode.path == 'README.rst' def test_prefers_readme_without_extension(commit_util): - commit = commit_util.commit_with_files(['README.rst', 'Readme']) + commit = commit_util.commit_with_files([b'README.rst', b'Readme']) finder = ReadmeFinder() filenode = finder.search(commit) assert filenode.path == 'Readme' @@ -84,23 +82,21 @@ def test_prefers_readme_without_extensio ('markdown', 'readme.md'), ]) def test_prefers_renderer_extensions(commit_util, renderer, expected): - commit = commit_util.commit_with_files( - ['readme.rst', 'readme.md', 'readme.txt']) + commit = commit_util.commit_with_files([b'readme.rst', b'readme.md', b'readme.txt']) finder = ReadmeFinder(default_renderer=renderer) filenode = finder.search(commit) assert filenode.path == expected def test_finds_readme_in_subdirectory(commit_util): - commit = commit_util.commit_with_files(['doc/README.rst', 'LIESMICH']) + commit = commit_util.commit_with_files([b'doc/README.rst', b'LIESMICH']) finder = ReadmeFinder() filenode = finder.search(commit) assert filenode.path == 'doc/README.rst' def test_prefers_subdirectory_with_priority(commit_util): - commit = commit_util.commit_with_files( - ['Doc/Readme.rst', 'Docs/Readme.rst']) + commit = commit_util.commit_with_files([b'Doc/Readme.rst', b'Docs/Readme.rst']) finder = ReadmeFinder() filenode = finder.search(commit) assert filenode.path == 'Doc/Readme.rst' diff --git a/rhodecode/tests/models/test_repos.py b/rhodecode/tests/models/test_repos.py --- a/rhodecode/tests/models/test_repos.py +++ b/rhodecode/tests/models/test_repos.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/test_scm.py b/rhodecode/tests/models/test_scm.py --- a/rhodecode/tests/models/test_scm.py +++ b/rhodecode/tests/models/test_scm.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -168,7 +167,7 @@ def assert_contains_only_unicode(structu @pytest.mark.backends("hg", "git") def test_get_non_unicode_reference(backend): model = scm.ScmModel() - non_unicode_list = ["Adını".decode("cp1254")] + non_unicode_list = ["Adını".encode("cp1254")] def scm_instance(): return Mock( @@ -179,11 +178,11 @@ def test_get_non_unicode_reference(backe choices, __ = model.get_repo_landing_revs(translator=lambda s: s, repo=repo) if backend.alias == 'hg': valid_choices = [ - 'rev:tip', u'branch:Ad\xc4\xb1n\xc4\xb1', - u'book:Ad\xc4\xb1n\xc4\xb1', u'tag:Ad\xc4\xb1n\xc4\xb1'] + 'rev:tip', 'branch:Ad\xc4\xb1n\xc4\xb1', + 'book:Ad\xc4\xb1n\xc4\xb1', 'tag:Ad\xc4\xb1n\xc4\xb1'] else: valid_choices = [ - 'rev:tip', u'branch:Ad\xc4\xb1n\xc4\xb1', - u'tag:Ad\xc4\xb1n\xc4\xb1'] + 'rev:tip', 'branch:Ad\xc4\xb1n\xc4\xb1', + 'tag:Ad\xc4\xb1n\xc4\xb1'] assert choices == valid_choices diff --git a/rhodecode/tests/models/test_user_group_permissions_on_repo_groups.py b/rhodecode/tests/models/test_user_group_permissions_on_repo_groups.py --- a/rhodecode/tests/models/test_user_group_permissions_on_repo_groups.py +++ b/rhodecode/tests/models/test_user_group_permissions_on_repo_groups.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/test_user_groups.py b/rhodecode/tests/models/test_user_groups.py --- a/rhodecode/tests/models/test_user_groups.py +++ b/rhodecode/tests/models/test_user_groups.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/test_user_permissions_on_repo_groups.py b/rhodecode/tests/models/test_user_permissions_on_repo_groups.py --- a/rhodecode/tests/models/test_user_permissions_on_repo_groups.py +++ b/rhodecode/tests/models/test_user_permissions_on_repo_groups.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/models/test_users.py b/rhodecode/tests/models/test_users.py --- a/rhodecode/tests/models/test_users.py +++ b/rhodecode/tests/models/test_users.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -21,7 +20,6 @@ import pytest import mock -from rhodecode.lib.utils2 import safe_unicode from rhodecode.model.db import ( true, User, UserGroup, UserGroupMember, UserEmailMap, Permission, UserIpMap) from rhodecode.model.meta import Session @@ -30,6 +28,8 @@ from rhodecode.model.user_group import U from rhodecode.model.repo import RepoModel from rhodecode.model.repo_group import RepoGroupModel from rhodecode.tests.fixture import Fixture +from rhodecode.lib.str_utils import safe_str + fixture = Fixture() @@ -69,7 +69,7 @@ class TestGetUsers(object): fake_users = [u for u in users if u['last_name'].startswith('Fake')] assert len(fake_users) == 2 for user in fake_users: - assert user['last_name'] == safe_unicode('Fake ünicode user') + assert user['last_name'] == safe_str('Fake ünicode user') def test_returns_user_filtered_by_first_name(self, backend, user_util): created_users = [] @@ -88,7 +88,7 @@ class TestGetUsers(object): fake_users = [u for u in users if u['last_name'].startswith('Fake')] assert len(fake_users) == 2 for user in fake_users: - assert user['first_name'] == safe_unicode('Fake ünicode user') + assert user['first_name'] == safe_str('Fake ünicode user') def test_returns_user_filtered_by_username(self, backend, user_util): created_users = [] @@ -122,10 +122,10 @@ class TestGetUsers(object): @pytest.fixture() def test_user(request, baseapp): usr = UserModel().create_or_update( - username=u'test_user', - password=u'qweqwe', - email=u'main_email@rhodecode.org', - firstname=u'u1', lastname=u'u1') + username='test_user', + password='qweqwe', + email='main_email@rhodecode.org', + firstname='u1', lastname=u'u1') Session().commit() assert User.get_by_username(u'test_user') == usr diff --git a/rhodecode/tests/other/__init__.py b/rhodecode/tests/other/__init__.py --- a/rhodecode/tests/other/__init__.py +++ b/rhodecode/tests/other/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/other/test_base_controller.py b/rhodecode/tests/other/test_base_controller.py --- a/rhodecode/tests/other/test_base_controller.py +++ b/rhodecode/tests/other/test_base_controller.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/other/test_validators.py b/rhodecode/tests/other/test_validators.py --- a/rhodecode/tests/other/test_validators.py +++ b/rhodecode/tests/other/test_validators.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/other/test_views_utils.py b/rhodecode/tests/other/test_views_utils.py --- a/rhodecode/tests/other/test_views_utils.py +++ b/rhodecode/tests/other/test_views_utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini --- a/rhodecode/tests/rhodecode.ini +++ b/rhodecode/tests/rhodecode.ini @@ -62,6 +62,9 @@ use = egg:gunicorn#main ; The maximum number of simultaneous clients per worker. Valid only for gevent #worker_connections = 10 +; The maximum number of pending connections worker will queue to handle +#backlog = 64 + ; Max number of requests that worker will handle before being gracefully restarted. ; Prevents memory leaks, jitter adds variability so not all workers are restarted at once. #max_requests = 1000 @@ -76,6 +79,40 @@ use = egg:gunicorn#main ; 0 for unlimited #limit_request_line = 0 +; Limit the number of HTTP headers fields in a request. +; By default this value is 100 and can't be larger than 32768. +#limit_request_fields = 32768 + +; Limit the allowed size of an HTTP request header field. +; Value is a positive number or 0. +; Setting it to 0 will allow unlimited header field sizes. +#limit_request_field_size = 0 + +; Timeout for graceful workers restart. +; After receiving a restart signal, workers have this much time to finish +; serving requests. Workers still alive after the timeout (starting from the +; receipt of the restart signal) are force killed. +; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h) +#graceful_timeout = 3600 + +# The number of seconds to wait for requests on a Keep-Alive connection. +# Generally set in the 1-5 seconds range. +#keepalive = 2 + +; Maximum memory usage that each worker can use before it will receive a +; graceful restart signal 0 = memory monitoring is disabled +; Examples: 268435456 (256MB), 536870912 (512MB) +; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB) +#memory_max_usage = 0 + +; How often in seconds to check for memory usage for each gunicorn worker +#memory_usage_check_interval = 60 + +; Threshold value for which we don't recycle worker if GarbageCollection +; frees up enough resources. Before each restart we try to run GC on worker +; in case we get enough free memory after that, restart will not happen. +#memory_usage_recovery_threshold = 0.8 + ; Prefix middleware for RhodeCode. ; recommended when using proxy setup. @@ -108,19 +145,9 @@ use = egg:rhodecode-enterprise-ce ; enable proxy prefix middleware, defined above #filter-with = proxy-prefix - -## RHODECODE PLUGINS ## -rhodecode.includes = rhodecode.api - -# api prefix url -rhodecode.api.url = /_admin/api - - -## END RHODECODE PLUGINS ## - -## encryption key used to encrypt social plugin tokens, -## remote_urls with credentials etc, if not set it defaults to -## `beaker.session.secret` +; encryption key used to encrypt social plugin tokens, +; remote_urls with credentials etc, if not set it defaults to +; `beaker.session.secret` #rhodecode.encrypted_values.secret = ; decryption strict mode (enabled by default). It controls if decryption raises @@ -146,12 +173,6 @@ lang = en ; Settings this to true could lead to very long startup time. startup.import_repos = true -; Uncomment and set this path to use archive download cache. -; Once enabled, generated archives will be cached at this location -; and served from the cache during subsequent requests for the same archive of -; the repository. -#archive_cache_dir = /tmp/tarballcache - ; URL at which the application is running. This is used for Bootstrapping ; requests in context when no web request is available. Used in ishell, or ; SSH calls. Set this for events to receive proper url for SSH calls. @@ -160,8 +181,14 @@ app.base_url = http://rhodecode.local ; Unique application ID. Should be a random unique string for security. app_instance_uuid = rc-production -## cut off limit for large diffs (size in bytes) +; Cut off limit for large diffs (size in bytes). If overall diff size on +; commit, or pull request exceeds this limit this diff will be displayed +; partially. E.g 512000 == 512Kb cut_off_limit_diff = 1024000 + +; Cut off limit for large files inside diffs (size in bytes). Each individual +; file inside diff which exceeds this limit will be displayed partially. +; E.g 128000 == 128Kb cut_off_limit_file = 256000 ; Use cached version of vcs repositories everywhere. Recommended to be `true` @@ -259,16 +286,20 @@ allow_repo_location_change = true ; allows to setup custom hooks in settings page allow_custom_hooks_settings = true -## generated license token, goto license page in RhodeCode settings to obtain -## new token +; Generated license token required for EE edition license. +; New generated token value can be found in Admin > settings > license page. license_token = abra-cada-bra1-rce3 -## supervisor connection uri, for managing supervisor and logs. +; This flag hides sensitive information on the license page such as token, and license data +license.hide_license_info = false + +; supervisor connection uri, for managing supervisor and logs. supervisor.uri = -## supervisord group name/id we only want this RC instance to handle + +; supervisord group name/id we only want this RC instance to handle supervisor.group_id = dev -## Display extended labs settings +; Display extended labs settings labs_settings_active = true ; Custom exception store path, defaults to TMPDIR @@ -295,6 +326,20 @@ file_store.backend = local ; path to store the uploaded binaries file_store.storage_path = %(here)s/data/file_store +; Uncomment and set this path to control settings for archive download cache. +; Generated repo archives will be cached at this location +; and served from the cache during subsequent requests for the same archive of +; the repository. This path is important to be shared across filesystems and with +; RhodeCode and vcsserver + +; Default is $cache_dir/archive_cache if not set +archive_cache.store_dir = /tmp/rc-test-data/archive_cache + +; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb +archive_cache.cache_size_gb = 10 + +; By default cache uses sharding technique, this specifies how many shards are there +archive_cache.cache_shards = 10 ; ############# ; CELERY CONFIG @@ -325,35 +370,108 @@ celery.task_always_eager = false ; Default cache dir for caches. Putting this into a ramdisk can boost performance. ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space -cache_dir = %(here)s/data - -## locking and default file storage for Beaker. Putting this into a ramdisk -## can boost performance, eg. %(here)s/data_ramdisk/cache/beaker_data -beaker.cache.data_dir = %(here)s/rc/data/cache/beaker_data -beaker.cache.lock_dir = %(here)s/rc/data/cache/beaker_lock - -beaker.cache.regions = long_term - -beaker.cache.long_term.type = memory -beaker.cache.long_term.expire = 36000 -beaker.cache.long_term.key_length = 256 +cache_dir = %(here)s/rc-test-data - -##################################### -### DOGPILE CACHE #### -##################################### - -## permission tree cache settings -rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace -rc_cache.cache_perms.expiration_time = 0 -rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1 - - -## cache settings for SQL queries +; ********************************************* +; `sql_cache_short` cache for heavy SQL queries +; Only supported backend is `memory_lru` +; ********************************************* rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru rc_cache.sql_cache_short.expiration_time = 0 +; ***************************************************** +; `cache_repo_longterm` cache for repo object instances +; Only supported backend is `memory_lru` +; ***************************************************** +rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru +; by default we use 30 Days, cache is still invalidated on push +rc_cache.cache_repo_longterm.expiration_time = 2592000 +; max items in LRU cache, set to smaller number to save memory, and expire last used caches +rc_cache.cache_repo_longterm.max_size = 10000 + + +; ********************************************* +; `cache_general` cache for general purpose use +; for simplicity use rc.file_namespace backend, +; for performance and scale use rc.redis +; ********************************************* +rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace +rc_cache.cache_general.expiration_time = 43200 +; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set +rc_cache.cache_general.arguments.filename = %(here)s/cache-backend/cache_general_db + +; alternative `cache_general` redis backend with distributed lock +#rc_cache.cache_general.backend = dogpile.cache.rc.redis +#rc_cache.cache_general.expiration_time = 300 + +; redis_expiration_time needs to be greater then expiration_time +#rc_cache.cache_general.arguments.redis_expiration_time = 7200 + +#rc_cache.cache_general.arguments.host = localhost +#rc_cache.cache_general.arguments.port = 6379 +#rc_cache.cache_general.arguments.db = 0 +#rc_cache.cache_general.arguments.socket_timeout = 30 +; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends +#rc_cache.cache_general.arguments.distributed_lock = true + +; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen +#rc_cache.cache_general.arguments.lock_auto_renewal = true + +; ************************************************* +; `cache_perms` cache for permission tree, auth TTL +; for simplicity use rc.file_namespace backend, +; for performance and scale use rc.redis +; ************************************************* +rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace +rc_cache.cache_perms.expiration_time = 0 +; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set +rc_cache.cache_perms.arguments.filename = %(here)s/cache-backend/cache_perms_db + +; alternative `cache_perms` redis backend with distributed lock +#rc_cache.cache_perms.backend = dogpile.cache.rc.redis +#rc_cache.cache_perms.expiration_time = 300 + +; redis_expiration_time needs to be greater then expiration_time +#rc_cache.cache_perms.arguments.redis_expiration_time = 7200 + +#rc_cache.cache_perms.arguments.host = localhost +#rc_cache.cache_perms.arguments.port = 6379 +#rc_cache.cache_perms.arguments.db = 0 +#rc_cache.cache_perms.arguments.socket_timeout = 30 +; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends +#rc_cache.cache_perms.arguments.distributed_lock = true + +; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen +#rc_cache.cache_perms.arguments.lock_auto_renewal = true + +; *************************************************** +; `cache_repo` cache for file tree, Readme, RSS FEEDS +; for simplicity use rc.file_namespace backend, +; for performance and scale use rc.redis +; *************************************************** +rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace +rc_cache.cache_repo.expiration_time = 2592000 +; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set +rc_cache.cache_repo.arguments.filename = %(here)s/cache-backend/cache_repo_db + +; alternative `cache_repo` redis backend with distributed lock +#rc_cache.cache_repo.backend = dogpile.cache.rc.redis +#rc_cache.cache_repo.expiration_time = 2592000 + +; redis_expiration_time needs to be greater then expiration_time +#rc_cache.cache_repo.arguments.redis_expiration_time = 2678400 + +#rc_cache.cache_repo.arguments.host = localhost +#rc_cache.cache_repo.arguments.port = 6379 +#rc_cache.cache_repo.arguments.db = 1 +#rc_cache.cache_repo.arguments.socket_timeout = 30 +; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends +#rc_cache.cache_repo.arguments.distributed_lock = true + +; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen +#rc_cache.cache_repo.arguments.lock_auto_renewal = true + ; ############## ; BEAKER SESSION ; ############## @@ -362,7 +480,7 @@ rc_cache.sql_cache_short.expiration_time ; types are file, ext:redis, ext:database, ext:memcached, and memory (default if not specified). ; Fastest ones are Redis and ext:database beaker.session.type = file -beaker.session.data_dir = %(here)s/rc/data/sessions/data +beaker.session.data_dir = %(here)s/rc-tests/data/sessions ; Redis based sessions #beaker.session.type = ext:redis @@ -378,7 +496,7 @@ beaker.session.data_dir = %(here)s/rc/da beaker.session.key = rhodecode beaker.session.secret = test-rc-uytcxaz -beaker.session.lock_dir = %(here)s/rc/data/sessions/lock +beaker.session.lock_dir = %(here)s/data/sessions/lock ; Secure encrypted cookie. Requires AES and AES python libraries ; you must disable beaker.session.secret to use this @@ -476,7 +594,8 @@ sqlalchemy.db1.pool_recycle = 3600 ; VCS CONFIG ; ########## vcs.server.enable = true -vcs.server = localhost:9901 +#vcs.server = localhost:9901 +vcs.server = vcsserver:10010 ; Web server connectivity protocol, responsible for web based VCS operations ; Available protocols are: @@ -491,8 +610,9 @@ vcs.scm_app_implementation = http ; `http` - use http-rpc backend (default) vcs.hooks.protocol = http -; Host on which this instance is listening for hooks. If vcsserver is in other location -; this should be adjusted. +; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be +; accessible via network. +; Use vcs.hooks.host = "*" to bind to current hostname (for Docker) vcs.hooks.host = * ; Start VCSServer with this instance as a subprocess, useful for development @@ -617,7 +737,7 @@ custom.conf = 1 ; ##################### [loggers] -keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper +keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper, dogpile [handlers] keys = console, console_sql @@ -651,6 +771,12 @@ handlers = qualname = beaker.container propagate = 1 +[logger_dogpile] +level = INFO +handlers = console +qualname = dogpile +propagate = 1 + [logger_rhodecode] level = DEBUG handlers = diff --git a/rhodecode/tests/scripts/test_concurency.py b/rhodecode/tests/scripts/test_concurency.py --- a/rhodecode/tests/scripts/test_concurency.py +++ b/rhodecode/tests/scripts/test_concurency.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/scripts/test_crawler.py b/rhodecode/tests/scripts/test_crawler.py --- a/rhodecode/tests/scripts/test_crawler.py +++ b/rhodecode/tests/scripts/test_crawler.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -28,8 +27,12 @@ watch -n1 ./rhodecode/tests/mem_watch import cookielib -import urllib.request, urllib.parse, urllib.error -import urllib.request, urllib.error, urllib.parse +import urllib.request +import urllib.parse +import urllib.error +import urllib.request +import urllib.error +import urllib.parse import time import os import sys diff --git a/rhodecode/tests/server_utils.py b/rhodecode/tests/server_utils.py --- a/rhodecode/tests/server_utils.py +++ b/rhodecode/tests/server_utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -58,7 +57,7 @@ def assert_no_running_instance(url): class ServerBase(object): _args = [] log_file_name = 'NOT_DEFINED.log' - status_url_tmpl = 'http://{host}:{port}' + status_url_tmpl = 'http://{host}:{port}/_admin/ops/ping' def __init__(self, config_file, log_file): self.config_file = config_file @@ -99,6 +98,10 @@ class ServerBase(object): with open(self.log_file) as f: return f.read() + def assert_message_in_server_logs(self, message): + server_logs = self.get_rc_log() + assert message in server_logs + def wait_until_ready(self, timeout=30): host = self._config['host'] port = self._config['port'] @@ -140,10 +143,15 @@ class RcVCSServer(ServerBase): log_file_name = 'rc-vcsserver.log' status_url_tmpl = 'http://{host}:{port}/status' - def __init__(self, config_file, log_file=None): + def __init__(self, config_file, log_file=None, workers='2'): super(RcVCSServer, self).__init__(config_file, log_file) self._args = [ - 'gunicorn', '--bind', self.bind_addr, + 'gunicorn', + '--bind', self.bind_addr, + '--worker-class', 'gevent', + '--backlog', '16', + '--timeout', '300', + '--workers', workers, '--paste', self.config_file] def start(self): @@ -172,10 +180,15 @@ class RcWebServer(ServerBase): log_file_name = 'rc-web.log' status_url_tmpl = 'http://{host}:{port}/_admin/ops/ping' - def __init__(self, config_file, log_file=None): + def __init__(self, config_file, log_file=None, workers='1'): super(RcWebServer, self).__init__(config_file, log_file) self._args = [ - 'gunicorn', '--bind', self.bind_addr, '--worker-class', 'gevent', + 'gunicorn', + '--bind', self.bind_addr, + '--worker-class', 'gevent', + '--backlog', '16', + '--timeout', '300', + '--workers', workers, '--paste', self.config_file] def start(self): @@ -204,5 +217,5 @@ class RcWebServer(ServerBase): 'cloned_repo': repo_name, } params.update(**kwargs) - _url = 'http://%(user)s:%(passwd)s@%(host)s/%(cloned_repo)s' % params + _url = f"http://{params['user']}:{params['passwd']}@{params['host']}/{params['cloned_repo']}" return _url diff --git a/rhodecode/tests/utils.py b/rhodecode/tests/utils.py --- a/rhodecode/tests/utils.py +++ b/rhodecode/tests/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -78,28 +77,25 @@ class CustomTestResponse(TestResponse): else: no = [] if kw: - raise TypeError( - "The only keyword argument allowed is 'no' got %s" % kw) + raise TypeError(f"The only keyword argument allowed is 'no' got {kw}") f = self._save_output(str(self)) for s in strings: if s not in self: - print_stderr("Actual response (no %r):" % s) - print_stderr("body output saved as `%s`" % f) + print_stderr(f"Actual response (no {s!r}):") + print_stderr(f"body output saved as `{f}`") if print_body: print_stderr(str(self)) - raise IndexError( - "Body does not contain string %r, body output saved as %s" % (s, f)) + raise IndexError(f"Body does not contain string {s!r}, body output saved as {f}") for no_s in no: if no_s in self: - print_stderr("Actual response (has %r)" % no_s) - print_stderr("body output saved as `%s`" % f) + print_stderr(f"Actual response (has {no_s!r})") + print_stderr(f"body output saved as `{f}`") if print_body: print_stderr(str(self)) - raise IndexError( - "Body contains bad string %r, body output saved as %s" % (no_s, f)) + raise IndexError(f"Body contains bad string {no_s!r}, body output saved as {f}") def assert_response(self): return AssertResponse(self) @@ -166,6 +162,10 @@ class CustomTestApp(TestApp): def _pyramid_settings(self): return self._pyramid_registry.settings + def do_request(self, req, status=None, expect_errors=None): + # you can put custom code here + return super().do_request(req, status, expect_errors) + def set_anonymous_access(enabled): """(Dis)allows anonymous access depending on parameter `enabled`""" @@ -406,12 +406,12 @@ def wait_for_url(url, timeout=10): pytest.fail(f"Timeout while waiting for URL {url}") -def is_url_reachable(url: str, log_exc: bool = True) -> bool: +def is_url_reachable(url: str, log_exc: bool = False) -> bool: try: urllib.request.urlopen(url) except urllib.error.URLError: if log_exc: - log.exception('URL `{}` reach error'.format(url)) + log.exception(f'URL `{url}` reach error') return False return True @@ -425,7 +425,7 @@ def repo_on_filesystem(repo_name): def commit_change( - repo, filename, content, message, vcs_type, parent=None, newfile=False): + repo, filename: bytes, content: bytes, message, vcs_type, parent=None, newfile=False): from rhodecode.tests import TEST_USER_ADMIN_LOGIN repo = Repository.get_by_repo_name(repo) diff --git a/rhodecode/tests/vcs/__init__.py b/rhodecode/tests/vcs/__init__.py --- a/rhodecode/tests/vcs/__init__.py +++ b/rhodecode/tests/vcs/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/vcs/conftest.py b/rhodecode/tests/vcs/conftest.py --- a/rhodecode/tests/vcs/conftest.py +++ b/rhodecode/tests/vcs/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -24,6 +23,7 @@ import datetime import pytest +from rhodecode.lib.str_utils import safe_bytes from rhodecode.lib.vcs.backends import get_backend from rhodecode.lib.vcs.backends.base import Config from rhodecode.lib.vcs.nodes import FileNode @@ -137,17 +137,25 @@ def _add_commits_to_repo(repo, commits): for commit in commits: for node in commit.get('added', []): - imc.add(FileNode(node.path, content=node.content)) + if not isinstance(node, FileNode): + node = FileNode(safe_bytes(node.path), content=node.content) + imc.add(node) + for node in commit.get('changed', []): - imc.change(FileNode(node.path, content=node.content)) + if not isinstance(node, FileNode): + node = FileNode(safe_bytes(node.path), content=node.content) + imc.change(node) + for node in commit.get('removed', []): - imc.remove(FileNode(node.path)) + imc.remove(FileNode(safe_bytes(node.path))) tip = imc.commit( message=str(commit['message']), author=str(commit['author']), date=commit['date'], - branch=commit.get('branch')) + branch=commit.get('branch') + ) + return tip @@ -167,7 +175,7 @@ def vcs_repo(request, backend_alias): @pytest.fixture() def generate_repo_with_commits(vcs_repo): """ - Creates a fabric to generate N comits with some file nodes on a randomly + Creates a fabric to generate N commits with some file nodes on a randomly generated repository """ @@ -179,11 +187,11 @@ def generate_repo_with_commits(vcs_repo) 'author': 'Joe Doe ', 'date': start_date + datetime.timedelta(hours=12 * x), 'added': [ - FileNode('file_%d.txt' % x, content='Foobar %d' % x), + FileNode(b'file_%d.txt' % x, content=b'Foobar %d' % x), ], 'modified': [ - FileNode('file_%d.txt' % x, - content='Foobar %d modified' % (x-1)), + FileNode(b'file_%d.txt' % x, + content=b'Foobar %d modified' % (x-1)), ] } @@ -229,24 +237,24 @@ class BackendTestMixin(object): def _get_commits(cls): commits = [ { - 'message': u'Initial commit', - 'author': u'Joe Doe ', + 'message': 'Initial commit', + 'author': 'Joe Doe ', 'date': datetime.datetime(2010, 1, 1, 20), 'added': [ - FileNode('foobar', content='Foobar'), - FileNode('foobar2', content='Foobar II'), - FileNode('foo/bar/baz', content='baz here!'), + FileNode(b'foobar', content=b'Foobar'), + FileNode(b'foobar2', content=b'Foobar II'), + FileNode(b'foo/bar/baz', content=b'baz here!'), ], }, { - 'message': u'Changes...', - 'author': u'Jane Doe ', + 'message': 'Changes...', + 'author': 'Jane Doe ', 'date': datetime.datetime(2010, 1, 1, 21), 'added': [ - FileNode('some/new.txt', content='news...'), + FileNode(b'some/new.txt', content=b'news...'), ], 'changed': [ - FileNode('foobar', 'Foobar I'), + FileNode(b'foobar', b'Foobar I'), ], 'removed': [], }, diff --git a/rhodecode/tests/vcs/test_archives.py b/rhodecode/tests/vcs/test_archives.py --- a/rhodecode/tests/vcs/test_archives.py +++ b/rhodecode/tests/vcs/test_archives.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -22,30 +21,29 @@ import datetime import os import shutil import tarfile -import tempfile import zipfile import io import mock import pytest +import rhodecode +from rhodecode.lib.rc_cache.archive_cache import get_archival_config +from rhodecode.lib.str_utils import ascii_bytes from rhodecode.lib.vcs.backends import base from rhodecode.lib.vcs.exceptions import ImproperArchiveTypeError, VCSError from rhodecode.lib.vcs.nodes import FileNode from rhodecode.tests.vcs.conftest import BackendTestMixin +@pytest.fixture() +def d_cache_config(): + return get_archival_config(config=rhodecode.CONFIG) + + @pytest.mark.usefixtures("vcs_repository_support") class TestArchives(BackendTestMixin): - @pytest.fixture(autouse=True) - def tempfile(self, request): - self.temp_file = tempfile.mkstemp()[1] - - @request.addfinalizer - def cleanup(): - os.remove(self.temp_file) - @classmethod def _get_commits(cls): start_date = datetime.datetime(2010, 1, 1, 20) @@ -54,9 +52,9 @@ class TestArchives(BackendTestMixin): 'author': 'Joe Doe ', 'date': start_date + datetime.timedelta(hours=12), 'added': [ - FileNode('executable_0o100755', '...', mode=0o100755), - FileNode('executable_0o100500', '...', mode=0o100500), - FileNode('not_executable', '...', mode=0o100644), + FileNode(b'executable_0o100755', b'mode_755', mode=0o100755), + FileNode(b'executable_0o100500', b'mode_500', mode=0o100500), + FileNode(b'not_executable', b'mode_644', mode=0o100644), ], } for x in range(5): @@ -65,22 +63,27 @@ class TestArchives(BackendTestMixin): 'author': 'Joe Doe ', 'date': start_date + datetime.timedelta(hours=12 * x), 'added': [ - FileNode('%d/file_%d.txt' % (x, x), content='Foobar %d' % x), + FileNode(b'%d/file_%d.txt' % (x, x), content=b'Foobar %d' % x), ], } @pytest.mark.parametrize('compressor', ['gz', 'bz2']) - def test_archive_tar(self, compressor): - self.tip.archive_repo( - self.temp_file, kind='t{}'.format(compressor), archive_dir_name='repo') - out_dir = tempfile.mkdtemp() - out_file = tarfile.open(self.temp_file, 'r|{}'.format(compressor)) + def test_archive_tar(self, compressor, tmpdir, tmp_path, d_cache_config): + + archive_node = tmp_path / 'archive-node' + archive_node.touch() + + archive_lnk = self.tip.archive_repo( + str(archive_node), kind=f't{compressor}', archive_dir_name='repo', cache_config=d_cache_config) + + out_dir = tmpdir + out_file = tarfile.open(str(archive_lnk), f'r|{compressor}') out_file.extractall(out_dir) out_file.close() for x in range(5): node_path = '%d/file_%d.txt' % (x, x) - with open(os.path.join(out_dir, 'repo/' + node_path)) as f: + with open(os.path.join(out_dir, 'repo/' + node_path), 'rb') as f: file_content = f.read() assert file_content == self.tip.get_node(node_path).content @@ -88,53 +91,72 @@ class TestArchives(BackendTestMixin): @pytest.mark.parametrize('compressor', ['gz', 'bz2']) def test_archive_tar_symlink(self, compressor): - return False + pytest.skip('Not supported') @pytest.mark.parametrize('compressor', ['gz', 'bz2']) - def test_archive_tar_file_modes(self, compressor): - self.tip.archive_repo( - self.temp_file, kind='t{}'.format(compressor), archive_dir_name='repo') - out_dir = tempfile.mkdtemp() - out_file = tarfile.open(self.temp_file, 'r|{}'.format(compressor)) + def test_archive_tar_file_modes(self, compressor, tmpdir, tmp_path, d_cache_config): + archive_node = tmp_path / 'archive-node' + archive_node.touch() + + archive_lnk = self.tip.archive_repo( + str(archive_node), kind='t{}'.format(compressor), archive_dir_name='repo', cache_config=d_cache_config) + + out_dir = tmpdir + out_file = tarfile.open(str(archive_lnk), 'r|{}'.format(compressor)) out_file.extractall(out_dir) out_file.close() - dest = lambda inp: os.path.join(out_dir, 'repo/' + inp) - assert oct(os.stat(dest('not_executable')).st_mode) == '0100644' + def dest(inp): + return os.path.join(out_dir, "repo/" + inp) + + assert oct(os.stat(dest('not_executable')).st_mode) == '0o100644' - def test_archive_zip(self): - self.tip.archive_repo(self.temp_file, kind='zip', archive_dir_name='repo') - out = zipfile.ZipFile(self.temp_file) + def test_archive_zip(self, tmp_path, d_cache_config): + archive_node = tmp_path / 'archive-node' + archive_node.touch() + + archive_lnk = self.tip.archive_repo(str(archive_node), kind='zip', + archive_dir_name='repo', cache_config=d_cache_config) + zip_file = zipfile.ZipFile(str(archive_lnk)) for x in range(5): node_path = '%d/file_%d.txt' % (x, x) - decompressed = io.StringIO() - decompressed.write(out.read('repo/' + node_path)) + data = zip_file.read(f'repo/{node_path}') + + decompressed = io.BytesIO() + decompressed.write(data) assert decompressed.getvalue() == \ self.tip.get_node(node_path).content decompressed.close() - def test_archive_zip_with_metadata(self): - self.tip.archive_repo(self.temp_file, kind='zip', - archive_dir_name='repo', write_metadata=True) + def test_archive_zip_with_metadata(self, tmp_path, d_cache_config): + archive_node = tmp_path / 'archive-node' + archive_node.touch() - out = zipfile.ZipFile(self.temp_file) - metafile = out.read('repo/.archival.txt') + archive_lnk = self.tip.archive_repo(str(archive_node), kind='zip', + archive_dir_name='repo', write_metadata=True, cache_config=d_cache_config) - raw_id = self.tip.raw_id - assert 'commit_id:%s' % raw_id in metafile + zip_file = zipfile.ZipFile(str(archive_lnk)) + metafile = zip_file.read('repo/.archival.txt') + + raw_id = ascii_bytes(self.tip.raw_id) + assert b'commit_id:%b' % raw_id in metafile for x in range(5): node_path = '%d/file_%d.txt' % (x, x) - decompressed = io.StringIO() - decompressed.write(out.read('repo/' + node_path)) + data = zip_file.read(f'repo/{node_path}') + decompressed = io.BytesIO() + decompressed.write(data) assert decompressed.getvalue() == \ self.tip.get_node(node_path).content decompressed.close() - def test_archive_wrong_kind(self): + def test_archive_wrong_kind(self, tmp_path, d_cache_config): + archive_node = tmp_path / 'archive-node' + archive_node.touch() + with pytest.raises(ImproperArchiveTypeError): - self.tip.archive_repo(self.temp_file, kind='wrong kind') + self.tip.archive_repo(str(archive_node), kind='wrong kind', cache_config=d_cache_config) @pytest.fixture() @@ -144,15 +166,14 @@ def base_commit(): """ commit = base.BaseCommit() commit.repository = mock.Mock() - commit.repository.name = u'fake_repo' + commit.repository.name = 'fake_repo' commit.short_id = 'fake_id' return commit -@pytest.mark.parametrize("prefix", [u"unicode-prefix", u"Ünïcödë"]) -def test_validate_archive_prefix_enforces_bytes_as_prefix(prefix, base_commit): - with pytest.raises(ValueError): - base_commit._validate_archive_prefix(prefix) +def test_validate_archive_prefix_enforces_non_ascii_as_prefix(base_commit): + with pytest.raises(VCSError): + base_commit._validate_archive_prefix("Ünïcödë") def test_validate_archive_prefix_empty_prefix(base_commit): diff --git a/rhodecode/tests/vcs/test_branches.py b/rhodecode/tests/vcs/test_branches.py --- a/rhodecode/tests/vcs/test_branches.py +++ b/rhodecode/tests/vcs/test_branches.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -54,9 +53,9 @@ class TestBranches(BackendTestMixin): # This check must not be removed to ensure the 'branches' LazyProperty # gets hit *before* the new 'foobar' branch got created: assert 'foobar' not in self.repo.branches - self.imc.add(FileNode( - 'docs/index.txt', - content='Documentation\n')) + self.imc.add( + FileNode(b'docs/index.txt', content=b'Documentation\n') + ) foobar_tip = self.imc.commit( message=u'New branch: foobar', author=u'joe ', @@ -68,9 +67,10 @@ class TestBranches(BackendTestMixin): @pytest.mark.backends("git", "hg") def test_new_head(self): tip = self.repo.get_commit() - self.imc.add(FileNode( - 'docs/index.txt', - content='Documentation\n')) + self.imc.add( + FileNode(b'docs/index.txt', + content=b'Documentation\n') + ) foobar_tip = self.imc.commit( message=u'New branch: foobar', author=u'joe ', @@ -78,8 +78,8 @@ class TestBranches(BackendTestMixin): parents=[tip], ) self.imc.change(FileNode( - 'docs/index.txt', - content='Documentation\nand more...\n')) + b'docs/index.txt', + content=b'Documentation\nand more...\n')) newtip = self.imc.commit( message=u'At default branch', author=u'joe ', @@ -99,7 +99,7 @@ class TestBranches(BackendTestMixin): @pytest.mark.backends("git", "hg") def test_branch_with_slash_in_name(self): - self.imc.add(FileNode('extrafile', content='Some data\n')) + self.imc.add(FileNode(b'extrafile', content=b'Some data\n')) self.imc.commit( u'Branch with a slash!', author=u'joe ', branch='issue/123') @@ -107,11 +107,11 @@ class TestBranches(BackendTestMixin): @pytest.mark.backends("git", "hg") def test_branch_with_slash_in_name_and_similar_without(self): - self.imc.add(FileNode('extrafile', content='Some data\n')) + self.imc.add(FileNode(b'extrafile', content=b'Some data\n')) self.imc.commit( u'Branch with a slash!', author=u'joe ', branch='issue/123') - self.imc.add(FileNode('extrafile II', content='Some data\n')) + self.imc.add(FileNode(b'extrafile II', content=b'Some data\n')) self.imc.commit( u'Branch without a slash...', author=u'joe ', branch='123') @@ -138,10 +138,10 @@ class TestSvnBranches(object): 'branches/argparse', 'trunk', ] - assert repo.branches.keys() == expected_branches + assert list(repo.branches.keys()) == expected_branches def test_discovers_ordered_tags(self, vcsbackend_svn): repo = vcsbackend_svn['svn-simple-layout'] expected_tags = [ 'tags/v0.1', 'tags/v0.2', 'tags/v0.3', 'tags/v0.5'] - assert repo.tags.keys() == expected_tags + assert list(repo.tags.keys()) == expected_tags diff --git a/rhodecode/tests/vcs/test_client_http.py b/rhodecode/tests/vcs/test_client_http.py --- a/rhodecode/tests/vcs/test_client_http.py +++ b/rhodecode/tests/vcs/test_client_http.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/vcs/test_commits.py b/rhodecode/tests/vcs/test_commits.py --- a/rhodecode/tests/vcs/test_commits.py +++ b/rhodecode/tests/vcs/test_commits.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -23,6 +22,7 @@ import time import pytest +from rhodecode.lib.str_utils import safe_bytes from rhodecode.lib.vcs.backends.base import ( CollectionGenerator, FILEMODE_DEFAULT, EmptyCommit) from rhodecode.lib.vcs.exceptions import ( @@ -62,7 +62,8 @@ class TestCommitsInNonEmptyRepo(BackendT 'author': 'Joe Doe ', 'date': start_date + datetime.timedelta(hours=12 * x), 'added': [ - FileNode('file_%d.txt' % x, content='Foobar %d' % x), + FileNode(b'file_%d.txt' % x, + content=b'Foobar %d' % x), ], } @@ -72,42 +73,39 @@ class TestCommitsInNonEmptyRepo(BackendT @pytest.mark.backends("git", "hg") def test_new_branch(self): - self.imc.add(FileNode('docs/index.txt', - content='Documentation\n')) + self.imc.add(FileNode(b'docs/index.txt', content=b'Documentation\n')) foobar_tip = self.imc.commit( - message=u'New branch: foobar', - author=u'joe ', + message='New branch: foobar', + author='joe ', branch='foobar', ) assert 'foobar' in self.repo.branches assert foobar_tip.branch == 'foobar' # 'foobar' should be the only branch that contains the new commit - branch = self.repo.branches.values() + branch = list(self.repo.branches.values()) assert branch[0] != branch[1] @pytest.mark.backends("git", "hg") def test_new_head_in_default_branch(self): tip = self.repo.get_commit() - self.imc.add(FileNode('docs/index.txt', - content='Documentation\n')) + self.imc.add(FileNode(b'docs/index.txt', content=b'Documentation\n')) foobar_tip = self.imc.commit( - message=u'New branch: foobar', - author=u'joe ', + message='New branch: foobar', + author='joe ', branch='foobar', parents=[tip], ) - self.imc.change(FileNode('docs/index.txt', - content='Documentation\nand more...\n')) + self.imc.change(FileNode(b'docs/index.txt', content=b'Documentation\nand more...\n')) newtip = self.imc.commit( - message=u'At default branch', - author=u'joe ', + message='At default branch', + author='joe ', branch=foobar_tip.branch, parents=[foobar_tip], ) newest_tip = self.imc.commit( - message=u'Merged with %s' % foobar_tip.raw_id, - author=u'joe ', + message='Merged with %s' % foobar_tip.raw_id, + author='joe ', branch=self.backend_class.DEFAULT_BRANCH_NAME, parents=[newtip, foobar_tip], ) @@ -131,31 +129,31 @@ class TestCommitsInNonEmptyRepo(BackendT TEST_BRANCH = 'docs' org_tip = self.repo.get_commit() - self.imc.add(FileNode('readme.txt', content='Document\n')) + self.imc.add(FileNode(b'readme.txt', content=b'Document\n')) initial = self.imc.commit( - message=u'Initial commit', - author=u'joe ', + message='Initial commit', + author='joe ', parents=[org_tip], branch=DEFAULT_BRANCH,) - self.imc.add(FileNode('newdoc.txt', content='foobar\n')) + self.imc.add(FileNode(b'newdoc.txt', content=b'foobar\n')) docs_branch_commit1 = self.imc.commit( - message=u'New branch: docs', - author=u'joe ', + message='New branch: docs', + author='joe ', parents=[initial], branch=TEST_BRANCH,) - self.imc.add(FileNode('newdoc2.txt', content='foobar2\n')) + self.imc.add(FileNode(b'newdoc2.txt', content=b'foobar2\n')) docs_branch_commit2 = self.imc.commit( - message=u'New branch: docs2', - author=u'joe ', + message='New branch: docs2', + author='joe ', parents=[docs_branch_commit1], branch=TEST_BRANCH,) - self.imc.add(FileNode('newfile', content='hello world\n')) + self.imc.add(FileNode(b'newfile', content=b'hello world\n')) self.imc.commit( - message=u'Back in default branch', - author=u'joe ', + message='Back in default branch', + author='joe ', parents=[initial], branch=DEFAULT_BRANCH,) @@ -210,11 +208,12 @@ class TestCommits(BackendTestMixin): start_date = datetime.datetime(2010, 1, 1, 20) for x in range(5): yield { - 'message': u'Commit %d' % x, - 'author': u'Joe Doe ', + 'message': 'Commit %d' % x, + 'author': 'Joe Doe ', 'date': start_date + datetime.timedelta(hours=12 * x), 'added': [ - FileNode('file_%d.txt' % x, content='Foobar %d' % x), + FileNode(b'file_%d.txt' % x, + content=b'Foobar %d' % x) ], } @@ -269,24 +268,24 @@ class TestCommits(BackendTestMixin): def test_author(self): tip = self.repo.get_commit() - assert_text_equal(tip.author, u'Joe Doe ') + assert_text_equal(tip.author, 'Joe Doe ') def test_author_name(self): tip = self.repo.get_commit() - assert_text_equal(tip.author_name, u'Joe Doe') + assert_text_equal(tip.author_name, 'Joe Doe') def test_author_email(self): tip = self.repo.get_commit() - assert_text_equal(tip.author_email, u'joe.doe@example.com') + assert_text_equal(tip.author_email, 'joe.doe@example.com') def test_message(self): tip = self.repo.get_commit() - assert_text_equal(tip.message, u'Commit 4') + assert_text_equal(tip.message, 'Commit 4') def test_diff(self): tip = self.repo.get_commit() diff = tip.diff() - assert "+Foobar 4" in diff.raw + assert b"+Foobar 4" in diff.raw.tobytes() def test_prev(self): tip = self.repo.get_commit() @@ -490,8 +489,8 @@ class TestCommits(BackendTestMixin): assert commit2 == commit2 assert commit1 != commit2 assert commit2 != commit1 - assert commit1 != None - assert None != commit1 + assert commit1 is not None + assert commit2 is not None assert 1 != commit1 assert 'string' != commit1 @@ -514,53 +513,49 @@ class TestCommitsChanges(BackendTestMixi def _get_commits(cls): return [ { - 'message': u'Initial', - 'author': u'Joe Doe ', + 'message': 'Initial', + 'author': 'Joe Doe ', 'date': datetime.datetime(2010, 1, 1, 20), 'added': [ - FileNode('foo/bar', content='foo'), - FileNode('foo/bał', content='foo'), - FileNode('foobar', content='foo'), - FileNode('qwe', content='foo'), + FileNode(b'foo/bar', content=b'foo'), + FileNode(safe_bytes('foo/bał'), content=b'foo'), + FileNode(b'foobar', content=b'foo'), + FileNode(b'qwe', content=b'foo'), ], }, { - 'message': u'Massive changes', - 'author': u'Joe Doe ', + 'message': 'Massive changes', + 'author': 'Joe Doe ', 'date': datetime.datetime(2010, 1, 1, 22), - 'added': [FileNode('fallout', content='War never changes')], + 'added': [FileNode(b'fallout', content=b'War never changes')], 'changed': [ - FileNode('foo/bar', content='baz'), - FileNode('foobar', content='baz'), + FileNode(b'foo/bar', content=b'baz'), + FileNode(b'foobar', content=b'baz'), ], - 'removed': [FileNode('qwe')], + 'removed': [FileNode(b'qwe')], }, ] def test_initial_commit(self, local_dt_to_utc): commit = self.repo.get_commit(commit_idx=0) - assert set(commit.added) == set([ + assert set(commit.added) == { commit.get_node('foo/bar'), commit.get_node('foo/bał'), commit.get_node('foobar'), - commit.get_node('qwe'), - ]) + commit.get_node('qwe') + } assert set(commit.changed) == set() assert set(commit.removed) == set() - assert set(commit.affected_files) == set( - ['foo/bar', 'foo/bał', 'foobar', 'qwe']) + assert set(commit.affected_files) == {'foo/bar', 'foo/bał', 'foobar', 'qwe'} assert commit.date == local_dt_to_utc( datetime.datetime(2010, 1, 1, 20, 0)) def test_head_added(self): commit = self.repo.get_commit() assert isinstance(commit.added, AddedFileNodesGenerator) - assert set(commit.added) == set([commit.get_node('fallout')]) + assert set(commit.added) == {commit.get_node('fallout')} assert isinstance(commit.changed, ChangedFileNodesGenerator) - assert set(commit.changed) == set([ - commit.get_node('foo/bar'), - commit.get_node('foobar'), - ]) + assert set(commit.changed) == {commit.get_node('foo/bar'), commit.get_node('foobar')} assert isinstance(commit.removed, RemovedFileNodesGenerator) assert len(commit.removed) == 1 assert list(commit.removed)[0].path == 'qwe' @@ -572,7 +567,7 @@ class TestCommitsChanges(BackendTestMixi def test_get_filemode_non_ascii(self): commit = self.repo.get_commit() assert FILEMODE_DEFAULT == commit.get_file_mode('foo/bał') - assert FILEMODE_DEFAULT == commit.get_file_mode(u'foo/bał') + assert FILEMODE_DEFAULT == commit.get_file_mode('foo/bał') def test_get_path_history(self): commit = self.repo.get_commit() diff --git a/rhodecode/tests/vcs/test_config.py b/rhodecode/tests/vcs/test_config.py --- a/rhodecode/tests/vcs/test_config.py +++ b/rhodecode/tests/vcs/test_config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/vcs/test_diff.py b/rhodecode/tests/vcs/test_diff.py --- a/rhodecode/tests/vcs/test_diff.py +++ b/rhodecode/tests/vcs/test_diff.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -34,6 +33,7 @@ class TestGetDiffValidation: def test_raises_if_commits_not_of_this_repository(self, vcsbackend): repo = vcsbackend.repo + target_repo = vcsbackend.create_repo(number_of_commits=1) repo_commit = repo[0] wrong_commit = target_repo[0] @@ -81,8 +81,8 @@ class TestRepositoryGetDiff(BackendTestM 'author': 'Joe Doe ', 'date': datetime.datetime(2010, 1, 1, 20), 'added': [ - FileNode('foobar', content='foobar'), - FileNode('foobar2', content='foobar2'), + FileNode(b'foobar', content=b'foobar'), + FileNode(b'foobar2', content=b'foobar2'), ], }, { @@ -90,10 +90,10 @@ class TestRepositoryGetDiff(BackendTestM 'author': 'Jane Doe ', 'date': datetime.datetime(2010, 1, 1, 21), 'added': [ - FileNode('foobar3', content='foobar3'), + FileNode(b'foobar3', content=b'foobar3'), ], 'changed': [ - FileNode('foobar', 'FOOBAR'), + FileNode(b'foobar', b'FOOBAR'), ], }, { @@ -101,16 +101,16 @@ class TestRepositoryGetDiff(BackendTestM 'author': 'Jane Doe ', 'date': datetime.datetime(2010, 1, 1, 22), 'changed': [ - FileNode('foobar3', content='FOOBAR\nFOOBAR\nFOOBAR\n'), + FileNode(b'foobar3', content=b'FOOBAR\nFOOBAR\nFOOBAR\n'), ], - 'removed': [FileNode('foobar')], + 'removed': [FileNode(b'foobar')], }, { 'message': 'Whitespace changes', 'author': 'Jane Doe ', 'date': datetime.datetime(2010, 1, 1, 23), 'changed': [ - FileNode('foobar3', content='FOOBAR \nFOOBAR\nFOOBAR\n'), + FileNode(b'foobar3', content=b'FOOBAR \nFOOBAR\nFOOBAR\n'), ], }, ] @@ -119,39 +119,39 @@ class TestRepositoryGetDiff(BackendTestM def test_initial_commit_diff(self): initial_commit = self.repo[0] diff = self.repo.get_diff(self.repo.EMPTY_COMMIT, initial_commit) - assert diff.raw == self.first_commit_diffs[self.repo.alias] + assert diff.raw.tobytes() == self.first_commit_diffs[self.repo.alias] def test_second_commit_diff(self): diff = self.repo.get_diff(self.repo[0], self.repo[1]) - assert diff.raw == self.second_commit_diffs[self.repo.alias] + assert diff.raw.tobytes() == self.second_commit_diffs[self.repo.alias] def test_third_commit_diff(self): diff = self.repo.get_diff(self.repo[1], self.repo[2]) - assert diff.raw == self.third_commit_diffs[self.repo.alias] + assert diff.raw.tobytes() == self.third_commit_diffs[self.repo.alias] def test_ignore_whitespace(self): diff = self.repo.get_diff( self.repo[2], self.repo[3], ignore_whitespace=True) - assert '@@' not in diff.raw + assert b'@@' not in diff.raw.tobytes() def test_only_one_file(self): diff = self.repo.get_diff( self.repo.EMPTY_COMMIT, self.repo[0], path='foobar') - assert 'foobar2' not in diff.raw + assert b'foobar2' not in diff.raw.tobytes() def test_context_parameter(self): first_commit = self.repo.get_commit(commit_idx=0) diff = self.repo.get_diff( self.repo.EMPTY_COMMIT, first_commit, context=2) - assert diff.raw == self.first_commit_diffs[self.repo.alias] + assert diff.raw.tobytes() == self.first_commit_diffs[self.repo.alias] def test_context_only_one_file(self): diff = self.repo.get_diff( self.repo.EMPTY_COMMIT, self.repo[0], path='foobar', context=2) - assert diff.raw == self.first_commit_one_file[self.repo.alias] + assert diff.raw.tobytes() == self.first_commit_one_file[self.repo.alias] first_commit_diffs = { - 'git': r"""diff --git a/foobar b/foobar + 'git': br"""diff --git a/foobar b/foobar new file mode 100644 index 0000000..f6ea049 --- /dev/null @@ -168,7 +168,7 @@ index 0000000..e8c9d6b +foobar2 \ No newline at end of file """, - 'hg': r"""diff --git a/foobar b/foobar + 'hg': br"""diff --git a/foobar b/foobar new file mode 100644 --- /dev/null +++ b/foobar @@ -183,7 +183,7 @@ new file mode 100644 +foobar2 \ No newline at end of file """, - 'svn': """Index: foobar + 'svn': b"""Index: foobar =================================================================== diff --git a/foobar b/foobar new file mode 10644 @@ -205,7 +205,7 @@ new file mode 10644 } second_commit_diffs = { - 'git': r"""diff --git a/foobar b/foobar + 'git': br"""diff --git a/foobar b/foobar index f6ea049..389865b 100644 --- a/foobar +++ b/foobar @@ -223,7 +223,7 @@ index 0000000..c11c37d +foobar3 \ No newline at end of file """, - 'hg': r"""diff --git a/foobar b/foobar + 'hg': br"""diff --git a/foobar b/foobar --- a/foobar +++ b/foobar @@ -1,1 +1,1 @@ @@ -239,7 +239,7 @@ new file mode 100644 +foobar3 \ No newline at end of file """, - 'svn': """Index: foobar + 'svn': b"""Index: foobar =================================================================== diff --git a/foobar b/foobar --- a/foobar\t(revision 1) @@ -262,7 +262,7 @@ new file mode 10644 } third_commit_diffs = { - 'git': r"""diff --git a/foobar b/foobar + 'git': br"""diff --git a/foobar b/foobar deleted file mode 100644 index 389865b..0000000 --- a/foobar @@ -281,7 +281,7 @@ index c11c37d..f932447 100644 +FOOBAR +FOOBAR """, - 'hg': r"""diff --git a/foobar b/foobar + 'hg': br"""diff --git a/foobar b/foobar deleted file mode 100644 --- a/foobar +++ /dev/null @@ -298,7 +298,7 @@ diff --git a/foobar3 b/foobar3 +FOOBAR +FOOBAR """, - 'svn': """Index: foobar + 'svn': b"""Index: foobar =================================================================== diff --git a/foobar b/foobar deleted file mode 10644 @@ -322,7 +322,7 @@ diff --git a/foobar3 b/foobar3 } first_commit_one_file = { - 'git': r"""diff --git a/foobar b/foobar + 'git': br"""diff --git a/foobar b/foobar new file mode 100644 index 0000000..f6ea049 --- /dev/null @@ -331,7 +331,7 @@ index 0000000..f6ea049 +foobar \ No newline at end of file """, - 'hg': r"""diff --git a/foobar b/foobar + 'hg': br"""diff --git a/foobar b/foobar new file mode 100644 --- /dev/null +++ b/foobar @@ -339,7 +339,7 @@ new file mode 100644 +foobar \ No newline at end of file """, - 'svn': """Index: foobar + 'svn': b"""Index: foobar =================================================================== diff --git a/foobar b/foobar new file mode 10644 @@ -363,9 +363,9 @@ class TestSvnGetDiff(object): commit1 = repo[-2] commit2 = repo[-1] diff = repo.get_diff(commit1, commit2, path=path, path1=path1) - assert diff.raw == self.expected_diff_v_0_2 + assert diff.raw.tobytes() == self.expected_diff_v_0_2 - expected_diff_v_0_2 = '''Index: example.py + expected_diff_v_0_2 = b'''Index: example.py =================================================================== diff --git a/example.py b/example.py --- a/example.py\t(revision 25) @@ -390,7 +390,7 @@ diff --git a/example.py b/example.py diff = repo.get_diff(repo[0], repo[1]) # TODO: johbo: Think about supporting svn directory nodes # a little bit better, source is here like a file - expected_diff = """Index: source + expected_diff = b"""Index: source =================================================================== diff --git a/source b/source deleted file mode 10644 @@ -403,7 +403,7 @@ new file mode 10644 --- /dev/null\t(revision 0) +++ b/target/file\t(revision 2) """ - assert diff.raw == expected_diff + assert diff.raw.tobytes() == expected_diff @pytest.mark.usefixtures("vcs_repository_support") @@ -412,8 +412,8 @@ class TestGetDiffBinary(BackendTestMixin recreate_repo_per_test = False # Note: "Fake" PNG files, has the correct magic as prefix - BINARY = """\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00""" - BINARY2 = """\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x01\x00\x00""" + BINARY = b"""\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00""" + BINARY2 = b"""\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x01\x00\x00""" @staticmethod def _get_commits(): @@ -423,21 +423,21 @@ class TestGetDiffBinary(BackendTestMixin 'author': 'Joe Doe ', 'date': datetime.datetime(2010, 1, 1, 20), 'added': [ - FileNode('image.png', content=TestGetDiffBinary.BINARY), + FileNode(b'image.png', content=TestGetDiffBinary.BINARY), ]}, { 'message': 'Modify image.png', 'author': 'Joe Doe ', 'date': datetime.datetime(2010, 1, 1, 21), 'changed': [ - FileNode('image.png', content=TestGetDiffBinary.BINARY2), + FileNode(b'image.png', content=TestGetDiffBinary.BINARY2), ]}, { 'message': 'Remove image.png', 'author': 'Joe Doe ', 'date': datetime.datetime(2010, 1, 1, 21), 'removed': [ - FileNode('image.png'), + FileNode(b'image.png'), ]}, ] return commits @@ -446,7 +446,7 @@ class TestGetDiffBinary(BackendTestMixin diff = self.repo.get_diff(self.repo.EMPTY_COMMIT, self.repo[0]) expected = { - 'git': """diff --git a/image.png b/image.png + 'git': b"""diff --git a/image.png b/image.png new file mode 100644 index 0000000000000000000000000000000000000000..28380fd4a25c58be1b68b523ba2a314f4459ee9c GIT binary patch @@ -457,15 +457,15 @@ literal 0 Hc$@', 'date': start_date + datetime.timedelta(hours=12 * x), 'added': [ - FileNode('file_%d.txt' % x, content='Foobar %d' % x), + FileNode(b'file_%d.txt' % x, content='Foobar %d' % x), ], } diff --git a/rhodecode/tests/vcs/test_getslice.py b/rhodecode/tests/vcs/test_getslice.py --- a/rhodecode/tests/vcs/test_getslice.py +++ b/rhodecode/tests/vcs/test_getslice.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -36,7 +35,7 @@ class TestGetslice(BackendTestMixin): 'author': 'Joe Doe ', 'date': start_date + datetime.timedelta(hours=12 * x), 'added': [ - FileNode('file_%d.txt' % x, content='Foobar %d' % x), + FileNode(b'file_%d.txt' % x, content='Foobar %d' % x), ], } diff --git a/rhodecode/tests/vcs/test_git.py b/rhodecode/tests/vcs/test_git.py --- a/rhodecode/tests/vcs/test_git.py +++ b/rhodecode/tests/vcs/test_git.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -41,6 +40,29 @@ from rhodecode.tests.vcs.conftest import pytestmark = pytest.mark.backends("git") +DIFF_FROM_REMOTE = br"""diff --git a/foobar b/foobar +new file mode 100644 +index 0000000..f6ea049 +--- /dev/null ++++ b/foobar +@@ -0,0 +1 @@ ++foobar +\ No newline at end of file +diff --git a/foobar2 b/foobar2 +new file mode 100644 +index 0000000..e8c9d6b +--- /dev/null ++++ b/foobar2 +@@ -0,0 +1 @@ ++foobar2 +\ No newline at end of file +""" + + +def callable_get_diff(*args, **kwargs): + return DIFF_FROM_REMOTE + + class TestGitRepository(object): @pytest.fixture(autouse=True) @@ -253,7 +275,7 @@ TODO: To be written... """ node = commit10.get_node('README.rst') assert node.kind == NodeKind.FILE - assert node.content == README + assert node.str_content == README def test_head(self): assert self.repo.head == self.repo.get_commit().raw_id @@ -420,7 +442,7 @@ TODO: To be written... def test_local_merge_raises_exception_on_conflict(self, vcsbackend_git): target_repo = vcsbackend_git.create_repo(number_of_commits=1) - vcsbackend_git.ensure_file('README', 'I will conflict with you!!!') + vcsbackend_git.ensure_file(b'README', b'I will conflict with you!!!') target_repo._local_fetch(self.repo.path, 'master') with pytest.raises(RepositoryError): @@ -971,11 +993,12 @@ class TestGitCommit(object): for commit in self.repo: assert type(commit.author) == str - def test_repo_files_content_is_unicode(self): + def test_repo_files_content_types(self): commit = self.repo.get_commit() for node in commit.get_node('/'): if node.is_file(): - assert type(node.content) == str + assert type(node.content) == bytes + assert type(node.str_content) == str def test_wrong_path(self): # There is 'setup.py' in the root dir but not there: @@ -1041,11 +1064,9 @@ class TestGitSpecificWithRepo(BackendTes 'author': 'Joe Doe ', 'date': datetime.datetime(2010, 1, 1, 20), 'added': [ - FileNode('foobar/static/js/admin/base.js', content='base'), - FileNode( - 'foobar/static/admin', content='admin', - mode=0o120000), # this is a link - FileNode('foo', content='foo'), + FileNode(b'foobar/static/js/admin/base.js', content=b'base'), + FileNode(b'foobar/static/admin', content=b'admin', mode=0o120000), # this is a link + FileNode(b'foo', content=b'foo'), ], }, { @@ -1053,7 +1074,7 @@ class TestGitSpecificWithRepo(BackendTes 'author': 'Joe Doe ', 'date': datetime.datetime(2010, 1, 1, 22), 'added': [ - FileNode('foo2', content='foo2'), + FileNode(b'foo2', content=b'foo2'), ], }, ] @@ -1061,17 +1082,18 @@ class TestGitSpecificWithRepo(BackendTes def test_paths_slow_traversing(self): commit = self.repo.get_commit() assert commit.get_node('foobar').get_node('static').get_node('js')\ - .get_node('admin').get_node('base.js').content == 'base' + .get_node('admin').get_node('base.js').content == b'base' def test_paths_fast_traversing(self): commit = self.repo.get_commit() - assert commit.get_node('foobar/static/js/admin/base.js').content == 'base' + assert commit.get_node('foobar/static/js/admin/base.js').content == b'base' def test_get_diff_runs_git_command_with_hashes(self): comm1 = self.repo[0] comm2 = self.repo[1] - with mock.patch.object(self.repo, '_remote') as remote_mock: + with mock.patch.object(self.repo, '_remote', return_value=mock.Mock()) as remote_mock: + remote_mock.diff = mock.MagicMock(side_effect=callable_get_diff) self.repo.get_diff(comm1, comm2) remote_mock.diff.assert_called_once_with( @@ -1080,8 +1102,11 @@ class TestGitSpecificWithRepo(BackendTes def test_get_diff_runs_git_command_with_str_hashes(self): comm2 = self.repo[1] - with mock.patch.object(self.repo, '_remote') as remote_mock: + + with mock.patch.object(self.repo, '_remote', return_value=mock.Mock()) as remote_mock: + remote_mock.diff = mock.MagicMock(side_effect=callable_get_diff) self.repo.get_diff(self.repo.EMPTY_COMMIT, comm2) + remote_mock.diff.assert_called_once_with( self.repo.EMPTY_COMMIT.raw_id, comm2.raw_id, file_filter=None, opt_ignorews=False, context=3) @@ -1089,8 +1114,11 @@ class TestGitSpecificWithRepo(BackendTes def test_get_diff_runs_git_command_with_path_if_its_given(self): comm1 = self.repo[0] comm2 = self.repo[1] - with mock.patch.object(self.repo, '_remote') as remote_mock: + + with mock.patch.object(self.repo, '_remote', return_value=mock.Mock()) as remote_mock: + remote_mock.diff = mock.MagicMock(side_effect=callable_get_diff) self.repo.get_diff(comm1, comm2, 'foo') + remote_mock.diff.assert_called_once_with( self.repo._lookup_commit(0), comm2.raw_id, file_filter='foo', opt_ignorews=False, context=3) @@ -1107,9 +1135,9 @@ class TestGitRegression(BackendTestMixin 'author': 'Joe Doe ', 'date': datetime.datetime(2010, 1, 1, 20), 'added': [ - FileNode('bot/__init__.py', content='base'), - FileNode('bot/templates/404.html', content='base'), - FileNode('bot/templates/500.html', content='base'), + FileNode(b'bot/__init__.py', content=b'base'), + FileNode(b'bot/templates/404.html', content=b'base'), + FileNode(b'bot/templates/500.html', content=b'base'), ], }, { @@ -1117,14 +1145,12 @@ class TestGitRegression(BackendTestMixin 'author': 'Joe Doe ', 'date': datetime.datetime(2010, 1, 1, 22), 'added': [ - FileNode('bot/build/migrations/1.py', content='foo2'), - FileNode('bot/build/migrations/2.py', content='foo2'), - FileNode( - 'bot/build/static/templates/f.html', content='foo2'), - FileNode( - 'bot/build/static/templates/f1.html', content='foo2'), - FileNode('bot/build/templates/err.html', content='foo2'), - FileNode('bot/build/templates/err2.html', content='foo2'), + FileNode(b'bot/build/migrations/1.py', content=b'foo2'), + FileNode(b'bot/build/migrations/2.py', content=b'foo2'), + FileNode(b'bot/build/static/templates/f.html', content=b'foo2'), + FileNode(b'bot/build/static/templates/f1.html', content=b'foo2'), + FileNode(b'bot/build/templates/err.html', content=b'foo2'), + FileNode(b'bot/build/templates/err2.html', content=b'foo2'), ], }, ] diff --git a/rhodecode/tests/vcs/test_hg.py b/rhodecode/tests/vcs/test_hg.py --- a/rhodecode/tests/vcs/test_hg.py +++ b/rhodecode/tests/vcs/test_hg.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -23,6 +22,7 @@ import os import mock import pytest +from rhodecode.lib.str_utils import safe_bytes from rhodecode.lib.utils import make_db_config from rhodecode.lib.vcs import backends from rhodecode.lib.vcs.backends.base import ( @@ -84,23 +84,23 @@ class TestMercurialRepository(object): def test_unicode_path_repo(self): with pytest.raises(VCSError): - MercurialRepository(u'iShouldFail') + MercurialRepository('iShouldFail') def test_unicode_commit_id(self): with pytest.raises(CommitDoesNotExistError): - self.repo.get_commit(u'unicode-commit-id') + self.repo.get_commit('unicode-commit-id') with pytest.raises(CommitDoesNotExistError): - self.repo.get_commit(u'unícøde-spéçial-chäråcter-commit-id') + self.repo.get_commit('unícøde-spéçial-chäråcter-commit-id') def test_unicode_bookmark(self): - self.repo.bookmark(u'unicode-bookmark') - self.repo.bookmark(u'unícøde-spéçial-chäråcter-bookmark') + self.repo.bookmark('unicode-bookmark') + self.repo.bookmark('unícøde-spéçial-chäråcter-bookmark') def test_unicode_branch(self): with pytest.raises(KeyError): - self.repo.branches[u'unicode-branch'] + assert self.repo.branches['unicode-branch'] with pytest.raises(KeyError): - self.repo.branches[u'unícøde-spéçial-chäråcter-branch'] + assert self.repo.branches['unícøde-spéçial-chäråcter-branch'] def test_repo_clone(self): if os.path.exists(TEST_HG_REPO_CLONE): @@ -141,27 +141,17 @@ class TestMercurialRepository(object): def test_commit_ids(self): # there are 21 commits at bitbucket now # so we can assume they would be available from now on - subset = set([ - 'b986218ba1c9b0d6a259fac9b050b1724ed8e545', - '3d8f361e72ab303da48d799ff1ac40d5ac37c67e', - '6cba7170863a2411822803fa77a0a264f1310b35', - '56349e29c2af3ac913b28bde9a2c6154436e615b', - '2dda4e345facb0ccff1a191052dd1606dba6781d', - '6fff84722075f1607a30f436523403845f84cd9e', - '7d4bc8ec6be56c0f10425afb40b6fc315a4c25e7', - '3803844fdbd3b711175fc3da9bdacfcd6d29a6fb', - 'dc5d2c0661b61928834a785d3e64a3f80d3aad9c', - 'be90031137367893f1c406e0a8683010fd115b79', - 'db8e58be770518cbb2b1cdfa69146e47cd481481', - '84478366594b424af694a6c784cb991a16b87c21', - '17f8e105dddb9f339600389c6dc7175d395a535c', - '20a662e756499bde3095ffc9bc0643d1def2d0eb', - '2e319b85e70a707bba0beff866d9f9de032aa4f9', - '786facd2c61deb9cf91e9534735124fb8fc11842', - '94593d2128d38210a2fcd1aabff6dda0d6d9edf8', - 'aa6a0de05b7612707db567078e130a6cd114a9a7', - 'eada5a770da98ab0dd7325e29d00e0714f228d09' - ]) + subset = {'b986218ba1c9b0d6a259fac9b050b1724ed8e545', '3d8f361e72ab303da48d799ff1ac40d5ac37c67e', + '6cba7170863a2411822803fa77a0a264f1310b35', '56349e29c2af3ac913b28bde9a2c6154436e615b', + '2dda4e345facb0ccff1a191052dd1606dba6781d', '6fff84722075f1607a30f436523403845f84cd9e', + '7d4bc8ec6be56c0f10425afb40b6fc315a4c25e7', '3803844fdbd3b711175fc3da9bdacfcd6d29a6fb', + 'dc5d2c0661b61928834a785d3e64a3f80d3aad9c', 'be90031137367893f1c406e0a8683010fd115b79', + 'db8e58be770518cbb2b1cdfa69146e47cd481481', '84478366594b424af694a6c784cb991a16b87c21', + '17f8e105dddb9f339600389c6dc7175d395a535c', '20a662e756499bde3095ffc9bc0643d1def2d0eb', + '2e319b85e70a707bba0beff866d9f9de032aa4f9', '786facd2c61deb9cf91e9534735124fb8fc11842', + '94593d2128d38210a2fcd1aabff6dda0d6d9edf8', 'aa6a0de05b7612707db567078e130a6cd114a9a7', + 'eada5a770da98ab0dd7325e29d00e0714f228d09' + } assert subset.issubset(set(self.repo.commit_ids)) # check if we have the proper order of commits @@ -301,7 +291,7 @@ TODO: To be written... """ node = commit10.get_node('README.rst') assert node.kind == NodeKind.FILE - assert node.content == README + assert node.str_content == README def test_local_clone(self): clone_path = next(REPO_PATH_GENERATOR) @@ -583,10 +573,10 @@ TODO: To be written... vcsbackend_hg.add_file(target_repo, 'README_MERGE1', 'Version 1') vcsbackend_hg.add_file(source_repo, 'README_MERGE2', 'Version 2') imc = source_repo.in_memory_commit - imc.add(FileNode('file_x', content=source_repo.name)) + imc.add(FileNode(b'file_x', content=source_repo.name)) imc.commit( - message=u'Automatic commit from repo merge test', - author=u'Automatic ') + message='Automatic commit from repo merge test', + author='Automatic ') target_commit = target_repo.get_commit() source_commit = source_repo.get_commit() default_branch = target_repo.DEFAULT_BRANCH_NAME @@ -627,10 +617,10 @@ TODO: To be written... target_repo = vcsbackend_hg.create_repo(number_of_commits=1) source_repo = vcsbackend_hg.clone_repo(target_repo) imc = source_repo.in_memory_commit - imc.add(FileNode('file_x', content=source_repo.name)) + imc.add(FileNode(b'file_x', content=source_repo.name)) imc.commit( - message=u'Automatic commit from repo merge test', - author=u'Automatic ') + message='Automatic commit from repo merge test', + author='Automatic ') target_commit = target_repo.get_commit() source_commit = source_repo.get_commit() default_branch = target_repo.DEFAULT_BRANCH_NAME @@ -665,11 +655,11 @@ TODO: To be written... # add an extra head to the target repo imc = target_repo.in_memory_commit - imc.add(FileNode('file_x', content='foo')) + imc.add(FileNode(b'file_x', content='foo')) commits = list(target_repo.get_commits()) imc.commit( - message=u'Automatic commit from repo merge test', - author=u'Automatic ', parents=commits[0:1]) + message='Automatic commit from repo merge test', + author='Automatic ', parents=commits[0:1]) target_commit = target_repo.get_commit() source_commit = source_repo.get_commit() @@ -698,11 +688,13 @@ TODO: To be written... source_repo = vcsbackend_hg.clone_repo(target_repo) vcsbackend_hg.add_file(target_repo, 'README_MERGE1', 'Version 1') vcsbackend_hg.add_file(source_repo, 'README_MERGE2', 'Version 2') + imc = source_repo.in_memory_commit - imc.add(FileNode('file_x', content=source_repo.name)) + imc.add(FileNode(b'file_x', content=safe_bytes(source_repo.name))) imc.commit( - message=u'Automatic commit from repo merge test', - author=u'Automatic ') + message='Automatic commit from repo merge test', + author='Automatic ') + target_commit = target_repo.get_commit() source_commit = source_repo.get_commit() @@ -1090,11 +1082,12 @@ class TestMercurialCommit(object): for cm in self.repo: assert type(cm.author) == str - def test_repo_files_content_is_unicode(self): + def test_repo_files_content_type(self): test_commit = self.repo.get_commit(commit_idx=100) for node in test_commit.get_node('/'): if node.is_file(): - assert type(node.content) == str + assert type(node.content) == bytes + assert type(node.str_content) == str def test_wrong_path(self): # There is 'setup.py' in the root dir but not there: diff --git a/rhodecode/tests/vcs/test_hg_vcsserver_cache_invalidation.py b/rhodecode/tests/vcs/test_hg_vcsserver_cache_invalidation.py --- a/rhodecode/tests/vcs/test_hg_vcsserver_cache_invalidation.py +++ b/rhodecode/tests/vcs/test_hg_vcsserver_cache_invalidation.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- + # Copyright (C) 2016-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/vcs/test_inmemory.py b/rhodecode/tests/vcs/test_inmemory.py --- a/rhodecode/tests/vcs/test_inmemory.py +++ b/rhodecode/tests/vcs/test_inmemory.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -25,7 +24,7 @@ import datetime import pytest -from rhodecode.lib.utils2 import safe_unicode +from rhodecode.lib.str_utils import safe_bytes, safe_str from rhodecode.lib.vcs.exceptions import ( EmptyRepositoryError, NodeAlreadyAddedError, NodeAlreadyExistsError, NodeAlreadyRemovedError, NodeAlreadyChangedError, NodeDoesNotExistError, @@ -37,20 +36,18 @@ from rhodecode.tests.vcs.conftest import @pytest.fixture() def nodes(): nodes = [ - FileNode('foobar', content='Foo & bar'), - FileNode('foobar2', content='Foo & bar, doubled!'), - FileNode('foo bar with spaces', content=''), - FileNode('foo/bar/baz', content='Inside'), - FileNode( - 'foo/bar/file.bin', - content=( - '\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1\x00\x00\x00\x00\x00\x00' - '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00;\x00\x03\x00\xfe' - '\xff\t\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' - '\x01\x00\x00\x00\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00' - '\x00\x18\x00\x00\x00\x01\x00\x00\x00\xfe\xff\xff\xff\x00\x00' - '\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff' - '\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' + FileNode(b'foobar', content=b'Foo & bar'), + FileNode(b'foobar2', content=b'Foo & bar, doubled!'), + FileNode(b'foo bar with spaces', content=b''), + FileNode(b'foo/bar/baz', content=b'Inside'), + FileNode(b'foo/bar/file.bin', content=( + b'\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00;\x00\x03\x00\xfe' + b'\xff\t\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x01\x00\x00\x00\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00' + b'\x00\x18\x00\x00\x00\x01\x00\x00\x00\xfe\xff\xff\xff\x00\x00' + b'\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff' + b'\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff' ) ), ] @@ -77,94 +74,98 @@ class TestInMemoryCommit(BackendTestMixi self.imc.add(node) self.commit() - self.assert_succesful_commit(nodes) + self.assert_successful_commit(nodes) @pytest.mark.backends("hg") def test_add_on_branch_hg(self, nodes): for node in nodes: self.imc.add(node) - self.commit(branch=u'stable') - self.assert_succesful_commit(nodes) + self.commit(branch='stable') + self.assert_successful_commit(nodes) @pytest.mark.backends("git") def test_add_on_branch_git(self, nodes): for node in nodes: self.imc.add(node) - self.commit(branch=u'stable') - self.assert_succesful_commit(nodes) + self.commit(branch='stable') + self.assert_successful_commit(nodes) def test_add_in_bulk(self, nodes): self.imc.add(*nodes) self.commit() - self.assert_succesful_commit(nodes) + self.assert_successful_commit(nodes) def test_add_non_ascii_files(self): nodes = [ - FileNode('żółwik/zwierzątko_utf8_str', content='ćććć'), - FileNode(u'żółwik/zwierzątko_unicode', content=u'ćććć'), + FileNode(safe_bytes('żółwik/zwierzątko_utf8_str'), + content=safe_bytes('ćććć')), + FileNode(safe_bytes('żółwik/zwierzątko_unicode'), + content=safe_bytes('ćććć')), ] for node in nodes: self.imc.add(node) self.commit() - self.assert_succesful_commit(nodes) + self.assert_successful_commit(nodes) def commit(self, branch=None): self.old_commit_count = len(self.repo.commit_ids) - self.commit_message = u'Test commit with unicode: żółwik' - self.commit_author = u'{} '.format(self.__class__.__name__) + self.commit_message = 'Test commit with unicode: żółwik' + self.commit_author = f'{self.__class__.__name__} ' self.commit = self.imc.commit( message=self.commit_message, author=self.commit_author, branch=branch) def test_add_actually_adds_all_nodes_at_second_commit_too(self): to_add = [ - FileNode('foo/bar/image.png', content='\0'), - FileNode('foo/README.txt', content='readme!'), + FileNode(b'foo/bar/image.png', content=b'\0'), + FileNode(b'foo/README.txt', content=b'readme!'), ] self.imc.add(*to_add) - commit = self.imc.commit(u'Initial', u'joe doe ') + commit = self.imc.commit('Initial', 'joe doe ') assert isinstance(commit.get_node('foo'), DirNode) assert isinstance(commit.get_node('foo/bar'), DirNode) self.assert_nodes_in_commit(commit, to_add) # commit some more files again to_add = [ - FileNode('foo/bar/foobaz/bar', content='foo'), - FileNode('foo/bar/another/bar', content='foo'), - FileNode('foo/baz.txt', content='foo'), - FileNode('foobar/foobaz/file', content='foo'), - FileNode('foobar/barbaz', content='foo'), + FileNode(b'foo/bar/foobaz/bar', content=b'foo'), + FileNode(b'foo/bar/another/bar', content=b'foo'), + FileNode(b'foo/baz.txt', content=b'foo'), + FileNode(b'foobar/foobaz/file', content=b'foo'), + FileNode(b'foobar/barbaz', content=b'foo'), ] self.imc.add(*to_add) - commit = self.imc.commit(u'Another', u'joe doe ') + commit = self.imc.commit('Another', 'joe doe ') self.assert_nodes_in_commit(commit, to_add) def test_add_raise_already_added(self): - node = FileNode('foobar', content='baz') + node = FileNode(b'foobar', content=b'baz') self.imc.add(node) with pytest.raises(NodeAlreadyAddedError): self.imc.add(node) def test_check_integrity_raise_already_exist(self): - node = FileNode('foobar', content='baz') + node = FileNode(b'foobar', content=b'baz') self.imc.add(node) - self.imc.commit(message=u'Added foobar', author=u'{} '.format(self)) + self.imc.commit(message='Added foobar', + author='Some Name ') self.imc.add(node) with pytest.raises(NodeAlreadyExistsError): - self.imc.commit(message='new message', author=u'{} '.format(self)) + self.imc.commit(message='new message', + author='Some Name ') def test_change(self): - self.imc.add(FileNode('foo/bar/baz', content='foo')) - self.imc.add(FileNode('foo/fbar', content='foobar')) - tip = self.imc.commit(u'Initial', u'joe doe ') + self.imc.add(FileNode(b'foo/bar/baz', content=b'foo')) + self.imc.add(FileNode(b'foo/fbar', content=b'foobar')) + tip = self.imc.commit('Initial', 'joe doe ') # Change node's content - node = FileNode('foo/bar/baz', content='My **changed** content') + node = FileNode(b'foo/bar/baz', content=b'My **changed** content') self.imc.change(node) - self.imc.commit(u'Changed %s' % node.path, u'joe doe ') + self.imc.commit('Changed %s' % node.path, 'joe doe ') newtip = self.repo.get_commit() assert tip != newtip @@ -173,25 +174,28 @@ class TestInMemoryCommit(BackendTestMixi def test_change_non_ascii(self): to_add = [ - FileNode('żółwik/zwierzątko', content='ćććć'), - FileNode(u'żółwik/zwierzątko_uni', content=u'ćććć'), + FileNode(safe_bytes('żółwik/zwierzątko'), + content=safe_bytes('ćććć')), + FileNode(safe_bytes('żółwik/zwierzątko_uni'), + content=safe_bytes('ćććć')), ] for node in to_add: self.imc.add(node) - tip = self.imc.commit(u'Initial', u'joe doe ') + tip = self.imc.commit('Initial', 'joe doe ') # Change node's content - node = FileNode('żółwik/zwierzątko', content='My **changed** content') + node = FileNode(safe_bytes('żółwik/zwierzątko'), + content=b'My **changed** content') self.imc.change(node) - self.imc.commit(u'Changed %s' % safe_unicode(node.path), - author=u'joe doe ') + self.imc.commit('Changed %s' % safe_str(node.path), + author='joe doe ') - node_uni = FileNode( - u'żółwik/zwierzątko_uni', content=u'My **changed** content') + node_uni = FileNode(safe_bytes('żółwik/zwierzątko_uni'), + content=b'My **changed** content') self.imc.change(node_uni) - self.imc.commit(u'Changed %s' % safe_unicode(node_uni.path), - author=u'joe doe ') + self.imc.commit('Changed %s' % safe_str(node_uni.path), + author='joe doe ') newtip = self.repo.get_commit() assert tip != newtip @@ -200,24 +204,24 @@ class TestInMemoryCommit(BackendTestMixi self.assert_nodes_in_commit(newtip, (node, node_uni)) def test_change_raise_empty_repository(self): - node = FileNode('foobar') + node = FileNode(b'foobar') with pytest.raises(EmptyRepositoryError): self.imc.change(node) def test_check_integrity_change_raise_node_does_not_exist(self): - node = FileNode('foobar', content='baz') + node = FileNode(b'foobar', content=b'baz') self.imc.add(node) - self.imc.commit(message=u'Added foobar', author=u'{} '.format(self)) - node = FileNode('not-foobar', content='') + self.imc.commit(message='Added foobar', author='Some Name ') + node = FileNode(b'not-foobar', content=b'') self.imc.change(node) with pytest.raises(NodeDoesNotExistError): - self.imc.commit(message='Changed not existing node', author=u'{} '.format(self)) + self.imc.commit(message='Changed not existing node', author='Some Name ') def test_change_raise_node_already_changed(self): - node = FileNode('foobar', content='baz') + node = FileNode(b'foobar', content=b'baz') self.imc.add(node) - self.imc.commit(message=u'Added foobar', author=u'{} '.format(self)) - node = FileNode('foobar', content='more baz') + self.imc.commit(message='Added foobar', author='Some Nam ') + node = FileNode(b'foobar', content=b'more baz') self.imc.change(node) with pytest.raises(NodeAlreadyChangedError): self.imc.change(node) @@ -225,18 +229,18 @@ class TestInMemoryCommit(BackendTestMixi def test_check_integrity_change_raise_node_not_changed(self, nodes): self.test_add(nodes) # Performs first commit - node = FileNode(nodes[0].path, content=nodes[0].content) + node = FileNode(nodes[0].bytes_path, content=nodes[0].content) self.imc.change(node) with pytest.raises(NodeNotChangedError): self.imc.commit( - message=u'Trying to mark node as changed without touching it', - author=u'{} '.format(self)) + message='Trying to mark node as changed without touching it', + author='Some Name ') def test_change_raise_node_already_removed(self): - node = FileNode('foobar', content='baz') + node = FileNode(b'foobar', content=b'baz') self.imc.add(node) - self.imc.commit(message=u'Added foobar', author=u'{} '.format(self)) - self.imc.remove(FileNode('foobar')) + self.imc.commit(message='Added foobar', author='Some Name ') + self.imc.remove(FileNode(b'foobar')) with pytest.raises(NodeAlreadyRemovedError): self.imc.change(node) @@ -247,8 +251,7 @@ class TestInMemoryCommit(BackendTestMixi node = nodes[0] assert node.content == tip.get_node(node.path).content self.imc.remove(node) - self.imc.commit( - message=u'Removed %s' % node.path, author=u'{} '.format(self)) + self.imc.commit(message=f'Removed {node.path}', author='Some Name ') newtip = self.repo.get_commit() assert tip != newtip @@ -257,12 +260,12 @@ class TestInMemoryCommit(BackendTestMixi newtip.get_node(node.path) def test_remove_last_file_from_directory(self): - node = FileNode('omg/qwe/foo/bar', content='foobar') + node = FileNode(b'omg/qwe/foo/bar', content=b'foobar') self.imc.add(node) - self.imc.commit(u'added', author=u'joe doe ') + self.imc.commit('added', author='joe doe ') self.imc.remove(node) - tip = self.imc.commit(u'removed', u'joe doe ') + tip = self.imc.commit('removed', 'joe doe ') with pytest.raises(NodeDoesNotExistError): tip.get_node('omg/qwe/foo/bar') @@ -271,22 +274,22 @@ class TestInMemoryCommit(BackendTestMixi with pytest.raises(NodeDoesNotExistError): self.imc.commit( message='Trying to remove node at empty repository', - author=u'{} '.format(self)) + author='Some Name ') def test_check_integrity_remove_raise_node_does_not_exist(self, nodes): self.test_add(nodes) # Performs first commit - node = FileNode('no-such-file') + node = FileNode(b'no-such-file') self.imc.remove(node) with pytest.raises(NodeDoesNotExistError): self.imc.commit( - message=u'Trying to remove not existing node', - author=u'{} '.format(self)) + message='Trying to remove not existing node', + author='Some Name ') def test_remove_raise_node_already_removed(self, nodes): self.test_add(nodes) # Performs first commit - node = FileNode(nodes[0].path) + node = FileNode(nodes[0].bytes_path) self.imc.remove(node) with pytest.raises(NodeAlreadyRemovedError): self.imc.remove(node) @@ -294,15 +297,15 @@ class TestInMemoryCommit(BackendTestMixi def test_remove_raise_node_already_changed(self, nodes): self.test_add(nodes) # Performs first commit - node = FileNode(nodes[0].path, content='Bending time') + node = FileNode(nodes[0].bytes_path, content=b'Bending time') self.imc.change(node) with pytest.raises(NodeAlreadyChangedError): self.imc.remove(node) def test_reset(self): - self.imc.add(FileNode('foo', content='bar')) - # self.imc.change(FileNode('baz', content='new')) - # self.imc.remove(FileNode('qwe')) + self.imc.add(FileNode(b'foo', content=b'bar')) + # self.imc.change(FileNode(b'baz', content='new')) + # self.imc.remove(FileNode(b'qwe')) self.imc.reset() assert not any((self.imc.added, self.imc.changed, self.imc.removed)) @@ -310,11 +313,11 @@ class TestInMemoryCommit(BackendTestMixi N = 3 # number of commits to perform last = None for x in range(N): - fname = 'file%s' % str(x).rjust(5, '0') - content = 'foobar\n' * x + fname = safe_bytes('file%s' % str(x).rjust(5, '0')) + content = safe_bytes('foobar\n' * x) node = FileNode(fname, content=content) self.imc.add(node) - commit = self.imc.commit(u"Commit no. %s" % (x + 1), author=u'vcs ') + commit = self.imc.commit("Commit no. %s" % (x + 1), author='Vcs User ') assert last != commit last = commit @@ -326,16 +329,16 @@ class TestInMemoryCommit(BackendTestMixi assert len(repo.commit_ids) == N def test_date_attr(self, local_dt_to_utc): - node = FileNode('foobar.txt', content='Foobared!') + node = FileNode(b'foobar.txt', content=b'Foobared!') self.imc.add(node) date = datetime.datetime(1985, 1, 30, 1, 45) commit = self.imc.commit( - u"Committed at time when I was born ;-)", - author=u'{} '.format(self), date=date) + "Committed at time when I was born ;-)", + author='Test User ', date=date) assert commit.date == local_dt_to_utc(date) - def assert_succesful_commit(self, added_nodes): + def assert_successful_commit(self, added_nodes): newtip = self.repo.get_commit() assert self.commit == newtip assert self.old_commit_count + 1 == len(self.repo.commit_ids) @@ -346,4 +349,5 @@ class TestInMemoryCommit(BackendTestMixi def assert_nodes_in_commit(self, commit, nodes): for node in nodes: + assert commit.get_node(node.path).path == node.path assert commit.get_node(node.path).content == node.content diff --git a/rhodecode/tests/vcs/test_nodes.py b/rhodecode/tests/vcs/test_nodes.py --- a/rhodecode/tests/vcs/test_nodes.py +++ b/rhodecode/tests/vcs/test_nodes.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -22,6 +21,7 @@ import stat import pytest +from rhodecode.lib.str_utils import safe_bytes from rhodecode.lib.vcs.nodes import DirNode from rhodecode.lib.vcs.nodes import FileNode from rhodecode.lib.vcs.nodes import Node @@ -34,29 +34,29 @@ from rhodecode.tests.vcs.conftest import def binary_filenode(): def node_maker(filename): data = ( - "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00" - "\x10\x08\x06\x00\x00\x00\x1f??a\x00\x00\x00\x04gAMA\x00\x00\xaf?7" - "\x05\x8a?\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq?e<\x00" - "\x00\x025IDAT8?\xa5\x93?K\x94Q\x14\x87\x9f\xf7?Q\x1bs4?\x03\x9a" - "\xa8?B\x02\x8b$\x10[U;i\x13?6h?&h[?\"\x14j?\xa2M\x7fB\x14F\x9aQ?&" - "\x842?\x0b\x89\"\x82??!?\x9c!\x9c2l??{N\x8bW\x9dY\xb4\t/\x1c?=" - "\x9b?}????\xa9*;9!?\x83\x91?[?\\v*?D\x04\'`EpNp\xa2X\'U?pVq\"Sw." - "\x1e?\x08\x01D?jw????\xbc??7{|\x9b?\x89$\x01??W@\x15\x9c\x05q`Lt/" - "\x97?\x94\xa1d?\x18~?\x18?\x18W[%\xb0?\x83??\x14\x88\x8dB?\xa6H" - "\tL\tl\x19>/\x01`\xac\xabx?\x9cl\nx\xb0\x98\x07\x95\x88D$\"q[" - "\x19?d\x00(o\n\xa0??\x7f\xb9\xa4?\x1bF\x1f\x8e\xac\xa8?j??eUU}?.?" - "\x9f\x8cE??x\x94??\r\xbdtoJU5\"0N\x10U?\x00??V\t\x02\x9f\x81?U?" - "\x00\x9eM\xae2?r\x9b7\x83\x82\x8aP3????.?&\"?\xb7ZP \x0cJ?\x80\x15T\x95\x9a\x00??S\x8c\r?\xa1" - "\x03\x07?\x96\x9b\xa7\xab=E??\xa4\xb3?\x19q??B\x91=\x8d??k?J" - "\x0bV\"??\xf7x?\xa1\x00?\\.\x87\x87???\x02F@D\x99],??\x10#?X" - "\xb7=\xb9\x10?Z\x1by???cI??\x1ag?\x92\xbc?T?t[\x92\x81?<_\x17~" - "\x92\x88?H%?\x10Q\x02\x9f\n\x81qQ\x0bm?\x1bX?\xb1AK\xa6\x9e\xb9?u" - "\xb2?1\xbe|/\x92M@\xa2!F?\xa9>\"\r\x92\x8e?>\x9a9Qv\x127?a" - "\xac?Y?8?:??]X???9\x80\xb7?u?\x0b#BZ\x8d=\x1d?p\x00\x00\x00\x00" - "IEND\xaeB`\x82") + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x10\x00\x00\x00" + b"\x10\x08\x06\x00\x00\x00\x1f??a\x00\x00\x00\x04gAMA\x00\x00\xaf?7" + b"\x05\x8a?\x00\x00\x00\x19tEXtSoftware\x00Adobe ImageReadyq?e<\x00" + b"\x00\x025IDAT8?\xa5\x93?K\x94Q\x14\x87\x9f\xf7?Q\x1bs4?\x03\x9a" + b"\xa8?B\x02\x8b$\x10[U;i\x13?6h?&h[?\"\x14j?\xa2M\x7fB\x14F\x9aQ?&" + b"\x842?\x0b\x89\"\x82??!?\x9c!\x9c2l??{N\x8bW\x9dY\xb4\t/\x1c?=" + b"\x9b?}????\xa9*;9!?\x83\x91?[?\\v*?D\x04\'`EpNp\xa2X\'U?pVq\"Sw." + b"\x1e?\x08\x01D?jw????\xbc??7{|\x9b?\x89$\x01??W@\x15\x9c\x05q`Lt/" + b"\x97?\x94\xa1d?\x18~?\x18?\x18W[%\xb0?\x83??\x14\x88\x8dB?\xa6H" + b"\tL\tl\x19>/\x01`\xac\xabx?\x9cl\nx\xb0\x98\x07\x95\x88D$\"q[" + b"\x19?d\x00(o\n\xa0??\x7f\xb9\xa4?\x1bF\x1f\x8e\xac\xa8?j??eUU}?.?" + b"\x9f\x8cE??x\x94??\r\xbdtoJU5\"0N\x10U?\x00??V\t\x02\x9f\x81?U?" + b"\x00\x9eM\xae2?r\x9b7\x83\x82\x8aP3????.?&\"?\xb7ZP \x0cJ?\x80\x15T\x95\x9a\x00??S\x8c\r?\xa1" + b"\x03\x07?\x96\x9b\xa7\xab=E??\xa4\xb3?\x19q??B\x91=\x8d??k?J" + b"\x0bV\"??\xf7x?\xa1\x00?\\.\x87\x87???\x02F@D\x99],??\x10#?X" + b"\xb7=\xb9\x10?Z\x1by???cI??\x1ag?\x92\xbc?T?t[\x92\x81?<_\x17~" + b"\x92\x88?H%?\x10Q\x02\x9f\n\x81qQ\x0bm?\x1bX?\xb1AK\xa6\x9e\xb9?u" + b"\xb2?1\xbe|/\x92M@\xa2!F?\xa9>\"\r\x92\x8e?>\x9a9Qv\x127?a" + b"\xac?Y?8?:??]X???9\x80\xb7?u?\x0b#BZ\x8d=\x1d?p\x00\x00\x00\x00" + b"IEND\xaeB`\x82") return FileNode(filename, content=data) return node_maker @@ -68,8 +68,9 @@ class TestNodeBasics: "kind", [NodeKind.FILE, NodeKind.DIR], ids=["FILE", "DIR"]) def test_init_wrong_paths(self, path, kind): """ - Cannot innitialize Node objects with path with slash at the beginning. + Cannot initialize Node objects with path with slash at the beginning. """ + path = safe_bytes(path) with pytest.raises(NodeError): Node(path, kind) @@ -77,47 +78,49 @@ class TestNodeBasics: @pytest.mark.parametrize( "kind", [NodeKind.FILE, NodeKind.DIR], ids=["FILE", "DIR"]) def test_name(self, path, kind): + path = safe_bytes(path) node = Node(path, kind) assert node.name == 'path' def test_name_root(self): - node = Node('', NodeKind.DIR) + node = Node(b'', NodeKind.DIR) assert node.name == '' def test_root_node_cannot_be_file(self): with pytest.raises(NodeError): - Node('', NodeKind.FILE) + Node(b'', NodeKind.FILE) def test_kind_setter(self): - node = Node('', NodeKind.DIR) + node = Node(b'', NodeKind.DIR) with pytest.raises(NodeError): node.kind = NodeKind.FILE def test_compare_equal(self): - node1 = FileNode('test', content='') - node2 = FileNode('test', content='') + node1 = FileNode(b'test', content=b'') + node2 = FileNode(b'test', content=b'') assert node1 == node2 assert not node1 != node2 def test_compare_unequal(self): - node1 = FileNode('test', content='a') - node2 = FileNode('test', content='b') + node1 = FileNode(b'test', content=b'a') + node2 = FileNode(b'test', content=b'b') assert node1 != node2 assert not node1 == node2 @pytest.mark.parametrize("node_path, expected_parent_path", [ - ('', ''), - ('some/path/', 'some/'), - ('some/longer/path/', 'some/longer/'), + ('', b''), + ('some/path/', b'some/'), + ('some/longer/path/', b'some/longer/'), ]) def test_parent_path_new(self, node_path, expected_parent_path): """ Tests if node's parent path are properly computed. """ + node_path = safe_bytes(node_path) node = Node(node_path, NodeKind.DIR) parent_path = node.get_parent_path() - assert (parent_path.endswith('/') or - node.is_root() and parent_path == '') + assert (parent_path.endswith(b'/') or + node.is_root() and parent_path == b'') assert parent_path == expected_parent_path ''' @@ -134,34 +137,34 @@ class TestNodeBasics: ''' def test_is_file(self): - node = Node('any', NodeKind.FILE) + node = Node(b'any', NodeKind.FILE) assert node.is_file() - node = FileNode('any') + node = FileNode(b'any') assert node.is_file() with pytest.raises(AttributeError): - node.nodes + node.nodes # noqa def test_is_dir(self): - node = Node('any_dir', NodeKind.DIR) + node = Node(b'any_dir', NodeKind.DIR) assert node.is_dir() - node = DirNode('any_dir') + node = DirNode(b'any_dir') assert node.is_dir() with pytest.raises(NodeError): - node.content + node.content # noqa def test_dir_node_iter(self): nodes = [ - DirNode('docs'), - DirNode('tests'), - FileNode('bar'), - FileNode('foo'), - FileNode('readme.txt'), - FileNode('setup.py'), + DirNode(b'docs'), + DirNode(b'tests'), + FileNode(b'bar'), + FileNode(b'foo'), + FileNode(b'readme.txt'), + FileNode(b'setup.py'), ] - dirnode = DirNode('', nodes=nodes) + dirnode = DirNode(b'', nodes=nodes) for node in dirnode: assert node == dirnode.get_node(node.path) @@ -169,15 +172,15 @@ class TestNodeBasics: """ Without link to commit nodes should raise NodeError. """ - node = FileNode('anything') + node = FileNode(b'anything') with pytest.raises(NodeError): - node.state - node = DirNode('anything') + node.state # noqa + node = DirNode(b'anything') with pytest.raises(NodeError): - node.state + node.state # noqa def test_file_node_stat(self): - node = FileNode('foobar', 'empty... almost') + node = FileNode(b'foobar', b'empty... almost') mode = node.mode # default should be 0100644 assert mode & stat.S_IRUSR assert mode & stat.S_IWUSR @@ -190,29 +193,29 @@ class TestNodeBasics: assert not mode & stat.S_IXOTH def test_file_node_is_executable(self): - node = FileNode('foobar', 'empty... almost', mode=0o100755) + node = FileNode(b'foobar', b'empty... almost', mode=0o100755) assert node.is_executable - node = FileNode('foobar', 'empty... almost', mode=0o100500) + node = FileNode(b'foobar', b'empty... almost', mode=0o100500) assert node.is_executable - node = FileNode('foobar', 'empty... almost', mode=0o100644) + node = FileNode(b'foobar', b'empty... almost', mode=0o100644) assert not node.is_executable def test_file_node_is_not_symlink(self): - node = FileNode('foobar', 'empty...') + node = FileNode(b'foobar', b'empty...') assert not node.is_link() def test_mimetype(self): - py_node = FileNode('test.py') - tar_node = FileNode('test.tar.gz') + py_node = FileNode(b'test.py') + tar_node = FileNode(b'test.tar.gz') ext = 'CustomExtension' - my_node2 = FileNode('myfile2') + my_node2 = FileNode(b'myfile2') my_node2._mimetype = [ext] - my_node3 = FileNode('myfile3') + my_node3 = FileNode(b'myfile3') my_node3._mimetype = [ext, ext] assert py_node.mimetype == 'text/x-python' @@ -229,20 +232,20 @@ class TestNodeBasics: def test_lines_counts(self): lines = [ - 'line1\n', - 'line2\n', - 'line3\n', - '\n', - '\n', - 'line4\n', + b'line1\n', + b'line2\n', + b'line3\n', + b'\n', + b'\n', + b'line4\n', ] - py_node = FileNode('test.py', ''.join(lines)) + py_node = FileNode(b'test.py', b''.join(lines)) assert (len(lines), len(lines)) == py_node.lines() assert (len(lines), len(lines) - 2) == py_node.lines(count_empty=True) def test_lines_no_newline(self): - py_node = FileNode('test.py', 'oneline') + py_node = FileNode(b'test.py', b'oneline') assert (1, 1) == py_node.lines() assert (1, 1) == py_node.lines(count_empty=True) @@ -251,15 +254,15 @@ class TestNodeBasics: class TestNodeContent(object): def test_if_binary(self, binary_filenode): - filenode = binary_filenode('calendar.jpg') + filenode = binary_filenode(b'calendar.jpg') assert filenode.is_binary def test_binary_line_counts(self, binary_filenode): - tar_node = binary_filenode('archive.tar.gz') + tar_node = binary_filenode(b'archive.tar.gz') assert (0, 0) == tar_node.lines(count_empty=True) def test_binary_mimetype(self, binary_filenode): - tar_node = binary_filenode('archive.tar.gz') + tar_node = binary_filenode(b'archive.tar.gz') assert tar_node.mimetype == 'application/x-tar' @@ -271,5 +274,5 @@ class TestNodesCommits(BackendTestMixin) last_commit = repo.get_commit() for x in range(3): - node = last_commit.get_node('file_%s.txt' % x) + node = last_commit.get_node(f'file_{x}.txt') assert node.last_commit == repo[x] diff --git a/rhodecode/tests/vcs/test_repository.py b/rhodecode/tests/vcs/test_repository.py --- a/rhodecode/tests/vcs/test_repository.py +++ b/rhodecode/tests/vcs/test_repository.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -56,7 +55,7 @@ class TestRepositoryBase(BackendTestMixi self.Backend(path) def test_has_commits_attribute(self): - self.repo.commit_ids + assert self.repo.commit_ids def test_name(self): assert self.repo.name.startswith('vcs-test') @@ -81,11 +80,20 @@ class TestRepositoryBase(BackendTestMixi def test_bookmarks(self): assert len(self.repo.bookmarks) == 0 - # TODO: Cover two cases: Local repo path, remote URL - def test_check_url(self): + def test_check_url_on_path(self): config = Config() assert self.Backend.check_url(self.repo.path, config) + def test_check_url_on_remote_url(self): + config = Config() + url = { + 'hg': 'https://code.rhodecode.com/rhodecode-vcsserver', + 'svn': 'https://code.rhodecode.com/svn-doc', + 'git': 'https://code.rhodecode.com/appenlight', + }[self.repo.alias] + + assert self.Backend.check_url(url, config) + def test_check_url_invalid(self): config = Config() with pytest.raises(URLError): @@ -189,7 +197,7 @@ class TestRepositoryCompare: source_repo = vcsbackend.clone_repo(target_repo) assert target_repo != source_repo - vcsbackend.add_file(source_repo, 'newfile', 'somecontent') + vcsbackend.add_file(source_repo, b'newfile', b'somecontent') source_commit = source_repo.get_commit() target_repo.compare( @@ -232,7 +240,7 @@ class TestRepositoryGetCommonAncestor: source_repo = vcsbackend.clone_repo(target_repo) assert target_repo != source_repo - vcsbackend.add_file(source_repo, 'newfile', 'somecontent') + vcsbackend.add_file(source_repo, b'newfile', b'somecontent') source_commit = source_repo.get_commit() expected_ancestor = target_repo[4].raw_id @@ -273,10 +281,10 @@ class TestRepositoryMerge(object): def prepare_for_success(self, vcsbackend): self.target_repo = vcsbackend.create_repo(number_of_commits=1) self.source_repo = vcsbackend.clone_repo(self.target_repo) - vcsbackend.add_file(self.target_repo, 'README_MERGE1', 'Version 1') - vcsbackend.add_file(self.source_repo, 'README_MERGE2', 'Version 2') + vcsbackend.add_file(self.target_repo, b'README_MERGE1', b'Version 1') + vcsbackend.add_file(self.source_repo, b'README_MERGE2', b'Version 2') imc = self.source_repo.in_memory_commit - imc.add(FileNode('file_x', content=self.source_repo.name)) + imc.add(FileNode(b'file_x', content=self.source_repo.name)) imc.commit( message=u'Automatic commit from repo merge test', author=u'Automatic ') @@ -292,8 +300,8 @@ class TestRepositoryMerge(object): def prepare_for_conflict(self, vcsbackend): self.target_repo = vcsbackend.create_repo(number_of_commits=1) self.source_repo = vcsbackend.clone_repo(self.target_repo) - vcsbackend.add_file(self.target_repo, 'README_MERGE', 'Version 1') - vcsbackend.add_file(self.source_repo, 'README_MERGE', 'Version 2') + vcsbackend.add_file(self.target_repo, b'README_MERGE', b'Version 1') + vcsbackend.add_file(self.source_repo, b'README_MERGE', b'Version 2') self.target_commit = self.target_repo.get_commit() self.source_commit = self.source_repo.get_commit() # This only works for Git and Mercurial @@ -363,10 +371,10 @@ class TestRepositoryMerge(object): # Multiple merges may differ in their commit id. Therefore we set the # commit id to `None` before comparing the merge responses. - new_merge_ref = merge_response.merge_ref._replace(commit_id=None) + new_merge_ref = merge_response.merge_ref.commit_id = None merge_response.merge_ref = new_merge_ref - new_update_merge_ref = merge_response_update.merge_ref._replace(commit_id=None) + new_update_merge_ref = merge_response_update.merge_ref.commit_id = None merge_response_update.merge_ref = new_update_merge_ref assert merge_response == merge_response_update @@ -480,8 +488,8 @@ class TestRepositoryStrip(BackendTestMix 'date': datetime.datetime(2010, 1, 1, 20), 'branch': 'master', 'added': [ - FileNode('foobar', content='foobar'), - FileNode('foobar2', content='foobar2'), + FileNode(b'foobar', content='foobar'), + FileNode(b'foobar2', content='foobar2'), ], }, ] @@ -492,7 +500,7 @@ class TestRepositoryStrip(BackendTestMix 'date': datetime.datetime(2010, 1, 1, 21, x), 'branch': 'master', 'changed': [ - FileNode('foobar', 'FOOBAR - %s' % x), + FileNode(b'foobar', 'FOOBAR - %s' % x), ], } commits.append(commit_data) diff --git a/rhodecode/tests/vcs/test_svn.py b/rhodecode/tests/vcs/test_svn.py --- a/rhodecode/tests/vcs/test_svn.py +++ b/rhodecode/tests/vcs/test_svn.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -23,6 +22,7 @@ import os import mock import pytest +from rhodecode.lib.str_utils import safe_bytes from rhodecode.tests import SVN_REPO, TEST_DIR, TESTS_TMP_PATH from rhodecode.lib.vcs.backends.svn.repository import SubversionRepository from rhodecode.lib.vcs.conf import settings @@ -83,8 +83,8 @@ def test_commit_author(head): @pytest.mark.parametrize("filename, content, mime_type", [ - ('test.txt', 'Text content\n', None), - ('test.bin', '\0 binary \0', 'application/octet-stream'), + (b'test.txt', b'Text content\n', None), + (b'test.bin', b'\0 binary \0', 'application/octet-stream'), ], ids=['text', 'binary']) def test_sets_mime_type_correctly(vcsbackend, filename, content, mime_type): repo = vcsbackend.create_repo() @@ -132,12 +132,14 @@ def test_topnode_files_attribute(head): topnode.files + + @pytest.mark.parametrize("filename, content, branch, mime_type", [ - (u'branches/plain/test.txt', 'Text content\n', 'plain', None), - (u'branches/uniçö∂e/test.bin', '\0 binary \0', u'uniçö∂e', - 'application/octet-stream'), + ('branches/plain/test.txt', b'Text content\n', 'plain', None), + ('branches/uniçö∂e/test.bin', b'\0 binary \0', 'uniçö∂e', 'application/octet-stream'), ], ids=['text', 'binary']) def test_unicode_refs(vcsbackend, filename, content, branch, mime_type): + filename = safe_bytes(filename) repo = vcsbackend.create_repo() vcsbackend.ensure_file(filename, content) with mock.patch(("rhodecode.lib.vcs.backends.svn.repository" @@ -184,3 +186,10 @@ class TestSVNCommit(object): node_ids = [commit.raw_id for commit in node.history] assert ['18', '8'] == node_ids + + def test_repo_files_content_type(self): + test_commit = self.repo.get_commit(commit_idx=100) + for node in test_commit.get_node('/'): + if node.is_file(): + assert type(node.content) == bytes + assert type(node.str_content) == str diff --git a/rhodecode/tests/vcs/test_tags.py b/rhodecode/tests/vcs/test_tags.py --- a/rhodecode/tests/vcs/test_tags.py +++ b/rhodecode/tests/vcs/test_tags.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/vcs/test_utils.py b/rhodecode/tests/vcs/test_utils.py --- a/rhodecode/tests/vcs/test_utils.py +++ b/rhodecode/tests/vcs/test_utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -24,6 +23,7 @@ import subprocess import pytest +from rhodecode.lib.vcs.backends.git import GitRepository from rhodecode.lib.vcs.exceptions import VCSError from rhodecode.lib.vcs.utils import author_email, author_name from rhodecode.lib.vcs.utils.helpers import get_scm @@ -85,9 +85,10 @@ class TestGetScm(object): def test_get_two_scms_for_path(self, tmpdir): multialias_repo_path = str(tmpdir) + git_default_branch = GitRepository.DEFAULT_BRANCH_NAME subprocess.check_call(['hg', 'init', multialias_repo_path]) - subprocess.check_call(['git', 'init', multialias_repo_path]) + subprocess.check_call(['git', '-c', f'init.defaultBranch={git_default_branch}', 'init', multialias_repo_path]) with pytest.raises(VCSError): get_scm(multialias_repo_path) diff --git a/rhodecode/tests/vcs/test_vcs.py b/rhodecode/tests/vcs/test_vcs.py --- a/rhodecode/tests/vcs/test_vcs.py +++ b/rhodecode/tests/vcs/test_vcs.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # diff --git a/rhodecode/tests/vcs/utils.py b/rhodecode/tests/vcs/utils.py --- a/rhodecode/tests/vcs/utils.py +++ b/rhodecode/tests/vcs/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -67,7 +66,7 @@ class SCMFetcher(object): self.alias = alias self.test_repo_path = test_repo_path - def setup(self): + def setup_method(self): if not os.path.isdir(self.test_repo_path): self.fetch_repo() @@ -100,7 +99,7 @@ def get_normalized_path(path): m = matcher.match(name) if not m: # Haven't append number yet so return first - newname = '%s-00000' % name + newname = f'{name}-00000' newpath = os.path.join(dir, newname) if ext: newpath = '.'.join((newpath, ext)) diff --git a/rhodecode/tests/vcs_operations/__init__.py b/rhodecode/tests/vcs_operations/__init__.py --- a/rhodecode/tests/vcs_operations/__init__.py +++ b/rhodecode/tests/vcs_operations/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -95,33 +94,31 @@ def _add_files(vcs, dest, clone_url=None author_str = 'Marcin Kuźminski ' for i in range(kwargs.get('files_no', 3)): - cmd = """echo 'added_line%s' >> %s""" % (i, added_file) + cmd = f"""echo 'added_line{i}' >> {added_file}""" Command(cwd).execute(cmd) if vcs == 'hg': - cmd = """hg commit -m 'committed new %s' -u '%s' %s """ % ( - i, author_str, added_file - ) + cmd = f"""hg commit -m 'committed new {i}' -u '{author_str}' {added_file} """ elif vcs == 'git': - cmd = """%s && git commit -m 'committed new %s' %s""" % ( - git_ident, i, added_file) + cmd = f"""{git_ident} && git commit -m 'committed new {i}' {added_file}""" Command(cwd).execute(cmd) for tag in tags: if vcs == 'hg': Command(cwd).execute( - 'hg tag -m "{}" -u "{}" '.format(tag['commit'], author_str), tag['name']) + f"""hg tag -m "{tag['commit']}" -u "{author_str}" """, + tag['name']) elif vcs == 'git': if tag['commit']: # annotated tag _stdout, _stderr = Command(cwd).execute( - """%s && git tag -a %s -m "%s" """ % ( - git_ident, tag['name'], tag['commit'])) + f"""{git_ident} && git tag -a {tag['name']} -m "{tag['commit']}" """ + ) else: # lightweight tag _stdout, _stderr = Command(cwd).execute( - """%s && git tag %s""" % ( - git_ident, tag['name'])) + f"""{git_ident} && git tag {tag['name']}""" + ) def _add_files_and_push(vcs, dest, clone_url=None, tags=None, target_branch=None, @@ -130,9 +127,8 @@ def _add_files_and_push(vcs, dest, clone Generate some files, add it to DEST repo and push back vcs is git or hg and defines what VCS we want to make those files for """ - git_ident = "git config user.name {} && git config user.email {}".format( - 'Marcin Kuźminski', 'me@email.com') - cwd = path = jn(dest) + git_ident = "git config user.name Marcin Kuźminski && git config user.email me@email.com" + cwd = jn(dest) # commit some stuff into this repo _add_files(vcs, dest, clone_url, tags, target_branch, new_branch, **kwargs) @@ -151,7 +147,7 @@ def _add_files_and_push(vcs, dest, clone if new_branch: maybe_new_branch = '--new-branch' stdout, stderr = Command(cwd).execute( - 'hg push --verbose {} -r {} {}'.format(maybe_new_branch, target_branch, clone_url) + 'hg push --traceback --verbose {} -r {} {}'.format(maybe_new_branch, target_branch, clone_url) ) elif vcs == 'git': stdout, stderr = Command(cwd).execute( diff --git a/rhodecode/tests/vcs_operations/conftest.py b/rhodecode/tests/vcs_operations/conftest.py --- a/rhodecode/tests/vcs_operations/conftest.py +++ b/rhodecode/tests/vcs_operations/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -32,6 +31,7 @@ import tempfile import textwrap import pytest import logging +import requests from rhodecode import events from rhodecode.lib.str_utils import safe_bytes @@ -40,19 +40,33 @@ from rhodecode.model.db import Integrati from rhodecode.model.integration import IntegrationModel from rhodecode.model.db import Repository from rhodecode.model.meta import Session -from rhodecode.model.settings import SettingsModel from rhodecode.integrations.types.webhook import WebhookIntegrationType from rhodecode.tests import GIT_REPO, HG_REPO from rhodecode.tests.fixture import Fixture from rhodecode.tests.server_utils import RcWebServer + REPO_GROUP = 'a_repo_group' -HG_REPO_WITH_GROUP = '%s/%s' % (REPO_GROUP, HG_REPO) -GIT_REPO_WITH_GROUP = '%s/%s' % (REPO_GROUP, GIT_REPO) +HG_REPO_WITH_GROUP = f'{REPO_GROUP}/{HG_REPO}' +GIT_REPO_WITH_GROUP = f'{REPO_GROUP}/{GIT_REPO}' log = logging.getLogger(__name__) +# Docker image running httpbin... +HTTPBIN_DOMAIN = 'http://httpbin' +HTTPBIN_POST = HTTPBIN_DOMAIN + '/post' + + +def check_httpbin_connection(): + try: + response = requests.get(HTTPBIN_DOMAIN) + return response.status_code == 200 + except Exception as e: + print(e) + + return False + @pytest.fixture(scope="module") def rcextensions(request, db_connection, tmpdir_factory): @@ -74,10 +88,9 @@ def rcextensions(request, db_connection, pytest.fail( "Path for rcextensions already exists, please clean up before " "test run this path: %s" % (rcextensions_path, )) - return - - request.addfinalizer(rcextensions_path.remove) - init_path.write_binary(safe_bytes(init_content), ensure=True) + else: + request.addfinalizer(rcextensions_path.remove) + init_path.write_binary(safe_bytes(init_content), ensure=True) @pytest.fixture(scope="module") @@ -127,7 +140,7 @@ def rc_web_server( request, vcsserver_factory, available_port_factory, rc_web_server_config_factory, repos, rcextensions): """ - Run the web server as a subprocess. with it's own instance of vcsserver + Run the web server as a subprocess. with its own instance of vcsserver """ rcweb_port = available_port_factory() log.info('Using rcweb ops test port {}'.format(rcweb_port)) @@ -175,55 +188,6 @@ def disable_locking(baseapp): @pytest.fixture() -def enable_auth_plugins(request, baseapp, csrf_token): - """ - Return a factory object that when called, allows to control which - authentication plugins are enabled. - """ - def _enable_plugins(plugins_list, override=None): - override = override or {} - params = { - 'auth_plugins': ','.join(plugins_list), - } - - # helper translate some names to others - name_map = { - 'token': 'authtoken' - } - - for module in plugins_list: - plugin_name = module.partition('#')[-1] - if plugin_name in name_map: - plugin_name = name_map[plugin_name] - enabled_plugin = 'auth_%s_enabled' % plugin_name - cache_ttl = 'auth_%s_cache_ttl' % plugin_name - - # default params that are needed for each plugin, - # `enabled` and `cache_ttl` - params.update({ - enabled_plugin: True, - cache_ttl: 0 - }) - if override.get: - params.update(override.get(module, {})) - - validated_params = params - for k, v in validated_params.items(): - setting = SettingsModel().create_or_update_setting(k, v) - Session().add(setting) - Session().commit() - - SettingsModel().invalidate_settings_cache() - - def cleanup(): - _enable_plugins(['egg:rhodecode-enterprise-ce#rhodecode']) - - request.addfinalizer(cleanup) - - return _enable_plugins - - -@pytest.fixture() def fs_repo_only(request, rhodecode_fixtures): def fs_repo_fabric(repo_name, repo_type): rhodecode_fixtures.create_repo(repo_name, repo_type=repo_type) @@ -245,7 +209,7 @@ def enable_webhook_push_integration(requ Session().add(integration) settings = dict( - url='http://httpbin.org/post', + url=HTTPBIN_POST, secret_token='secret', username=None, password=None, diff --git a/rhodecode/tests/vcs_operations/test_vcs_calls_custom_auth_code_403.py b/rhodecode/tests/vcs_operations/test_vcs_calls_custom_auth_code_403.py --- a/rhodecode/tests/vcs_operations/test_vcs_calls_custom_auth_code_403.py +++ b/rhodecode/tests/vcs_operations/test_vcs_calls_custom_auth_code_403.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -37,7 +36,7 @@ from rhodecode.tests.vcs_operations impo def rc_web_server_config_modification(): return [ {'app:main': {'auth_ret_code': '403'}}, - {'app:main': {'auth_ret_code_detection': 'true'}}, + #{'app:main': {'auth_ret_code_detection': 'true'}}, ] diff --git a/rhodecode/tests/vcs_operations/test_vcs_calls_custom_auth_code_404.py b/rhodecode/tests/vcs_operations/test_vcs_calls_custom_auth_code_404.py --- a/rhodecode/tests/vcs_operations/test_vcs_calls_custom_auth_code_404.py +++ b/rhodecode/tests/vcs_operations/test_vcs_calls_custom_auth_code_404.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -37,7 +36,7 @@ from rhodecode.tests.vcs_operations impo def rc_web_server_config_modification(): return [ {'app:main': {'auth_ret_code': '404'}}, - {'app:main': {'auth_ret_code_detection': 'false'}}, + #{'app:main': {'auth_ret_code_detection': 'false'}}, ] diff --git a/rhodecode/tests/vcs_operations/test_vcs_calls_custom_auth_code_bad_code.py b/rhodecode/tests/vcs_operations/test_vcs_calls_custom_auth_code_bad_code.py --- a/rhodecode/tests/vcs_operations/test_vcs_calls_custom_auth_code_bad_code.py +++ b/rhodecode/tests/vcs_operations/test_vcs_calls_custom_auth_code_bad_code.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -37,7 +36,7 @@ from rhodecode.tests.vcs_operations impo def rc_web_server_config_modification(): return [ {'app:main': {'auth_ret_code': '600'}}, - {'app:main': {'auth_ret_code_detection': 'false'}}, + #{'app:main': {'auth_ret_code_detection': 'false'}}, ] @@ -48,10 +47,10 @@ class TestVCSOperationsOnCustomIniConfig clone_url = rc_web_server.repo_clone_url(HG_REPO, passwd='bad!') stdout, stderr = Command('/tmp').execute( 'hg clone', clone_url, tmpdir.strpath) - assert 'abort: HTTP Error 403: Forbidden' in stderr + assert 'abort: authorization failed' in stderr def test_clone_wrong_credentials_git_ret_code(self, rc_web_server, tmpdir): clone_url = rc_web_server.repo_clone_url(GIT_REPO, passwd='bad!') stdout, stderr = Command('/tmp').execute( 'git clone', clone_url, tmpdir.strpath) - assert 'The requested URL returned error: 403' in stderr + assert 'fatal: Authentication failed' in stderr diff --git a/rhodecode/tests/vcs_operations/test_vcs_calls_small_post_buffer.py b/rhodecode/tests/vcs_operations/test_vcs_calls_small_post_buffer.py --- a/rhodecode/tests/vcs_operations/test_vcs_calls_small_post_buffer.py +++ b/rhodecode/tests/vcs_operations/test_vcs_calls_small_post_buffer.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -28,7 +27,6 @@ Test suite for making push/pull operatio """ import os -import pytest from rhodecode.lib.vcs.backends.git.repository import GitRepository from rhodecode.lib.vcs.nodes import FileNode @@ -55,7 +53,7 @@ def test_git_push_with_small_push_buffer cmd.execute('git clone', clone_url) repo = GitRepository(os.path.join(tmpdir.strpath, empty_repo.repo_name)) - repo.in_memory_commit.add(FileNode('readme.md', content='## Hello')) + repo.in_memory_commit.add(FileNode(b'readme.md', content=b'## Hello')) repo.in_memory_commit.commit( message='Commit on branch Master', author='Automatic test ', @@ -63,5 +61,5 @@ def test_git_push_with_small_push_buffer repo_cmd = Command(repo.path) stdout, stderr = repo_cmd.execute( - 'git -c http.postBuffer=1024 push --verbose origin master') + f'git -c http.postBuffer=1024 push --verbose {clone_url} master') _check_proper_git_push(stdout, stderr, branch='master') diff --git a/rhodecode/tests/vcs_operations/test_vcs_operations.py b/rhodecode/tests/vcs_operations/test_vcs_operations.py --- a/rhodecode/tests/vcs_operations/test_vcs_operations.py +++ b/rhodecode/tests/vcs_operations/test_vcs_operations.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -29,6 +28,7 @@ Test suite for making push/pull operatio import time +import logging import pytest @@ -39,6 +39,7 @@ from rhodecode.model.meta import Session from rhodecode.model.repo import RepoModel from rhodecode.model.user import UserModel from rhodecode.tests import (GIT_REPO, HG_REPO, TEST_USER_ADMIN_LOGIN) +from rhodecode.tests.utils import assert_message_in_log from rhodecode.tests.vcs_operations import ( Command, _check_proper_clone, _check_proper_git_push, @@ -372,112 +373,3 @@ class TestVCSOperations(object): stdout, stderr = cmd.execute('git clone', clone_url, tmpdir.strpath) cmd.assert_returncode_success() _check_proper_clone(stdout, stderr, 'git') - - def test_clone_by_auth_token( - self, rc_web_server, tmpdir, user_util, enable_auth_plugins): - enable_auth_plugins(['egg:rhodecode-enterprise-ce#token', - 'egg:rhodecode-enterprise-ce#rhodecode']) - - user = user_util.create_user() - token = user.auth_tokens[1] - - clone_url = rc_web_server.repo_clone_url( - HG_REPO, user=user.username, passwd=token) - - stdout, stderr = Command('/tmp').execute( - 'hg clone', clone_url, tmpdir.strpath) - _check_proper_clone(stdout, stderr, 'hg') - - def test_clone_by_auth_token_expired( - self, rc_web_server, tmpdir, user_util, enable_auth_plugins): - enable_auth_plugins(['egg:rhodecode-enterprise-ce#token', - 'egg:rhodecode-enterprise-ce#rhodecode']) - - user = user_util.create_user() - auth_token = AuthTokenModel().create( - user.user_id, u'test-token', -10, AuthTokenModel.cls.ROLE_VCS) - token = auth_token.api_key - - clone_url = rc_web_server.repo_clone_url( - HG_REPO, user=user.username, passwd=token) - - stdout, stderr = Command('/tmp').execute( - 'hg clone', clone_url, tmpdir.strpath) - assert 'abort: authorization failed' in stderr - - def test_clone_by_auth_token_bad_role( - self, rc_web_server, tmpdir, user_util, enable_auth_plugins): - enable_auth_plugins(['egg:rhodecode-enterprise-ce#token', - 'egg:rhodecode-enterprise-ce#rhodecode']) - - user = user_util.create_user() - auth_token = AuthTokenModel().create( - user.user_id, u'test-token', -1, AuthTokenModel.cls.ROLE_API) - token = auth_token.api_key - - clone_url = rc_web_server.repo_clone_url( - HG_REPO, user=user.username, passwd=token) - - stdout, stderr = Command('/tmp').execute( - 'hg clone', clone_url, tmpdir.strpath) - assert 'abort: authorization failed' in stderr - - def test_clone_by_auth_token_user_disabled( - self, rc_web_server, tmpdir, user_util, enable_auth_plugins): - enable_auth_plugins(['egg:rhodecode-enterprise-ce#token', - 'egg:rhodecode-enterprise-ce#rhodecode']) - user = user_util.create_user() - user.active = False - Session().add(user) - Session().commit() - token = user.auth_tokens[1] - - clone_url = rc_web_server.repo_clone_url( - HG_REPO, user=user.username, passwd=token) - - stdout, stderr = Command('/tmp').execute( - 'hg clone', clone_url, tmpdir.strpath) - assert 'abort: authorization failed' in stderr - - def test_clone_by_auth_token_with_scope( - self, rc_web_server, tmpdir, user_util, enable_auth_plugins): - enable_auth_plugins(['egg:rhodecode-enterprise-ce#token', - 'egg:rhodecode-enterprise-ce#rhodecode']) - user = user_util.create_user() - auth_token = AuthTokenModel().create( - user.user_id, u'test-token', -1, AuthTokenModel.cls.ROLE_VCS) - token = auth_token.api_key - - # manually set scope - auth_token.repo = Repository.get_by_repo_name(HG_REPO) - Session().add(auth_token) - Session().commit() - - clone_url = rc_web_server.repo_clone_url( - HG_REPO, user=user.username, passwd=token) - - stdout, stderr = Command('/tmp').execute( - 'hg clone', clone_url, tmpdir.strpath) - _check_proper_clone(stdout, stderr, 'hg') - - def test_clone_by_auth_token_with_wrong_scope( - self, rc_web_server, tmpdir, user_util, enable_auth_plugins): - enable_auth_plugins(['egg:rhodecode-enterprise-ce#token', - 'egg:rhodecode-enterprise-ce#rhodecode']) - user = user_util.create_user() - auth_token = AuthTokenModel().create( - user.user_id, u'test-token', -1, AuthTokenModel.cls.ROLE_VCS) - token = auth_token.api_key - - # manually set scope - auth_token.repo = Repository.get_by_repo_name(GIT_REPO) - Session().add(auth_token) - Session().commit() - - clone_url = rc_web_server.repo_clone_url( - HG_REPO, user=user.username, passwd=token) - - stdout, stderr = Command('/tmp').execute( - 'hg clone', clone_url, tmpdir.strpath) - assert 'abort: authorization failed' in stderr - diff --git a/rhodecode/tests/vcs_operations/test_vcs_operations_branch_protection.py b/rhodecode/tests/vcs_operations/test_vcs_operations_branch_protection.py --- a/rhodecode/tests/vcs_operations/test_vcs_operations_branch_protection.py +++ b/rhodecode/tests/vcs_operations/test_vcs_operations_branch_protection.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -58,7 +57,7 @@ class TestVCSOperations(object): if branch_perm in ['branch.push', 'branch.push_force']: _check_proper_hg_push(stdout, stderr) else: - msg = "Branch `default` changes rejected by rule `*`=>{}".format(branch_perm) + msg = f"Branch `default` changes rejected by rule `*`=>{branch_perm}" assert msg in stdout assert "transaction abort" in stdout @@ -89,6 +88,6 @@ class TestVCSOperations(object): if branch_perm in ['branch.push', 'branch.push_force']: _check_proper_git_push(stdout, stderr) else: - msg = "Branch `master` changes rejected by rule `*`=>{}".format(branch_perm) + msg = f"Branch `master` changes rejected by rule `*`=>{branch_perm}" assert msg in stderr assert "(pre-receive hook declined)" in stderr diff --git a/rhodecode/tests/vcs_operations/test_vcs_operations_by_auth_tokens.py b/rhodecode/tests/vcs_operations/test_vcs_operations_by_auth_tokens.py new file mode 100644 --- /dev/null +++ b/rhodecode/tests/vcs_operations/test_vcs_operations_by_auth_tokens.py @@ -0,0 +1,174 @@ + +# Copyright (C) 2010-2020 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +""" +Test suite for making push/pull operations, on specially modified INI files + +.. important:: + + You must have git >= 1.8.5 for tests to work fine. With 68b939b git started + to redirect things to stderr instead of stdout. +""" + +import pytest + +from rhodecode.model.auth_token import AuthTokenModel +from rhodecode.model.db import Repository +from rhodecode.model.meta import Session +from rhodecode.tests import (GIT_REPO, HG_REPO) + +from rhodecode.tests.vcs_operations import (Command, _check_proper_clone) + + +@pytest.mark.usefixtures("disable_locking", "disable_anonymous_user") +class TestVCSOperations(object): + def test_clone_by_auth_token( + self, rc_web_server, tmpdir, user_util, enable_auth_plugins): + + enable_auth_plugins.enable([ + 'egg:rhodecode-enterprise-ce#token', + 'egg:rhodecode-enterprise-ce#rhodecode' + ]) + + user = user_util.create_user() + token = user.auth_tokens[1] + + clone_url = rc_web_server.repo_clone_url( + HG_REPO, user=user.username, passwd=token) + + stdout, stderr = Command('/tmp').execute( + 'hg clone', clone_url, tmpdir.strpath) + + _check_proper_clone(stdout, stderr, 'hg') + + def test_clone_by_auth_token_expired( + self, rc_web_server, tmpdir, user_util, enable_auth_plugins): + enable_auth_plugins.enable([ + 'egg:rhodecode-enterprise-ce#token', + 'egg:rhodecode-enterprise-ce#rhodecode' + ]) + + user = user_util.create_user() + auth_token = AuthTokenModel().create( + user.user_id, 'test-token', -10, AuthTokenModel.cls.ROLE_VCS) + token = auth_token.api_key + + clone_url = rc_web_server.repo_clone_url( + HG_REPO, user=user.username, passwd=token) + + stdout, stderr = Command('/tmp').execute( + 'hg clone', clone_url, tmpdir.strpath) + assert 'abort: authorization failed' in stderr + + msg = 'reason: bad or inactive token.' + rc_web_server.assert_message_in_server_logs(msg) + + def test_clone_by_auth_token_bad_role( + self, rc_web_server, tmpdir, user_util, enable_auth_plugins): + enable_auth_plugins.enable([ + 'egg:rhodecode-enterprise-ce#token', + 'egg:rhodecode-enterprise-ce#rhodecode' + ]) + + user = user_util.create_user() + auth_token = AuthTokenModel().create( + user.user_id, 'test-token', -1, AuthTokenModel.cls.ROLE_API) + token = auth_token.api_key + + clone_url = rc_web_server.repo_clone_url( + HG_REPO, user=user.username, passwd=token) + + stdout, stderr = Command('/tmp').execute( + 'hg clone', clone_url, tmpdir.strpath) + assert 'abort: authorization failed' in stderr + + def test_clone_by_auth_token_user_disabled( + self, rc_web_server, tmpdir, user_util, enable_auth_plugins): + enable_auth_plugins.enable([ + 'egg:rhodecode-enterprise-ce#token', + 'egg:rhodecode-enterprise-ce#rhodecode' + ]) + + user = user_util.create_user() + user.active = False + Session().add(user) + Session().commit() + token = user.auth_tokens[1] + + clone_url = rc_web_server.repo_clone_url( + HG_REPO, user=user.username, passwd=token) + + stdout, stderr = Command('/tmp').execute( + 'hg clone', clone_url, tmpdir.strpath) + assert 'abort: authorization failed' in stderr + + msg = 'reason: account not active.' + rc_web_server.assert_message_in_server_logs(msg) + + def test_clone_by_auth_token_with_scope( + self, rc_web_server, tmpdir, user_util, enable_auth_plugins): + enable_auth_plugins.enable([ + 'egg:rhodecode-enterprise-ce#token', + 'egg:rhodecode-enterprise-ce#rhodecode' + ]) + + user = user_util.create_user() + auth_token = AuthTokenModel().create( + user.user_id, 'test-token', -1, AuthTokenModel.cls.ROLE_VCS) + token = auth_token.api_key + + # manually set scope + auth_token.repo = Repository.get_by_repo_name(HG_REPO) + Session().add(auth_token) + Session().commit() + + clone_url = rc_web_server.repo_clone_url( + HG_REPO, user=user.username, passwd=token) + + stdout, stderr = Command('/tmp').execute( + 'hg clone', clone_url, tmpdir.strpath) + _check_proper_clone(stdout, stderr, 'hg') + + def test_clone_by_auth_token_with_wrong_scope( + self, rc_web_server, tmpdir, user_util, enable_auth_plugins): + enable_auth_plugins.enable([ + 'egg:rhodecode-enterprise-ce#token', + 'egg:rhodecode-enterprise-ce#rhodecode' + ]) + + user = user_util.create_user() + auth_token = AuthTokenModel().create( + user.user_id, 'test-token', -1, AuthTokenModel.cls.ROLE_VCS) + token = auth_token.api_key + + # manually set scope + auth_token.repo = Repository.get_by_repo_name(GIT_REPO) + Session().add(auth_token) + Session().commit() + + clone_url = rc_web_server.repo_clone_url( + HG_REPO, user=user.username, passwd=token) + + stdout, stderr = Command('/tmp').execute( + 'hg clone', clone_url, tmpdir.strpath) + + assert 'abort: authorization failed' in stderr + + msg = 'reason: bad or inactive token.' + rc_web_server.assert_message_in_server_logs(msg) diff --git a/rhodecode/tests/vcs_operations/test_vcs_operations_force_push.py b/rhodecode/tests/vcs_operations/test_vcs_operations_force_push.py --- a/rhodecode/tests/vcs_operations/test_vcs_operations_force_push.py +++ b/rhodecode/tests/vcs_operations/test_vcs_operations_force_push.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -47,7 +46,7 @@ class TestVCSOperations(object): _add_files('hg', tmpdir.strpath, clone_url=clone_url) stdout, stderr = Command(tmpdir.strpath).execute( - 'hg push --verbose -f {}'.format(clone_url)) + f'hg push --verbose -f {clone_url}') _check_proper_hg_push(stdout, stderr) @@ -65,7 +64,7 @@ class TestVCSOperations(object): Command(tmpdir.strpath).execute( 'git reset --hard HEAD~2') stdout, stderr = Command(tmpdir.strpath).execute( - 'git push -f {} master'.format(clone_url)) + f'git push -f {clone_url} master') assert '(forced update)' in stderr @@ -90,7 +89,7 @@ class TestVCSOperations(object): _add_files('hg', tmpdir.strpath, clone_url=clone_url) stdout, stderr = Command(tmpdir.strpath).execute( - 'hg push --verbose -f {}'.format(clone_url)) + f'hg push --verbose -f {clone_url}') assert "Branch `default` changes rejected by rule `*`=>branch.push" in stdout assert "FORCE PUSH FORBIDDEN" in stdout @@ -120,3 +119,4 @@ class TestVCSOperations(object): assert "Branch `master` changes rejected by rule `*`=>branch.push" in stderr assert "FORCE PUSH FORBIDDEN" in stderr assert "(pre-receive hook declined)" in stderr + diff --git a/rhodecode/tests/vcs_operations/test_vcs_operations_integrations_trigger.py b/rhodecode/tests/vcs_operations/test_vcs_operations_integrations_trigger.py --- a/rhodecode/tests/vcs_operations/test_vcs_operations_integrations_trigger.py +++ b/rhodecode/tests/vcs_operations/test_vcs_operations_integrations_trigger.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2010-2020 RhodeCode GmbH # @@ -28,24 +27,15 @@ Test suite for making push/pull operatio """ import pytest -import requests + from rhodecode.tests import GIT_REPO, HG_REPO from rhodecode.tests.vcs_operations import Command, _add_files_and_push - - -def check_connection(): - try: - response = requests.get('http://httpbin.org') - return response.status_code == 200 - except Exception as e: - print(e) - - return False +from rhodecode.tests.vcs_operations.conftest import check_httpbin_connection connection_available = pytest.mark.skipif( - not check_connection(), reason="No outside internet connection available") + not check_httpbin_connection(), reason="No outside internet connection available") @pytest.mark.usefixtures("baseapp", "enable_webhook_push_integration") @@ -54,8 +44,7 @@ class TestVCSOperationsOnCustomIniConfig def test_push_with_webhook_hg(self, rc_web_server, tmpdir): clone_url = rc_web_server.repo_clone_url(HG_REPO) - stdout, stderr = Command('/tmp').execute( - 'hg clone', clone_url, tmpdir.strpath) + Command('/tmp').execute('hg clone', clone_url, tmpdir.strpath) push_url = rc_web_server.repo_clone_url(HG_REPO) _add_files_and_push('hg', tmpdir.strpath, clone_url=push_url) @@ -65,11 +54,10 @@ class TestVCSOperationsOnCustomIniConfig assert "executing task TASK:<@task: rhodecode.integrations.types.webhook.post_to_webhook" in rc_log assert "handling event repo-push with integration