diff --git a/vcsserver/git_lfs/__init__.py b/vcsserver/git_lfs/__init__.py --- a/vcsserver/git_lfs/__init__.py +++ b/vcsserver/git_lfs/__init__.py @@ -16,4 +16,4 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from .app import create_app +from .app import create_app # noqa diff --git a/vcsserver/git_lfs/app.py b/vcsserver/git_lfs/app.py --- a/vcsserver/git_lfs/app.py +++ b/vcsserver/git_lfs/app.py @@ -17,7 +17,6 @@ import re import logging -from wsgiref.util import FileWrapper from pyramid.config import Configurator from pyramid.response import Response, FileIter @@ -33,7 +32,7 @@ from vcsserver.str_utils import safe_int log = logging.getLogger(__name__) -GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ? +GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' # +json ? GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))') diff --git a/vcsserver/hooks.py b/vcsserver/hooks.py --- a/vcsserver/hooks.py +++ b/vcsserver/hooks.py @@ -35,7 +35,7 @@ import mercurial.node from vcsserver.lib.rc_json import json from vcsserver import exceptions, subprocessio, settings from vcsserver.str_utils import ascii_str, safe_str -from vcsserver.remote.git import Repository +from vcsserver.remote.git_remote import Repository log = logging.getLogger(__name__) diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -40,20 +40,7 @@ from vcsserver.lib.rc_json import json from vcsserver.config.settings_maker import SettingsMaker from vcsserver.str_utils import safe_int from vcsserver.lib.statsd_client import StatsdClient -from vcsserver.tweens.request_wrapper import get_call_context, get_headers_call_context - -log = logging.getLogger(__name__) - -# due to Mercurial/glibc2.27 problems we need to detect if locale settings are -# causing problems and "fix" it in case they do and fallback to LC_ALL = C - -try: - locale.setlocale(locale.LC_ALL, '') -except locale.Error as e: - log.error( - 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e) - os.environ['LC_ALL'] = 'C' - +from vcsserver.tweens.request_wrapper import get_headers_call_context import vcsserver from vcsserver import remote_wsgi, scm_app, settings, hgpatches @@ -68,7 +55,7 @@ strict_vcs = True git_import_err = None try: - from vcsserver.remote.git import GitFactory, GitRemote + from vcsserver.remote.git_remote import GitFactory, GitRemote except ImportError as e: GitFactory = None GitRemote = None @@ -79,7 +66,7 @@ except ImportError as e: hg_import_err = None try: - from vcsserver.remote.hg import MercurialFactory, HgRemote + from vcsserver.remote.hg_remote import MercurialFactory, HgRemote except ImportError as e: MercurialFactory = None HgRemote = None @@ -90,7 +77,7 @@ except ImportError as e: svn_import_err = None try: - from vcsserver.remote.svn import SubversionFactory, SvnRemote + from vcsserver.remote.svn_remote import SubversionFactory, SvnRemote except ImportError as e: SubversionFactory = None SvnRemote = None @@ -98,6 +85,18 @@ except ImportError as e: if strict_vcs: raise +log = logging.getLogger(__name__) + +# due to Mercurial/glibc2.27 problems we need to detect if locale settings are +# causing problems and "fix" it in case they do and fallback to LC_ALL = C + +try: + locale.setlocale(locale.LC_ALL, '') +except locale.Error as e: + log.error( + 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e) + os.environ['LC_ALL'] = 'C' + def _is_request_chunked(environ): stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked' @@ -444,10 +443,7 @@ class HTTPApplication(object): method = method.split('stream:')[-1] chunk_size = safe_int(payload.get('chunk_size')) or 4096 - try: - resp = getattr(remote, method)(*args, **kwargs) - except Exception as e: - raise + resp = getattr(remote, method)(*args, **kwargs) def get_chunked_data(method_resp): stream = io.BytesIO(method_resp) @@ -694,7 +690,7 @@ class ResponseFilter(object): def sanitize_settings_and_apply_defaults(global_config, settings): - global_settings_maker = SettingsMaker(global_config) + _global_settings_maker = SettingsMaker(global_config) settings_maker = SettingsMaker(settings) settings_maker.make_setting('logging.autoconfigure', False, parser='bool') @@ -770,5 +766,3 @@ def main(global_config, **settings): total_time = time.time() - start_time log.info('Pyramid app created and configured in %.2fs', total_time) return pyramid_app - - diff --git a/vcsserver/remote/git.py b/vcsserver/remote/git_remote.py rename from vcsserver/remote/git.py rename to vcsserver/remote/git_remote.py diff --git a/vcsserver/remote/hg.py b/vcsserver/remote/hg_remote.py rename from vcsserver/remote/hg.py rename to vcsserver/remote/hg_remote.py diff --git a/vcsserver/remote/svn.py b/vcsserver/remote/svn_remote.py rename from vcsserver/remote/svn.py rename to vcsserver/remote/svn_remote.py --- a/vcsserver/remote/svn.py +++ b/vcsserver/remote/svn_remote.py @@ -37,12 +37,20 @@ import svn.fs # noqa import svn.repos # noqa from vcsserver import svn_diff, exceptions, subprocessio, settings -from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, store_archive_in_cache, BytesEnvelope, BinaryEnvelope +from vcsserver.base import ( + RepoFactory, + raise_from_original, + ArchiveNode, + store_archive_in_cache, + BytesEnvelope, + BinaryEnvelope, +) from vcsserver.exceptions import NoContentException from vcsserver.str_utils import safe_str, safe_bytes from vcsserver.type_utils import assert_bytes from vcsserver.vcs_base import RemoteBase from vcsserver.lib.svnremoterepo import svnremoterepo + log = logging.getLogger(__name__) diff --git a/vcsserver/tests/test_git.py b/vcsserver/tests/test_git.py --- a/vcsserver/tests/test_git.py +++ b/vcsserver/tests/test_git.py @@ -21,7 +21,7 @@ import pytest import dulwich.errors from mock import Mock, patch -from vcsserver.remote import git +from vcsserver.remote import git_remote SAMPLE_REFS = { 'HEAD': 'fd627b9e0dd80b47be81af07c4a98518244ed2f7', @@ -33,17 +33,17 @@ SAMPLE_REFS = { @pytest.fixture -def git_remote(): +def git_remote_fix(): """ A GitRemote instance with a mock factory. """ factory = Mock() - remote = git.GitRemote(factory) + remote = git_remote.GitRemote(factory) return remote -def test_discover_git_version(git_remote): - version = git_remote.discover_git_version() +def test_discover_git_version(git_remote_fix): + version = git_remote_fix.discover_git_version() assert version @@ -52,7 +52,7 @@ class TestGitFetch(object): self.mock_repo = Mock() factory = Mock() factory.repo = Mock(return_value=self.mock_repo) - self.remote_git = git.GitRemote(factory) + self.remote_git = git_remote.GitRemote(factory) def test_fetches_all_when_no_commit_ids_specified(self): def side_effect(determine_wants, *args, **kwargs): @@ -85,14 +85,14 @@ class TestGitFetch(object): def test_get_remote_refs(self): factory = Mock() - remote_git = git.GitRemote(factory) - url = 'http://example.com/test/test.git' + remote_git = git_remote.GitRemote(factory) + url = 'https://example.com/test/test.git' sample_refs = { 'refs/tags/v0.1.8': '74ebce002c088b8a5ecf40073db09375515ecd68', 'refs/tags/v0.1.3': '5a3a8fb005554692b16e21dee62bf02667d8dc3e', } - with patch('vcsserver.remote.git.Repo', create=False) as mock_repo: + with patch('vcsserver.remote.git_remote.Repo', create=False) as mock_repo: mock_repo().get_refs.return_value = sample_refs remote_refs = remote_git.get_remote_refs(wire={}, url=url) mock_repo().get_refs.assert_called_once_with() @@ -103,14 +103,14 @@ class TestReraiseSafeExceptions(object): def test_method_decorated_with_reraise_safe_exceptions(self): factory = Mock() - git_remote = git.GitRemote(factory) + git_remote_instance = git_remote.GitRemote(factory) def fake_function(): return None - decorator = git.reraise_safe_exceptions(fake_function) + decorator = git_remote.reraise_safe_exceptions(fake_function) - methods = inspect.getmembers(git_remote, predicate=inspect.ismethod) + methods = inspect.getmembers(git_remote_instance, predicate=inspect.ismethod) for method_name, method in methods: if not method_name.startswith('_') and method_name not in ['vcsserver_invalidate_cache']: assert method.__func__.__code__ == decorator.__code__ @@ -124,7 +124,7 @@ class TestReraiseSafeExceptions(object): (dulwich.errors.UnexpectedCommandError('test-cmd'), 'error'), ]) def test_safe_exceptions_reraised(self, side_effect, expected_type): - @git.reraise_safe_exceptions + @git_remote.reraise_safe_exceptions def fake_method(): raise side_effect @@ -137,9 +137,9 @@ class TestReraiseSafeExceptions(object): class TestDulwichRepoWrapper(object): def test_calls_close_on_delete(self): isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True) - with patch.object(git.Repo, 'close') as close_mock: + with patch.object(git_remote.Repo, 'close') as close_mock: with isdir_patcher: - repo = git.Repo('/tmp/abcde') + repo = git_remote.Repo('/tmp/abcde') assert repo is not None repo.__del__() # can't use del repo as in python3 this isn't always calling .__del__() @@ -152,11 +152,11 @@ class TestGitFactory(object): with patch('vcsserver.lib.rc_cache.region_meta.dogpile_cache_regions') as mock: mock.side_effect = {'repo_objects': ''} - factory = git.GitFactory() + factory = git_remote.GitFactory() wire = { 'path': '/tmp/abcde' } isdir_patcher = patch('dulwich.repo.os.path.isdir', return_value=True) with isdir_patcher: result = factory._create_repo(wire, True) - assert isinstance(result, git.Repo) + assert isinstance(result, git_remote.Repo) diff --git a/vcsserver/tests/test_hg.py b/vcsserver/tests/test_hg.py --- a/vcsserver/tests/test_hg.py +++ b/vcsserver/tests/test_hg.py @@ -24,19 +24,19 @@ from mercurial.error import LookupError from mock import Mock, patch from vcsserver import exceptions, hgcompat -from vcsserver.remote import hg +from vcsserver.remote import hg_remote class TestDiff(object): def test_raising_safe_exception_when_lookup_failed(self): factory = Mock() - hg_remote = hg.HgRemote(factory) + hg_remote_instance = hg_remote.HgRemote(factory) with patch('mercurial.patch.diff') as diff_mock: diff_mock.side_effect = LookupError(b'deadbeef', b'index', b'message') with pytest.raises(Exception) as exc_info: - hg_remote.diff( + hg_remote_instance.diff( wire={}, commit_id_1='deadbeef', commit_id_2='deadbee1', file_filter=None, opt_git=True, opt_ignorews=True, context=3) @@ -45,25 +45,27 @@ class TestDiff(object): class TestReraiseSafeExceptions(object): + original_traceback = None + def test_method_decorated_with_reraise_safe_exceptions(self): factory = Mock() - hg_remote = hg.HgRemote(factory) - methods = inspect.getmembers(hg_remote, predicate=inspect.ismethod) - decorator = hg.reraise_safe_exceptions(None) + hg_remote_instance = hg_remote.HgRemote(factory) + methods = inspect.getmembers(hg_remote_instance, predicate=inspect.ismethod) + decorator = hg_remote.reraise_safe_exceptions(None) for method_name, method in methods: if not method_name.startswith('_') and method_name not in ['vcsserver_invalidate_cache']: assert method.__func__.__code__ == decorator.__code__ @pytest.mark.parametrize('side_effect, expected_type', [ - (hgcompat.Abort('failed-abort'), 'abort'), - (hgcompat.InterventionRequired('intervention-required'), 'abort'), + (hgcompat.Abort(b'failed-abort'), 'abort'), + (hgcompat.InterventionRequired(b'intervention-required'), 'abort'), (hgcompat.RepoLookupError(), 'lookup'), (hgcompat.LookupError(b'deadbeef', b'index', b'message'), 'lookup'), (hgcompat.RepoError(), 'error'), (hgcompat.RequirementError(), 'requirement'), ]) def test_safe_exceptions_reraised(self, side_effect, expected_type): - @hg.reraise_safe_exceptions + @hg_remote.reraise_safe_exceptions def fake_method(): raise side_effect @@ -73,14 +75,16 @@ class TestReraiseSafeExceptions(object): assert exc_info.value._vcs_kind == expected_type def test_keeps_original_traceback(self): - @hg.reraise_safe_exceptions + + @hg_remote.reraise_safe_exceptions def fake_method(): try: - raise hgcompat.Abort('test-abort') + raise hgcompat.Abort(b'test-abort') except: self.original_traceback = traceback.format_tb(sys.exc_info()[2]) raise + new_traceback = None try: fake_method() except Exception: @@ -89,8 +93,8 @@ class TestReraiseSafeExceptions(object): new_traceback_tail = new_traceback[-len(self.original_traceback):] assert new_traceback_tail == self.original_traceback - def test_maps_unknow_exceptions_to_unhandled(self): - @hg.reraise_safe_exceptions + def test_maps_unknown_exceptions_to_unhandled(self): + @hg_remote.reraise_safe_exceptions def stub_method(): raise ValueError('stub') @@ -99,7 +103,7 @@ class TestReraiseSafeExceptions(object): assert exc_info.value._vcs_kind == 'unhandled' def test_does_not_map_known_exceptions(self): - @hg.reraise_safe_exceptions + @hg_remote.reraise_safe_exceptions def stub_method(): raise exceptions.LookupException()('stub') diff --git a/vcsserver/tests/test_svn.py b/vcsserver/tests/test_svn.py --- a/vcsserver/tests/test_svn.py +++ b/vcsserver/tests/test_svn.py @@ -46,11 +46,11 @@ INVALID_CERTIFICATE_STDERR = '\n'.join([ @pytest.mark.xfail(sys.platform == "cygwin", reason="SVN not packaged for Cygwin") def test_import_remote_repository_certificate_error(stderr, expected_reason): - from vcsserver.remote import svn + from vcsserver.remote import svn_remote factory = mock.Mock() factory.repo = mock.Mock(return_value=mock.Mock()) - remote = svn.SvnRemote(factory) + remote = svn_remote.SvnRemote(factory) remote.is_path_valid_repository = lambda wire, path: True with mock.patch('subprocess.Popen', @@ -64,7 +64,7 @@ def test_import_remote_repository_certif def test_svn_libraries_can_be_imported(): - import svn.client + import svn.client # noqa assert svn.client is not None @@ -76,12 +76,12 @@ def test_svn_libraries_can_be_imported() ('http://', ('', '', 'http://')), ]) def test_username_password_extraction_from_url(example_url, parts): - from vcsserver.remote import svn + from vcsserver.remote import svn_remote factory = mock.Mock() factory.repo = mock.Mock(return_value=mock.Mock()) - remote = svn.SvnRemote(factory) + remote = svn_remote.SvnRemote(factory) remote.is_path_valid_repository = lambda wire, path: True assert remote.get_url_and_credentials(example_url) == parts @@ -93,11 +93,11 @@ def test_username_password_extraction_fr b'https://marcink:qweqwe@svn.code.sf.net/p/svnbook/source/trunk/', ]) def test_check_url(call_url): - from vcsserver.remote import svn + from vcsserver.remote import svn_remote factory = mock.Mock() factory.repo = mock.Mock(return_value=mock.Mock()) - remote = svn.SvnRemote(factory) + remote = svn_remote.SvnRemote(factory) remote.is_path_valid_repository = lambda wire, path: True assert remote.check_url(call_url, {'dummy': 'config'}) diff --git a/vcsserver/tweens/request_wrapper.py b/vcsserver/tweens/request_wrapper.py --- a/vcsserver/tweens/request_wrapper.py +++ b/vcsserver/tweens/request_wrapper.py @@ -15,13 +15,13 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import base64 +import logging import time -import logging import msgpack import vcsserver -from vcsserver.str_utils import safe_str, ascii_str +from vcsserver.str_utils import safe_str log = logging.getLogger(__name__)