packages: move the str utils to it's own module

vcsserver/str_utils.py

@@ -0,0 +1,127 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2020 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import logging
+
+
+log = logging.getLogger(__name__)
+
+
+def safe_int(val, default=None) -> int:
+    """
+    Returns int() of val if val is not convertable to int use default
+    instead
+
+    :param val:
+    :param default:
+    """
+
+    try:
+        val = int(val)
+    except (ValueError, TypeError):
+        val = default
+
+    return val
+
+
+def safe_str(str_, to_encoding=None) -> str:
+    """
+    safe str function. Does few trick to turn unicode_ into string
+
+    :param str_: str to encode
+    :param to_encoding: encode to this type UTF8 default
+    :rtype: str
+    :returns: str object
+    """
+    if isinstance(str_, str):
+        return str_
+
+    # if it's bytes cast to str
+    if not isinstance(str_, bytes):
+        return str(str_)
+
+    to_encoding = to_encoding or ['utf8']
+    if not isinstance(to_encoding, (list, tuple)):
+        to_encoding = [to_encoding]
+
+    for enc in to_encoding:
+        try:
+            return str(str_, enc)
+        except UnicodeDecodeError:
+            pass
+
+    return str(str_, to_encoding[0], 'replace')
+
+
+def safe_bytes(str_, from_encoding=None) -> bytes:
+    """
+    safe bytes function. Does few trick to turn str_ into bytes string:
+
+    :param str_: string to decode
+    :param from_encoding: encode from this type UTF8 default
+    :rtype: unicode
+    :returns: unicode object
+    """
+    if isinstance(str_, bytes):
+        return str_
+
+    if not isinstance(str_, str):
+        raise ValueError('safe_bytes cannot convert other types than str: got: {}'.format(type(str_)))
+
+    from_encoding = from_encoding or ['utf8']
+    if not isinstance(from_encoding, (list, tuple)):
+        from_encoding = [from_encoding]
+
+    for enc in from_encoding:
+        try:
+            return str_.encode(enc)
+        except UnicodeDecodeError:
+            pass
+
+    return str_.encode(from_encoding[0], 'replace')
+
+
+def ascii_bytes(str_, allow_bytes=False) -> bytes:
+    """
+    Simple conversion from str to bytes, with assumption that str_ is pure ASCII.
+    Fails with UnicodeError on invalid input.
+    This should be used where encoding and "safe" ambiguity should be avoided.
+    Where strings already have been encoded in other ways but still are unicode
+    string - for example to hex, base64, json, urlencoding, or are known to be
+    identifiers.
+    """
+    if allow_bytes and isinstance(str_, bytes):
+        return str_
+
+    if not isinstance(str_, str):
+        raise ValueError('ascii_bytes cannot convert other types than str: got: {}'.format(type(str_)))
+    return str_.encode('ascii')
+
+
+def ascii_str(str_):
+    """
+    Simple conversion from bytes to str, with assumption that str_ is pure ASCII.
+    Fails with UnicodeError on invalid input.
+    This should be used where encoding and "safe" ambiguity should be avoided.
+    Where strings are encoded but also in other ways are known to be ASCII, and
+    where a unicode string is wanted without caring about encoding. For example
+    to hex, base64, urlencoding, or are known to be identifiers.
+    """
+
+    if not isinstance(str_, bytes):
+        raise ValueError('ascii_str cannot convert other types than bytes: got: {}'.format(type(str_)))
+    return str_.decode('ascii')

vcsserver/git_lfs/app.py

@@ -28,7 +28,7 b' from pyramid.httpexceptions import ('
 from vcsserver.lib.rc_json import json
 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
-from vcsserver.utils import safe_int
+from vcsserver.str_utils import safe_int
 
 log = logging.getLogger(__name__)
 

vcsserver/git_lfs/tests/test_lfs_app.py

@@ -20,7 +20,7 b' import pytest'
 from webtest.app import TestApp as WebObTestApp
 
 from vcsserver.lib.rc_json import json
-from vcsserver.utils import safe_bytes
+from vcsserver.str_utils import safe_bytes
 from vcsserver.git_lfs.app import create_app
 
 

vcsserver/git_lfs/tests/test_lib.py

@@ -17,7 +17,7 b''
 
 import os
 import pytest
-from vcsserver.utils import safe_bytes
+from vcsserver.str_utils import safe_bytes
 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
 
 

vcsserver/hgcompat.py

@@ -24,7 +24,7 b' from mercurial import demandimport'
 
 # patch demandimport, due to bug in mercurial when it always triggers
 # demandimport.enable()
-from vcsserver.utils import safe_bytes
+from vcsserver.str_utils import safe_bytes
 
 demandimport.enable = lambda *args, **kwargs: 1
 

vcsserver/hook_utils/__init__.py

@@ -25,7 +25,7 b' import logging'
 import pkg_resources
 
 import vcsserver
-from vcsserver.utils import safe_bytes
+from vcsserver.str_utils import safe_bytes
 
 log = logging.getLogger(__name__)
 

vcsserver/hooks.py

@@ -34,7 +34,7 b' import mercurial.node'
 
 from vcsserver.lib.rc_json import json
 from vcsserver import exceptions, subprocessio, settings
-from vcsserver.utils import safe_bytes
+from vcsserver.str_utils import safe_bytes
 
 log = logging.getLogger(__name__)
 

vcsserver/http_main.py

@@ -39,7 +39,7 b' from pyramid.response import Response'
 
 from vcsserver.lib.rc_json import json
 from vcsserver.config.settings_maker import SettingsMaker
-from vcsserver.utils import safe_int
+from vcsserver.str_utils import safe_int
 from vcsserver.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)

vcsserver/lib/memory_lru_dict.py

@@ -22,7 +22,7 b' import logging'
 
 from repoze.lru import LRUCache
 
-from vcsserver.utils import safe_str
+from vcsserver.str_utils import safe_str
 
 log = logging.getLogger(__name__)
 

vcsserver/lib/rc_cache/backends.py

@@ -33,7 +33,7 b' from dogpile.cache.util import memoized_'
 from pyramid.settings import asbool
 
 from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
-from vcsserver.utils import safe_str
+from vcsserver.str_utils import safe_str
 
 
 _default_max_size = 1024

vcsserver/lib/rc_cache/utils.py

@@ -23,7 +23,8 b' import decorator'
 
 from dogpile.cache import CacheRegion
 
-from vcsserver.utils import safe_bytes, sha1
+from vcsserver.str_utils import safe_bytes
+from vcsserver.utils import sha1
 from vcsserver.lib.rc_cache import region_meta
 
 log = logging.getLogger(__name__)

vcsserver/lib/svnremoterepo.py

@@ -24,7 +24,7 b' from svn import ra'
 
 from mercurial import error
 
-from vcsserver.utils import safe_bytes
+from vcsserver.str_utils import safe_bytes
 
 core.svn_config_ensure(None)
 svn_config = core.svn_config_get_config(None)

vcsserver/pygrack.py

@@ -27,7 +27,7 b' from webob import Request, Response, exc'
 
 from vcsserver.lib.rc_json import json
 from vcsserver import hooks, subprocessio
-from vcsserver.utils import ascii_bytes
+from vcsserver.str_utils import ascii_bytes
 
 
 log = logging.getLogger(__name__)

vcsserver/remote/git.py

@@ -40,7 +40,7 b' from dulwich.repo import Repo as Dulwich'
 from dulwich.server import update_server_info
 
 from vcsserver import exceptions, settings, subprocessio
-from vcsserver.utils import safe_str, safe_int
+from vcsserver.str_utils import safe_str, safe_int
 from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, archive_repo
 from vcsserver.hgcompat import (
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)

vcsserver/remote/hg.py

@@ -39,7 +39,7 b' from vcsserver.hgcompat import ('
     patch, peer, revrange, ui, hg_tag, Abort, LookupError, RepoError,
     RepoLookupError, InterventionRequired, RequirementError,
     alwaysmatcher, patternmatcher, hgutil, hgext_strip)
-from vcsserver.utils import ascii_bytes, ascii_str, safe_str, safe_bytes
+from vcsserver.str_utils import ascii_bytes, ascii_str, safe_str, safe_bytes
 from vcsserver.vcs_base import RemoteBase
 
 log = logging.getLogger(__name__)

vcsserver/remote/svn.py

@@ -38,7 +38,7 b' import svn.repos'
 from vcsserver import svn_diff, exceptions, subprocessio, settings
 from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, archive_repo
 from vcsserver.exceptions import NoContentException
-from vcsserver.utils import safe_str
+from vcsserver.str_utils import safe_str
 from vcsserver.vcs_base import RemoteBase
 from vcsserver.lib.svnremoterepo import svnremoterepo
 log = logging.getLogger(__name__)

vcsserver/scm_app.py

@@ -27,7 +27,7 b' import mercurial.hgweb.hgweb_mod'
 import webob.exc
 
 from vcsserver import pygrack, exceptions, settings, git_lfs
-from vcsserver.utils import ascii_bytes, safe_bytes
+from vcsserver.str_utils import ascii_bytes, safe_bytes
 
 log = logging.getLogger(__name__)
 

vcsserver/tests/test_install_hooks.py

@@ -23,7 +23,8 b' import vcsserver'
 import tempfile
 from vcsserver import hook_utils
 from vcsserver.tests.fixture import no_newline_id_generator
-from vcsserver.utils import AttributeDict, safe_bytes, safe_str
+from vcsserver.str_utils import safe_bytes, safe_str
+from vcsserver.utils import AttributeDict
 
 
 class TestCheckRhodecodeHook(object):

vcsserver/tests/test_pygrack.py

@@ -26,8 +26,7 b' import webtest'
 
 from vcsserver import hooks, pygrack
 
-# pylint: disable=redefined-outer-name,protected-access
-from vcsserver.utils import ascii_bytes
+from vcsserver.str_utils import ascii_bytes
 
 
 @pytest.fixture()

vcsserver/tests/test_scm_app.py

@@ -25,7 +25,7 b' import pytest'
 import webtest
 
 from vcsserver import scm_app
-from vcsserver.utils import ascii_bytes
+from vcsserver.str_utils import ascii_bytes
 
 
 def test_hg_does_not_accept_invalid_cmd(tmpdir):

vcsserver/tests/test_subprocessio.py

@@ -22,7 +22,7 b' import sys'
 import pytest
 
 from vcsserver import subprocessio
-from vcsserver.utils import ascii_bytes
+from vcsserver.str_utils import ascii_bytes
 
 
 class FileLikeObj(object):  # pragma: no cover

vcsserver/tests/test_svn.py

@@ -20,7 +20,7 b' import mock'
 import pytest
 import sys
 
-from vcsserver.utils import ascii_bytes
+from vcsserver.str_utils import ascii_bytes
 
 
 class MockPopen(object):

vcsserver/tests/test_utils.py

@@ -16,7 +16,7 b''
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import pytest
-from vcsserver.utils import ascii_bytes, ascii_str
+from vcsserver.str_utils import ascii_bytes, ascii_str
 
 
 @pytest.mark.parametrize('given, expected', [

vcsserver/tests/test_wsgi_app_caller.py

@@ -19,7 +19,7 b' import wsgiref.simple_server'
 import wsgiref.validate
 
 from vcsserver import wsgi_app_caller
-from vcsserver.utils import ascii_bytes, safe_str
+from vcsserver.str_utils import ascii_bytes, safe_str
 
 
 @wsgiref.validate.validator

vcsserver/tweens/request_wrapper.py

@@ -19,7 +19,7 b' import time'
 import logging
 
 import vcsserver
-from vcsserver.utils import safe_str, ascii_str
+from vcsserver.str_utils import safe_str, ascii_str
 
 log = logging.getLogger(__name__)
 

vcsserver/utils.py

@@ -20,118 +20,35 b' import hashlib'
 log = logging.getLogger(__name__)
 
 
-def safe_int(val, default=None):
-    """
-    Returns int() of val if val is not convertable to int use default
-    instead
-
-    :param val:
-    :param default:
-    """
-
-    try:
-        val = int(val)
-    except (ValueError, TypeError):
-        val = default
-
-    return val
-
-
-def safe_str(str_, to_encoding=None) -> str:
-    """
-    safe str function. Does few trick to turn unicode_ into string
-
-    :param str_: str to encode
-    :param to_encoding: encode to this type UTF8 default
-    :rtype: str
-    :returns: str object
-    """
-    if isinstance(str_, str):
-        return str_
-
-    # if it's bytes cast to str
-    if not isinstance(str_, bytes):
-        return str(str_)
-
-    to_encoding = to_encoding or ['utf8']
-    if not isinstance(to_encoding, (list, tuple)):
-        to_encoding = [to_encoding]
-
-    for enc in to_encoding:
-        try:
-            return str(str_, enc)
-        except UnicodeDecodeError:
-            pass
-
-    return str(str_, to_encoding[0], 'replace')
-
-
-def safe_bytes(str_, from_encoding=None) -> bytes:
-    """
-    safe bytes function. Does few trick to turn str_ into bytes string:
+class AttributeDictBase(dict):
+    def __getstate__(self):
+        odict = self.__dict__  # get attribute dictionary
+        return odict
 
-    :param str_: string to decode
-    :param from_encoding: encode from this type UTF8 default
-    :rtype: unicode
-    :returns: unicode object
-    """
-    if isinstance(str_, bytes):
-        return str_
-
-    if not isinstance(str_, str):
-        raise ValueError('safe_bytes cannot convert other types than str: got: {}'.format(type(str_)))
-
-    from_encoding = from_encoding or ['utf8']
-    if not isinstance(from_encoding, (list, tuple)):
-        from_encoding = [from_encoding]
-
-    for enc in from_encoding:
-        try:
-            return str_.encode(enc)
-        except UnicodeDecodeError:
-            pass
-
-    return str_.encode(from_encoding[0], 'replace')
-
+    def __setstate__(self, dict):
+        self.__dict__ = dict
 
-def ascii_bytes(str_, allow_bytes=False) -> bytes:
-    """
-    Simple conversion from str to bytes, with assumption that str_ is pure ASCII.
-    Fails with UnicodeError on invalid input.
-    This should be used where encoding and "safe" ambiguity should be avoided.
-    Where strings already have been encoded in other ways but still are unicode
-    string - for example to hex, base64, json, urlencoding, or are known to be
-    identifiers.
-    """
-    if allow_bytes and isinstance(str_, bytes):
-        return str_
-
-    if not isinstance(str_, str):
-        raise ValueError('ascii_bytes cannot convert other types than str: got: {}'.format(type(str_)))
-    return str_.encode('ascii')
-
-
-def ascii_str(str_):
-    """
-    Simple conversion from bytes to str, with assumption that str_ is pure ASCII.
-    Fails with UnicodeError on invalid input.
-    This should be used where encoding and "safe" ambiguity should be avoided.
-    Where strings are encoded but also in other ways are known to be ASCII, and
-    where a unicode string is wanted without caring about encoding. For example
-    to hex, base64, urlencoding, or are known to be identifiers.
-    """
-
-    if not isinstance(str_, bytes):
-        raise ValueError('ascii_str cannot convert other types than bytes: got: {}'.format(type(str_)))
-    return str_.decode('ascii')
-
-
-class AttributeDict(dict):
-    def __getattr__(self, attr):
-        return self.get(attr, None)
     __setattr__ = dict.__setitem__
     __delattr__ = dict.__delitem__
 
 
+class StrictAttributeDict(AttributeDictBase):
+    """
+    Strict Version of Attribute dict which raises an Attribute error when
+    requested attribute is not set
+    """
+    def __getattr__(self, attr):
+        try:
+            return self[attr]
+        except KeyError:
+            raise AttributeError('%s object has no attribute %s' % (
+                self.__class__, attr))
+
+
+class AttributeDict(AttributeDictBase):
+    def __getattr__(self, attr):
+        return self.get(attr, None)
+
+
 def sha1(val):
     return hashlib.sha1(val).hexdigest()

vcsserver/wsgi_app_caller.py

@@ -23,7 +23,7 b' import io'
 import logging
 import os
 
-from vcsserver.utils import ascii_bytes
+from vcsserver.str_utils import ascii_bytes
 
 log = logging.getLogger(__name__)