chore(sync-up): synced libs/settings maker from vcsserver

rhodecode/config/settings_maker.py

@@ -1,166 +1,184 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import textwrap
 import string
 import functools
 import logging
 import tempfile
 import logging.config
+
 from rhodecode.lib.type_utils import str2bool, aslist
 
 log = logging.getLogger(__name__)
 
 
 # skip keys, that are set here, so we don't double process those
 set_keys = {
     '__file__': ''
 }
 
 
-class SettingsMaker(object):
+class SettingsMaker:
 
     def __init__(self, app_settings):
         self.settings = app_settings
 
     @classmethod
     def _bool_func(cls, input_val):
+        if isinstance(input_val, bytes):
+            # decode to str
+            input_val = input_val.decode('utf8')
         return str2bool(input_val)
 
     @classmethod
     def _int_func(cls, input_val):
         return int(input_val)
 
     @classmethod
     def _float_func(cls, input_val):
         return float(input_val)
 
     @classmethod
     def _list_func(cls, input_val, sep=','):
         return aslist(input_val, sep=sep)
 
     @classmethod
     def _string_func(cls, input_val, lower=True):
         if lower:
             input_val = input_val.lower()
         return input_val
 
     @classmethod
+    def _string_no_quote_func(cls, input_val, lower=True):
+        """
+        Special case string function that detects if value is set to empty quote string
+        e.g.
+
+        core.binar_dir = ""
+        """
+
+        input_val = cls._string_func(input_val, lower=lower)
+        if input_val in ['""', "''"]:
+            return ''
+
+    @classmethod
     def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
 
         # ensure we have our dir created
         if not os.path.isdir(input_val) and ensure_dir:
-            os.makedirs(input_val, mode=mode)
+            os.makedirs(input_val, mode=mode, exist_ok=True)
 
         if not os.path.isdir(input_val):
             raise Exception(f'Dir at {input_val} does not exist')
         return input_val
 
     @classmethod
     def _file_path_func(cls, input_val, ensure_dir=False, mode=0o755):
         dirname = os.path.dirname(input_val)
         cls._dir_func(dirname, ensure_dir=ensure_dir)
         return input_val
 
     @classmethod
     def _key_transformator(cls, key):
         return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_'))
 
     def maybe_env_key(self, key):
         # now maybe we have this KEY in env, search and use the value with higher priority.
         transformed_key = self._key_transformator(key)
         envvar_value = os.environ.get(transformed_key)
         if envvar_value:
             log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
 
         return envvar_value
 
     def env_expand(self):
         replaced = {}
         for k, v in self.settings.items():
             if k not in set_keys:
                 envvar_value = self.maybe_env_key(k)
                 if envvar_value:
                     replaced[k] = envvar_value
                     set_keys[k] = envvar_value
 
         # replace ALL keys updated
         self.settings.update(replaced)
 
     def enable_logging(self, logging_conf=None, level='INFO', formatter='generic'):
         """
         Helper to enable debug on running instance
         :return:
         """
 
         if not str2bool(self.settings.get('logging.autoconfigure')):
             log.info('logging configuration based on main .ini file')
             return
 
         if logging_conf is None:
             logging_conf = self.settings.get('logging.logging_conf_file') or ''
 
         if not os.path.isfile(logging_conf):
             log.error('Unable to setup logging based on %s, '
                       'file does not exist.... specify path using logging.logging_conf_file= config setting. ', logging_conf)
             return
 
         with open(logging_conf, 'rt') as f:
             ini_template = textwrap.dedent(f.read())
             ini_template = string.Template(ini_template).safe_substitute(
                 RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or level,
                 RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or formatter
             )
 
         with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f:
             log.info('Saved Temporary LOGGING config at %s', f.name)
             f.write(ini_template)
 
         logging.config.fileConfig(f.name)
         os.remove(f.name)
 
     def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None):
         input_val = self.settings.get(key, default)
 
         if default_when_empty and not input_val:
             # use default value when value is set in the config but it is empty
             input_val = default
 
         parser_func = {
             'bool': self._bool_func,
             'int': self._int_func,
             'float': self._float_func,
             'list': self._list_func,
             'list:newline': functools.partial(self._list_func, sep='/n'),
             'list:spacesep': functools.partial(self._list_func, sep=' '),
             'string': functools.partial(self._string_func, lower=lower),
+            'string:noquote': functools.partial(self._string_no_quote_func, lower=lower),
             'dir': self._dir_func,
             'dir:ensured': functools.partial(self._dir_func, ensure_dir=True),
             'file': self._file_path_func,
             'file:ensured': functools.partial(self._file_path_func, ensure_dir=True),
             None: lambda i: i
         }[parser]
 
         envvar_value = self.maybe_env_key(key)
         if envvar_value:
             input_val = envvar_value
             set_keys[key] = input_val
 
         self.settings[key] = parser_func(input_val)
         return self.settings[key]

rhodecode/lib/str_utils.py

@@ -1,169 +1,183 b''
 # Copyright (C) 2011-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import typing
 import base64
 import logging
 from unidecode import unidecode
 
 import rhodecode
 from rhodecode.lib.type_utils import aslist
 
 
 log = logging.getLogger(__name__)
 
 
 def safe_int(val, default=None) -> int:
     """
     Returns int() of val if val is not convertable to int use default
     instead
 
     :param val:
     :param default:
     """
 
     try:
         val = int(val)
     except (ValueError, TypeError):
         val = default
 
     return val
 
 
 def safe_float(val, default=None) -> float:
     """
     Returns float() of val if val is not convertable to float use default
     instead
 
     :param val:
     :param default:
     """
 
     try:
         val = float(val)
     except (ValueError, TypeError):
         val = default
 
     return val
 
 
 def base64_to_str(text: str | bytes) -> str:
     return safe_str(base64.encodebytes(safe_bytes(text))).strip()
 
 
 def get_default_encodings() -> list[str]:
     return aslist(rhodecode.CONFIG.get('default_encoding', 'utf8'), sep=',')
 
 
 DEFAULT_ENCODINGS = get_default_encodings()
 
 
 def safe_str(str_, to_encoding=None) -> str:
     """
     safe str function. Does few trick to turn unicode_ into string
 
     :param str_: str to encode
     :param to_encoding: encode to this type UTF8 default
     """
     if isinstance(str_, str):
         return str_
 
     # if it's bytes cast to str
     if not isinstance(str_, bytes):
         return str(str_)
 
     to_encoding = to_encoding or DEFAULT_ENCODINGS
     if not isinstance(to_encoding, (list, tuple)):
         to_encoding = [to_encoding]
 
     for enc in to_encoding:
         try:
             return str(str_, enc)
         except UnicodeDecodeError:
             pass
 
     return str(str_, to_encoding[0], 'replace')
 
 
 def safe_bytes(str_, from_encoding=None) -> bytes:
     """
     safe bytes function. Does few trick to turn str_ into bytes string:
 
     :param str_: string to decode
     :param from_encoding: encode from this type UTF8 default
     """
     if isinstance(str_, bytes):
         return str_
 
     if not isinstance(str_, str):
         raise ValueError(f'safe_bytes cannot convert other types than str: got: {type(str_)}')
 
     from_encoding = from_encoding or get_default_encodings()
     if not isinstance(from_encoding, (list, tuple)):
         from_encoding = [from_encoding]
 
     for enc in from_encoding:
         try:
             return str_.encode(enc)
         except UnicodeDecodeError:
             pass
 
     return str_.encode(from_encoding[0], 'replace')
 
 
 def ascii_bytes(str_, allow_bytes=False) -> bytes:
     """
     Simple conversion from str to bytes, with assumption that str_ is pure ASCII.
     Fails with UnicodeError on invalid input.
     This should be used where encoding and "safe" ambiguity should be avoided.
     Where strings already have been encoded in other ways but still are unicode
     string - for example to hex, base64, json, urlencoding, or are known to be
     identifiers.
     """
     if allow_bytes and isinstance(str_, bytes):
         return str_
 
     if not isinstance(str_, str):
         raise ValueError(f'ascii_bytes cannot convert other types than str: got: {type(str_)}')
     return str_.encode('ascii')
 
 
 def ascii_str(str_) -> str:
     """
     Simple conversion from bytes to str, with assumption that str_ is pure ASCII.
     Fails with UnicodeError on invalid input.
     This should be used where encoding and "safe" ambiguity should be avoided.
     Where strings are encoded but also in other ways are known to be ASCII, and
     where a unicode string is wanted without caring about encoding. For example
     to hex, base64, urlencoding, or are known to be identifiers.
     """
 
     if not isinstance(str_, bytes):
         raise ValueError(f'ascii_str cannot convert other types than bytes: got: {type(str_)}')
     return str_.decode('ascii')
 
 
 def convert_special_chars(str_) -> str:
     """
         trie to replace non-ascii letters to their ascii representation eg::
 
             `żołw` converts into `zolw`
     """
     value = safe_str(str_)
     converted_value = unidecode(value)
     return converted_value
+
+
+def splitnewlines(text: bytes):
+    """
+    like splitlines, but only split on newlines.
+    """
+
+    lines = [_l + b'\n' for _l in text.split(b'\n')]
+    if lines:
+        if lines[-1] == b'\n':
+            lines.pop()
+        else:
+            lines[-1] = lines[-1][:-1]
+    return lines