fix(encryption): don't be strict on enc format when no enc headers are missing.

This fixes problem with migration of OLD unencrypted DBs into a encryption format

rhodecode/lib/encrypt.py

@@ -1,154 +1,155 b''
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Generic encryption library for RhodeCode
 """
 
 import base64
 import logging
 
 from Crypto.Cipher import AES
 from Crypto import Random
 from Crypto.Hash import HMAC, SHA256
 
 from rhodecode.lib.str_utils import safe_bytes, safe_str
 from rhodecode.lib.exceptions import signature_verification_error
 
 
 class InvalidDecryptedValue(str):
 
     def __new__(cls, content):
         """
         This will generate something like this::
             <InvalidDecryptedValue(QkWusFgLJXR6m42v...)>
         And represent a safe indicator that encryption key is broken
         """
         content = f'<{cls.__name__}({content[:16]}...)>'
         return str.__new__(cls, content)
 
+
 KEY_FORMAT = b'enc$aes_hmac${1}'
 
 
 class AESCipher(object):
 
     def __init__(self, key: bytes, hmac=False, strict_verification=True):
 
         if not key:
             raise ValueError('passed key variable is empty')
         self.strict_verification = strict_verification
         self.block_size = 32
         self.hmac_size = 32
         self.hmac = hmac
 
         self.key = SHA256.new(safe_bytes(key)).digest()
         self.hmac_key = SHA256.new(self.key).digest()
 
     def verify_hmac_signature(self, raw_data):
         org_hmac_signature = raw_data[-self.hmac_size:]
         data_without_sig = raw_data[:-self.hmac_size]
         recomputed_hmac = HMAC.new(
             self.hmac_key, data_without_sig, digestmod=SHA256).digest()
         return org_hmac_signature == recomputed_hmac
 
     def encrypt(self, raw: bytes):
         raw = self._pad(raw)
         iv = Random.new().read(AES.block_size)
         cipher = AES.new(self.key, AES.MODE_CBC, iv)
         enc_value = cipher.encrypt(raw)
 
         hmac_signature = b''
         if self.hmac:
             # compute hmac+sha256 on iv + enc text, we use
             # encrypt then mac method to create the signature
             hmac_signature = HMAC.new(
                 self.hmac_key, iv + enc_value, digestmod=SHA256).digest()
 
         return base64.b64encode(iv + enc_value + hmac_signature)
 
     def decrypt(self, enc, safe=True) -> bytes | InvalidDecryptedValue:
         enc_org = enc
         try:
             enc = base64.b64decode(enc)
         except Exception:
             logging.exception('Failed Base64 decode')
             raise signature_verification_error('Failed Base64 decode')
 
         if self.hmac and len(enc) > self.hmac_size:
             if self.verify_hmac_signature(enc):
                 # cut off the HMAC verification digest
                 enc = enc[:-self.hmac_size]
             else:
 
                 decrypt_fail = InvalidDecryptedValue(safe_str(enc_org))
                 if safe:
                     return decrypt_fail
                 raise signature_verification_error(decrypt_fail)
 
         iv = enc[:AES.block_size]
         cipher = AES.new(self.key, AES.MODE_CBC, iv)
         return self._unpad(cipher.decrypt(enc[AES.block_size:]))
 
     def _pad(self, s):
         block_pad = (self.block_size - len(s) % self.block_size)
         return s + block_pad * safe_bytes(chr(block_pad))
 
     @staticmethod
     def _unpad(s):
         return s[:-ord(s[len(s)-1:])]
 
 
 def validate_and_decrypt_data(enc_data, enc_key, enc_strict_mode=False, safe=True):
     enc_data = safe_str(enc_data)
 
     if '$' not in enc_data:
         # probably not encrypted values
         return enc_data
 
     parts = enc_data.split('$', 3)
     if len(parts) != 3:
         raise ValueError(f'Encrypted Data has invalid format, expected {KEY_FORMAT}, got {parts}, org value: {enc_data}')
 
     enc_type = parts[1]
     enc_data_part = parts[2]
 
     if parts[0] != 'enc':
         # parts ok but without our header?
         return enc_data
 
     # at that stage we know it's our encryption
     if enc_type == 'aes':
         decrypted_data = AESCipher(enc_key).decrypt(enc_data_part, safe=safe)
     elif enc_type == 'aes_hmac':
         decrypted_data = AESCipher(
             enc_key, hmac=True,
             strict_verification=enc_strict_mode).decrypt(enc_data_part, safe=safe)
 
     else:
         raise ValueError(
             f'Encryption type part is wrong, must be `aes` '
             f'or `aes_hmac`, got `{enc_type}` instead')
 
     return decrypted_data
 
 
 def encrypt_data(data, enc_key: bytes):
     enc_key = safe_bytes(enc_key)
     enc_value = AESCipher(enc_key, hmac=True).encrypt(safe_bytes(data))
     return KEY_FORMAT.replace(b'{1}', enc_value)

rhodecode/lib/encrypt2.py

@@ -1,97 +1,101 b''
 import os
 import base64
 from cryptography.fernet import Fernet, InvalidToken
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
 
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.exceptions import signature_verification_error
 
 
 class InvalidDecryptedValue(str):
 
     def __new__(cls, content):
         """
         This will generate something like this::
             <InvalidDecryptedValue(QkWusFgLJXR6m42v...)>
         And represent a safe indicator that encryption key is broken
         """
         content = f'<{cls.__name__}({content[:16]}...)>'
         return str.__new__(cls, content)
 
 
 class Encryptor(object):
     key_format = b'enc2$salt:{1}$data:{2}'
 
     pref_len = 5  # salt:, data:
 
     @classmethod
     def detect_enc_algo(cls, enc_data: bytes):
         parts = enc_data.split(b'$', 3)
-        if len(parts) != 3:
-            raise ValueError(f'Encrypted Data has invalid format, expected {cls.key_format}, got {parts}')
 
         if b'enc$aes_hmac$' in enc_data:
+            # we expect this data is encrypted, so validate the header
+            if len(parts) != 3:
+                raise ValueError(f'Encrypted Data has invalid format, expected {cls.key_format}, got `{parts}`')
             return 'aes'
         elif b'enc2$salt' in enc_data:
+            # we expect this data is encrypted, so validate the header
+            if len(parts) != 3:
+                raise ValueError(f'Encrypted Data has invalid format, expected {cls.key_format}, got `{parts}`')
             return 'fernet'
         return None
 
     def __init__(self, enc_key: bytes):
         self.enc_key = enc_key
 
     def b64_encode(self, data):
         return base64.urlsafe_b64encode(data)
 
     def b64_decode(self, data):
         return base64.urlsafe_b64decode(data)
 
     def get_encryptor(self, salt):
         """
         Uses Fernet as encryptor with HMAC signature
         :param salt: random salt used for encrypting the data
         """
         kdf = PBKDF2HMAC(
             algorithm=hashes.SHA512(),
             length=32,
             salt=salt,
             iterations=100000,
             backend=default_backend()
         )
         key = self.b64_encode(kdf.derive(self.enc_key))
         return Fernet(key)
 
     def _get_parts(self, enc_data):
         parts = enc_data.split(b'$', 3)
         if len(parts) != 3:
-            raise ValueError(f'Encrypted Data has invalid format, expected {self.key_format}, got {parts}')
+            raise ValueError(f'Encrypted Data has invalid format, expected {self.key_format}, got `{parts}`')
         prefix, salt, enc_data = parts
 
         try:
             salt = self.b64_decode(salt[self.pref_len:])
         except TypeError:
             # bad base64
             raise ValueError('Encrypted Data salt invalid format, expected base64 format')
 
         enc_data = enc_data[self.pref_len:]
         return prefix, salt, enc_data
 
     def encrypt(self, data) -> bytes:
         salt = os.urandom(64)
         encryptor = self.get_encryptor(salt)
         enc_data = encryptor.encrypt(data)
         return self.key_format.replace(b'{1}', self.b64_encode(salt)).replace(b'{2}', enc_data)
 
     def decrypt(self, data, safe=True) -> bytes | InvalidDecryptedValue:
         parts = self._get_parts(data)
         salt = parts[1]
         enc_data = parts[2]
         encryptor = self.get_encryptor(salt)
         try:
             return encryptor.decrypt(enc_data)
         except (InvalidToken,):
             decrypt_fail = InvalidDecryptedValue(safe_str(data))
             if safe:
                 return decrypt_fail
             raise signature_verification_error(decrypt_fail)