dbmigrate: fixed migration problems

rhodecode/lib/dbmigrate/schema/db_3_7_0_0.py

@@ -41,6 +41,7 b' from beaker.cache import cache_region, r'
 from webob.exc import HTTPNotFound
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
+from rhodecode.lib import enc_utils
 from rhodecode.translation import _
 
 from rhodecode.lib.vcs import get_backend
@@ -69,7 +70,7 b' log = logging.getLogger(__name__)'
 
 # this is propagated from .ini file beaker.session.secret
 # and initialized at environment.py
-ENCRYPTION_KEY = None
+ENCRYPTION_KEY: bytes = b''
 
 # used to sort permissions by types, '#' used here is not allowed to be in
 # usernames, and it's very early in sorted string.printable table.
@@ -106,30 +107,38 b' class EncryptedValue(TypeDecorator):'
     impl = String
 
     def process_bind_param(self, value, dialect):
-        if not value:
-            return value
-        if value.startswith('enc$aes$'):
-            # protect against double encrypting if someone manually starts doing
-            raise ValueError('value needs to be in unencrypted format, ie. '
-                             'not starting with enc$aes$')
-        return 'enc$aes$%s' % AESCipher(ENCRYPTION_KEY).encrypt(value)
-
-    def process_result_value(self, value, dialect):
+        """
+        Setter for storing value
+        """
+        import rhodecode
         if not value:
             return value
 
-        parts = value.split('$', 3)
-        if not len(parts) == 3:
-            # probably not encrypted values
-            return value
-        else:
-            if parts[0] != 'enc':
-                # parts ok but without our header ?
+        # protect against double encrypting if values is already encrypted
+        if value.startswith('enc$aes$') \
+                or value.startswith('enc$aes_hmac$') \
+                or value.startswith('enc2$'):
+            raise ValueError('value needs to be in unencrypted format, '
+                             'ie. not starting with enc$ or enc2$')
+
+        algo = rhodecode.CONFIG.get('rhodecode.encrypted_values.algorithm') or 'aes'
+        bytes_val = enc_utils.encrypt_value(value, enc_key=ENCRYPTION_KEY, algo=algo)
+        return safe_str(bytes_val)
+
+    def process_result_value(self, value, dialect):
+        """
+        Getter for retrieving value
+        """
+
+        import rhodecode
+        if not value:
                 return value
 
-            # at that stage we know it's our encryption
-            decrypted_data = AESCipher(ENCRYPTION_KEY).decrypt(parts[2])
-            return decrypted_data
+        enc_strict_mode = rhodecode.ConfigGet().get_bool('rhodecode.encrypted_values.strict', missing=True)
+
+        bytes_val = enc_utils.decrypt_value(value, enc_key=ENCRYPTION_KEY, strict_mode=enc_strict_mode)
+
+        return safe_str(bytes_val)
 
 
 class BaseModel(object):

rhodecode/lib/dbmigrate/versions/008_version_1_5_0.py

@@ -35,7 +35,7 b' def upgrade(migrate_engine):'
     username.create(table=tbl)
 
     _Session = meta.Session()
-    ## after adding that column fix all usernames
+    # after adding that column fix all usernames
     users_log = _Session.query(db_1_5_0.UserLog)\
             .options(joinedload(db_1_5_0.UserLog.user))\
             .options(joinedload(db_1_5_0.UserLog.repository)).all()
@@ -74,7 +74,7 b' def get_by_name(cls, key):'
 
 def fixups(models, _SESSION):
     # ** create default permissions ** #
-    #=====================================
+
     for p in models.Permission.PERMS:
         if not get_by_key(models.Permission, p[0]):
             new_perm = models.Permission()
@@ -86,7 +86,6 b' def fixups(models, _SESSION):'
     _SESSION().commit()
 
     # ** populate default permissions ** #
-    #=====================================
 
     user = models.User.query().filter(models.User.username == 'default').scalar()
 
@@ -100,9 +99,10 b' def fixups(models, _SESSION):'
         return '.'.join(perm_name.split('.')[:1])
 
     perms = models.UserToPerm.query().filter(models.UserToPerm.user == user).all()
-    defined_perms_groups = map(
-        _get_group, (x.permission.permission_name for x in perms))
-    log.debug('GOT ALREADY DEFINED:%s', perms)
+    defined_perms_groups = list(map(
+        _get_group, (x.permission.permission_name for x in perms)))
+    perms_show = [p.__dict__ for p in perms]
+    log.debug('GOT ALREADY DEFINED:%s', perms_show)
     DEFAULT_PERMS = models.Permission.DEFAULT_USER_PERMISSIONS
 
     # for every default permission that needs to be created, we check if

rhodecode/lib/dbmigrate/versions/012_version_1_7_0.py

@@ -70,11 +70,15 b' def downgrade(migrate_engine):'
     meta.bind = migrate_engine
 
 
+def get_by_key(cls, key):
+    return cls.query().filter(cls.permission_name == key).scalar()
+
+
 def fixups(models, _SESSION):
     # ** create default permissions ** #
     #=====================================
     for p in models.Permission.PERMS:
-        if not models.Permission.get_by_key(p[0]):
+        if not get_by_key(models.Permission, p[0]):
             new_perm = models.Permission()
             new_perm.permission_name = p[0]
             new_perm.permission_longname = p[0]  #translation err with p[1]
@@ -97,8 +101,7 b' def fixups(models, _SESSION):'
         return '.'.join(perm_name.split('.')[:1])
 
     perms = models.UserToPerm.query().filter(models.UserToPerm.user == user).all()
-    defined_perms_groups = map(_get_group,
-                              (x.permission.permission_name for x in perms))
+    defined_perms_groups = list(map(_get_group, (x.permission.permission_name for x in perms)))
     log.debug('GOT ALREADY DEFINED:%s', perms)
     DEFAULT_PERMS = models.Permission.DEFAULT_USER_PERMISSIONS
 

rhodecode/lib/dbmigrate/versions/048_version_3_8_0.py

@@ -10,6 +10,7 b' from sqlalchemy.engine import reflection'
 from sqlalchemy.sql import text
 
 from rhodecode.lib.dbmigrate.versions import _reset_base
+from rhodecode.lib.hash_utils import sha1_safe
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.model import meta, init_model_encryption
 
@@ -69,11 +70,9 b' def downgrade(migrate_engine):'
 def _generate_repo_name_hashes(models, op, session):
     repositories = models.Repository.get_all()
     for repository in repositories:
-        hash_ = hashlib.sha1(safe_str(repository.repo_name)).hexdigest()
+        hash_ = sha1_safe(repository.repo_name)
         params = {'hash': hash_, 'id': repository.repo_id}
-        query = text(
-            'UPDATE repositories SET repo_name_hash = :hash'
-            ' WHERE repo_id = :id').bindparams(**params)
+        query = text('UPDATE repositories SET repo_name_hash = :hash WHERE repo_id = :id').bindparams(**params)
         op.execute(query)
     session().commit()
 

rhodecode/lib/encrypt.py

@@ -117,9 +117,13 b' class AESCipher(object):'
 def validate_and_decrypt_data(enc_data, enc_key, enc_strict_mode=False, safe=True):
     enc_data = safe_str(enc_data)
 
+    if '$' not in enc_data:
+        # probably not encrypted values
+        return enc_data
+
     parts = enc_data.split('$', 3)
     if len(parts) != 3:
-        raise ValueError(f'Encrypted Data has invalid format, expected {KEY_FORMAT}, got {parts}')
+        raise ValueError(f'Encrypted Data has invalid format, expected {KEY_FORMAT}, got {parts}, org value: {enc_data}')
 
     enc_type = parts[1]
     enc_data_part = parts[2]