diff --git a/rhodecode/model/comment.py b/rhodecode/model/comment.py --- a/rhodecode/model/comment.py +++ b/rhodecode/model/comment.py @@ -319,6 +319,7 @@ class CommentsModel(BaseModel): repo=repo.repo_id, user=user.user_id, )) + is_draft = validated_kwargs['is_draft'] comment = ChangesetComment() diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -25,7 +25,6 @@ import re import os import time import string -import hashlib import logging import datetime import uuid @@ -36,14 +35,15 @@ import traceback import collections from sqlalchemy import ( - or_, and_, not_, func, cast, TypeDecorator, event, + or_, and_, not_, func, cast, TypeDecorator, event, select, + true, false, null, Index, Sequence, UniqueConstraint, ForeignKey, CheckConstraint, Column, Boolean, String, Unicode, UnicodeText, DateTime, Integer, LargeBinary, Text, Float, PickleType, BigInteger) -from sqlalchemy.sql.expression import true, false, case, null +from sqlalchemy.sql.expression import case from sqlalchemy.sql.functions import coalesce, count # pragma: no cover from sqlalchemy.orm import ( - relationship, lazyload, joinedload, class_mapper, validates, aliased) + relationship, lazyload, joinedload, class_mapper, validates, aliased, load_only) from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.exc import IntegrityError # pragma: no cover @@ -61,8 +61,9 @@ from rhodecode.lib.utils2 import ( str2bool, safe_str, get_commit_safe, sha1_safe, time_to_datetime, aslist, Optional, safe_int, get_clone_url, AttributeDict, glob2re, StrictAttributeDict, cleaned_uri, datetime_to_time) -from rhodecode.lib.jsonalchemy import MutationObj, MutationList, JsonType, \ - JsonRaw +from rhodecode.lib.jsonalchemy import ( + MutationObj, MutationList, JsonType, JsonRaw) +from rhodecode.lib.hash_utils import sha1 from rhodecode.lib import ext_json from rhodecode.lib import enc_utils from rhodecode.lib.ext_json import json @@ -81,7 +82,7 @@ log = logging.getLogger(__name__) # this is propagated from .ini file rhodecode.encrypted_values.secret or # beaker.session.secret if first is not set. # and initialized at environment.py -ENCRYPTION_KEY = '' +ENCRYPTION_KEY: bytes = b'' # used to sort permissions by types, '#' used here is not allowed to be in # usernames, and it's very early in sorted string.printable table. @@ -166,6 +167,7 @@ class EncryptedTextValue(TypeDecorator): This column is intelligent so if value is in unencrypted form it return unencrypted form, but on save it always encrypts """ + cache_ok = True impl = Text def process_bind_param(self, value, dialect): @@ -184,7 +186,8 @@ class EncryptedTextValue(TypeDecorator): 'ie. not starting with enc$ or enc2$') algo = rhodecode.CONFIG.get('rhodecode.encrypted_values.algorithm') or 'aes' - return enc_utils.encrypt_value(value, enc_key=ENCRYPTION_KEY, algo=algo) + bytes_val = enc_utils.encrypt_value(value, enc_key=ENCRYPTION_KEY, algo=algo) + return safe_str(bytes_val) def process_result_value(self, value, dialect): """ @@ -195,9 +198,11 @@ class EncryptedTextValue(TypeDecorator): if not value: return value - enc_strict_mode = str2bool(rhodecode.CONFIG.get('rhodecode.encrypted_values.strict') or True) - - return enc_utils.decrypt_value(value, enc_key=ENCRYPTION_KEY, strict_mode=enc_strict_mode) + enc_strict_mode = rhodecode.ConfigGet().get_bool('rhodecode.encrypted_values.strict', missing=True) + + bytes_val = enc_utils.decrypt_value(value, enc_key=ENCRYPTION_KEY, strict_mode=enc_strict_mode) + + return safe_str(bytes_val) class BaseModel(object): @@ -250,6 +255,29 @@ class BaseModel(object): return Session().query(cls) @classmethod + def select(cls, custom_cls=None): + """ + stmt = cls.select().where(cls.user_id==1) + # optionally + stmt = cls.select(User.user_id).where(cls.user_id==1) + result = cls.execute(stmt) | cls.scalars(stmt) + """ + + if custom_cls: + stmt = select(custom_cls) + else: + stmt = select(cls) + return stmt + + @classmethod + def execute(cls, stmt): + return Session().execute(stmt) + + @classmethod + def scalars(cls, stmt): + return Session().scalars(stmt) + + @classmethod def get(cls, id_): if id_: return cls.query().get(id_) @@ -296,14 +324,12 @@ class BaseModel(object): 'value %s found with same name: %r', attr_name, value, exist_in_session) + @property + def cls_name(self): + return self.__class__.__name__ + def __repr__(self): - if hasattr(self, '__str__'): - # python repr needs to return str - try: - return self.__str__() - except UnicodeDecodeError: - pass - return f'' + return f'' class RhodeCodeSetting(Base, BaseModel): @@ -384,9 +410,9 @@ class RhodeCodeSetting(Base, BaseModel): .filter(RhodeCodeSetting.app_settings_name.startswith(prefix))\ .all() - def __str__(self): + def __repr__(self): return "<%s('%s:%s[%s]')>" % ( - self.__class__.__name__, + self.cls_name, self.app_settings_name, self.app_settings_value, self.app_settings_type ) @@ -398,6 +424,7 @@ class RhodeCodeUi(Base, BaseModel): UniqueConstraint('ui_key'), base_table_args ) + # Sync those values with vcsserver.config.hooks HOOK_REPO_SIZE = 'changegroup.repo_size' # HG @@ -436,8 +463,8 @@ class RhodeCodeUi(Base, BaseModel): ui_active = Column( "ui_active", Boolean(), nullable=True, unique=None, default=True) - def __str__(self): - return '<%s[%s]%s=>%s]>' % (self.__class__.__name__, self.ui_section, + def __repr__(self): + return '<%s[%s]%s=>%s]>' % (self.cls_name, self.ui_section, self.ui_key, self.ui_value) @@ -466,7 +493,7 @@ class RepoRhodeCodeSetting(Base, BaseMod "app_settings_type", String(255), nullable=True, unique=None, default=None) - repository = relationship('Repository') + repository = relationship('Repository', viewonly=True) def __init__(self, repository_id, key='', val='', type='unicode'): self.repository_id = repository_id @@ -508,9 +535,9 @@ class RepoRhodeCodeSetting(Base, BaseMod % (SETTINGS_TYPES.keys(), val)) self._app_settings_type = val - def __str__(self): + def __repr__(self): return "<%s('%s:%s:%s[%s]')>" % ( - self.__class__.__name__, self.repository.repo_name, + self.cls_name, self.repository.repo_name, self.app_settings_name, self.app_settings_value, self.app_settings_type ) @@ -540,11 +567,11 @@ class RepoRhodeCodeUi(Base, BaseModel): ui_active = Column( "ui_active", Boolean(), nullable=True, unique=None, default=True) - repository = relationship('Repository') - - def __str__(self): + repository = relationship('Repository', viewonly=True) + + def __repr__(self): return '<%s[%s:%s]%s=>%s]>' % ( - self.__class__.__name__, self.repository.repo_name, + self.cls_name, self.repository.repo_name, self.ui_section, self.ui_key, self.ui_value) @@ -580,55 +607,51 @@ class User(Base, BaseModel): created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) _user_data = Column("user_data", LargeBinary(), nullable=True) # JSON data - user_log = relationship('UserLog') + user_log = relationship('UserLog', back_populates='user') user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all, delete-orphan') - repositories = relationship('Repository') - repository_groups = relationship('RepoGroup') - user_groups = relationship('UserGroup') - - user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all') - followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all') + repositories = relationship('Repository', back_populates='user') + repository_groups = relationship('RepoGroup', back_populates='user') + user_groups = relationship('UserGroup', back_populates='user') + + user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all', back_populates='follows_user') + followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all', back_populates='user') repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all, delete-orphan') - repo_group_to_perm = relationship('UserRepoGroupToPerm', primaryjoin='UserRepoGroupToPerm.user_id==User.user_id', cascade='all, delete-orphan') - user_group_to_perm = relationship('UserUserGroupToPerm', primaryjoin='UserUserGroupToPerm.user_id==User.user_id', cascade='all, delete-orphan') - - group_member = relationship('UserGroupMember', cascade='all') - - notifications = relationship('UserNotification', cascade='all') + repo_group_to_perm = relationship('UserRepoGroupToPerm', primaryjoin='UserRepoGroupToPerm.user_id==User.user_id', cascade='all, delete-orphan', back_populates='user') + user_group_to_perm = relationship('UserUserGroupToPerm', primaryjoin='UserUserGroupToPerm.user_id==User.user_id', cascade='all, delete-orphan', back_populates='user') + + group_member = relationship('UserGroupMember', cascade='all', back_populates='user') + + notifications = relationship('UserNotification', cascade='all', back_populates='user') # notifications assigned to this user - user_created_notifications = relationship('Notification', cascade='all') + user_created_notifications = relationship('Notification', cascade='all', back_populates='created_by_user') # comments created by this user - user_comments = relationship('ChangesetComment', cascade='all') + user_comments = relationship('ChangesetComment', cascade='all', back_populates='author') # user profile extra info - user_emails = relationship('UserEmailMap', cascade='all') - user_ip_map = relationship('UserIpMap', cascade='all') - user_auth_tokens = relationship('UserApiKeys', cascade='all') - user_ssh_keys = relationship('UserSshKeys', cascade='all') + user_emails = relationship('UserEmailMap', cascade='all', back_populates='user') + user_ip_map = relationship('UserIpMap', cascade='all', back_populates='user') + user_auth_tokens = relationship('UserApiKeys', cascade='all', back_populates='user') + user_ssh_keys = relationship('UserSshKeys', cascade='all', back_populates='user') # gists - user_gists = relationship('Gist', cascade='all') + user_gists = relationship('Gist', cascade='all', back_populates='owner') # user pull requests - user_pull_requests = relationship('PullRequest', cascade='all') + user_pull_requests = relationship('PullRequest', cascade='all', back_populates='author') # external identities - external_identities = relationship( - 'ExternalIdentity', - primaryjoin="User.user_id==ExternalIdentity.local_user_id", - cascade='all') + external_identities = relationship('ExternalIdentity', primaryjoin="User.user_id==ExternalIdentity.local_user_id", cascade='all') # review rules - user_review_rules = relationship('RepoReviewRuleUser', cascade='all') + user_review_rules = relationship('RepoReviewRuleUser', cascade='all', back_populates='user') # artifacts owned - artifacts = relationship('FileStore', primaryjoin='FileStore.user_id==User.user_id') + artifacts = relationship('FileStore', primaryjoin='FileStore.user_id==User.user_id', back_populates='upload_user') # no cascade, set NULL - scope_artifacts = relationship('FileStore', primaryjoin='FileStore.scope_user_id==User.user_id') - - - def __str__(self): - return f"<{self.__class__.__name__}('id:{self.user_id}:{self.username}')>" + scope_artifacts = relationship('FileStore', primaryjoin='FileStore.scope_user_id==User.user_id', cascade='', back_populates='user') + + def __repr__(self): + return f"<{self.cls_name}('id={self.user_id}, username={self.username}')>" @hybrid_property def email(self): @@ -856,7 +879,7 @@ class User(Base, BaseModel): @property def username_and_name(self): - return '%s (%s %s)' % (self.username, self.first_name, self.last_name) + return f'{self.username} ({self.first_name} {self.last_name})' @property def username_or_name_or_email(self): @@ -865,20 +888,20 @@ class User(Base, BaseModel): @property def full_name(self): - return '%s %s' % (self.first_name, self.last_name) + return f'{self.first_name} {self.last_name}' @property def full_name_or_username(self): - return ('%s %s' % (self.first_name, self.last_name) + return (f'{self.first_name} {self.last_name}' if (self.first_name and self.last_name) else self.username) @property def full_contact(self): - return '%s %s <%s>' % (self.first_name, self.last_name, self.email) + return f'{self.first_name} {self.last_name} <{self.email}>' @property def short_contact(self): - return '%s %s' % (self.first_name, self.last_name) + return f'{self.first_name} {self.last_name}' @property def is_admin(self): @@ -910,74 +933,73 @@ class User(Base, BaseModel): if not isinstance(val, dict): raise Exception('user_data must be dict, got %s' % type(val)) try: - self._user_data = json.dumps(val) + self._user_data = safe_bytes(json.dumps(val)) except Exception: log.error(traceback.format_exc()) @classmethod def get_by_username(cls, username, case_insensitive=False, - cache=False, identity_cache=False): - session = Session() + cache=False): if case_insensitive: - q = cls.query().filter( + q = cls.select().where( func.lower(cls.username) == func.lower(username)) else: - q = cls.query().filter(cls.username == username) + q = cls.select().where(cls.username == username) if cache: - if identity_cache: - val = cls.identity_cache(session, 'username', username) - if val: - return val - else: - cache_key = "get_user_by_name_%s" % _hash_key(username) - q = q.options( - FromCache("sql_cache_short", cache_key)) - - return q.scalar() + hash_key = _hash_key(username) + q = q.options( + FromCache("sql_cache_short", f"get_user_by_name_{hash_key}")) + + return cls.execute(q).scalar_one_or_none() @classmethod def get_by_auth_token(cls, auth_token, cache=False): - q = UserApiKeys.query()\ - .filter(UserApiKeys.api_key == auth_token)\ - .filter(or_(UserApiKeys.expires == -1, - UserApiKeys.expires >= time.time())) + + q = cls.select(User)\ + .join(UserApiKeys)\ + .where(UserApiKeys.api_key == auth_token)\ + .where(or_(UserApiKeys.expires == -1, + UserApiKeys.expires >= time.time())) + if cache: q = q.options( FromCache("sql_cache_short", f"get_auth_token_{auth_token}")) - match = q.first() - if match: - return match.user + matched_user = cls.execute(q).scalar_one_or_none() + + return matched_user @classmethod def get_by_email(cls, email, case_insensitive=False, cache=False): if case_insensitive: - q = cls.query().filter(func.lower(cls.email) == func.lower(email)) - + q = cls.select().where(func.lower(cls.email) == func.lower(email)) else: - q = cls.query().filter(cls.email == email) - - email_key = _hash_key(email) + q = cls.select().where(cls.email == email) + if cache: + email_key = _hash_key(email) q = q.options( FromCache("sql_cache_short", f"get_email_key_{email_key}")) - ret = q.scalar() + ret = cls.execute(q).scalar_one_or_none() + if ret is None: - q = UserEmailMap.query() + q = cls.select(UserEmailMap) # try fetching in alternate email map if case_insensitive: - q = q.filter(func.lower(UserEmailMap.email) == func.lower(email)) + q = q.where(func.lower(UserEmailMap.email) == func.lower(email)) else: - q = q.filter(UserEmailMap.email == email) + q = q.where(UserEmailMap.email == email) q = q.options(joinedload(UserEmailMap.user)) if cache: q = q.options( FromCache("sql_cache_short", f"get_email_map_key_{email_key}")) - ret = getattr(q.scalar(), 'user', None) + + result = cls.execute(q).scalar_one_or_none() + ret = getattr(result, 'user', None) return ret @@ -1023,10 +1045,8 @@ class User(Base, BaseModel): @classmethod def get_first_super_admin(cls): - user = User.query()\ - .filter(User.admin == true()) \ - .order_by(User.user_id.asc()) \ - .first() + stmt = cls.select().where(User.admin == true()).order_by(User.user_id.asc()) + user = cls.scalars(stmt).first() if user is None: raise Exception('FATAL: Missing administrative account!') @@ -1064,6 +1084,7 @@ class User(Base, BaseModel): # A call to refresh() ensures that the # latest state from the database is used. Session().refresh(user) + return user @classmethod @@ -1168,17 +1189,17 @@ class UserApiKeys(Base, BaseModel): repo_id = Column( 'repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None) - repo = relationship('Repository', lazy='joined') + repo = relationship('Repository', lazy='joined', back_populates='scoped_tokens') repo_group_id = Column( 'repo_group_id', Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None) repo_group = relationship('RepoGroup', lazy='joined') - user = relationship('User', lazy='joined') - - def __str__(self): - return f"<{self.__class__.__name__}('{self.role}')>" + user = relationship('User', lazy='joined', back_populates='user_auth_tokens') + + def __repr__(self): + return f"<{self.cls_name}('{self.role}')>" def __json__(self): data = { @@ -1260,6 +1281,7 @@ class UserEmailMap(Base, BaseModel): __tablename__ = 'user_email_map' __table_args__ = ( Index('uem_email_idx', 'email'), + Index('uem_user_id_idx', 'user_id'), UniqueConstraint('email'), base_table_args ) @@ -1268,7 +1290,7 @@ class UserEmailMap(Base, BaseModel): email_id = Column("email_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) _email = Column("email", String(255), nullable=True, unique=False, default=None) - user = relationship('User', lazy='joined') + user = relationship('User', lazy='joined', back_populates='user_emails') @validates('_email') def validate_email(self, key, email): @@ -1300,7 +1322,7 @@ class UserIpMap(Base, BaseModel): ip_addr = Column("ip_addr", String(255), nullable=True, unique=False, default=None) active = Column("active", Boolean(), nullable=True, unique=None, default=True) description = Column("description", String(10000), nullable=True, unique=None, default=None) - user = relationship('User', lazy='joined') + user = relationship('User', lazy='joined', back_populates='user_ip_map') @hybrid_property def description_safe(self): @@ -1309,7 +1331,7 @@ class UserIpMap(Base, BaseModel): @classmethod def _get_ip_range(cls, ip_addr): - net = ipaddress.ip_network(safe_unicode(ip_addr), strict=False) + net = ipaddress.ip_network(safe_str(ip_addr), strict=False) return [str(net.network_address), str(net.broadcast_address)] def __json__(self): @@ -1318,8 +1340,8 @@ class UserIpMap(Base, BaseModel): 'ip_range': self._get_ip_range(self.ip_addr), } - def __str__(self): - return "<%s('user_id:%s=>%s')>" % (self.__class__.__name__, self.user_id, self.ip_addr) + def __repr__(self): + return f"<{self.cls_name}('user_id={self.user_id} => ip={self.ip_addr}')>" class UserSshKeys(Base, BaseModel): @@ -1343,7 +1365,7 @@ class UserSshKeys(Base, BaseModel): accessed_on = Column('accessed_on', DateTime(timezone=False), nullable=True, default=None) user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) - user = relationship('User', lazy='joined') + user = relationship('User', lazy='joined', back_populates='user_ssh_keys') def __json__(self): data = { @@ -1380,10 +1402,11 @@ class UserLog(Base, BaseModel): version = Column("version", String(255), nullable=True, default=VERSION_1) user_data = Column('user_data_json', MutationObj.as_mutable(JsonType(dialect_map=dict(mysql=LONGTEXT())))) action_data = Column('action_data_json', MutationObj.as_mutable(JsonType(dialect_map=dict(mysql=LONGTEXT())))) - - - def __str__(self): - return f"<{self.__class__.__name__}('id:{self.repository_name}:{self.action}')>" + user = relationship('User', cascade='', back_populates='user_log') + repository = relationship('Repository', cascade='', back_populates='logs') + + def __repr__(self): + return f"<{self.cls_name}('id:{self.repository_name}:{self.action}')>" def __json__(self): return { @@ -1404,9 +1427,6 @@ class UserLog(Base, BaseModel): def action_as_day(self): return datetime.date(*self.action_date.timetuple()[:3]) - user = relationship('User') - repository = relationship('Repository', cascade='') - class UserGroup(Base, BaseModel): __tablename__ = 'users_groups' @@ -1423,15 +1443,16 @@ class UserGroup(Base, BaseModel): created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) _group_data = Column("group_data", LargeBinary(), nullable=True) # JSON data - members = relationship('UserGroupMember', cascade="all, delete-orphan", lazy="joined") - users_group_to_perm = relationship('UserGroupToPerm', cascade='all') - users_group_repo_to_perm = relationship('UserGroupRepoToPerm', cascade='all') - users_group_repo_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all') - user_user_group_to_perm = relationship('UserUserGroupToPerm', cascade='all') - user_group_user_group_to_perm = relationship('UserGroupUserGroupToPerm ', primaryjoin="UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id", cascade='all') - - user_group_review_rules = relationship('RepoReviewRuleUserGroup', cascade='all') - user = relationship('User', primaryjoin="User.user_id==UserGroup.user_id") + members = relationship('UserGroupMember', cascade="all, delete-orphan", lazy="joined", back_populates='users_group') + users_group_to_perm = relationship('UserGroupToPerm', cascade='all', back_populates='users_group') + users_group_repo_to_perm = relationship('UserGroupRepoToPerm', cascade='all', back_populates='users_group') + users_group_repo_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all', back_populates='users_group') + user_user_group_to_perm = relationship('UserUserGroupToPerm', cascade='all', back_populates='user_group') + + user_group_user_group_to_perm = relationship('UserGroupUserGroupToPerm', primaryjoin="UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id", cascade='all', back_populates='target_user_group') + + user_group_review_rules = relationship('RepoReviewRuleUserGroup', cascade='all', back_populates='users_group') + user = relationship('User', primaryjoin="User.user_id==UserGroup.user_id", back_populates='user_groups') @classmethod def _load_group_data(cls, column): @@ -1472,8 +1493,8 @@ class UserGroup(Base, BaseModel): def sync(self): return self._load_sync(self.group_data) - def __str__(self): - return f"<{self.__class__.__name__}('id:{self.users_group_id}:{self.users_group_name}')>" + def __repr__(self): + return f"<{self.cls_name}('id:{self.users_group_id}:{self.users_group_name}')>" @classmethod def get_by_group_name(cls, group_name, cache=False, @@ -1624,8 +1645,8 @@ class UserGroupMember(Base, BaseModel): users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) - user = relationship('User', lazy='joined') - users_group = relationship('UserGroup') + user = relationship('User', lazy='joined', back_populates='group_member') + users_group = relationship('UserGroup', back_populates='members') def __init__(self, gr_id='', u_id=''): self.users_group_id = gr_id @@ -1650,7 +1671,7 @@ class RepositoryField(Base, BaseModel): field_type = Column("field_type", String(255), nullable=False, unique=None) created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) - repository = relationship('Repository') + repository = relationship('Repository', back_populates='extra_fields') @property def field_key_prefixed(self): @@ -1745,44 +1766,48 @@ class Repository(Base, BaseModel): "group_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=False, default=None) - user = relationship('User', lazy='joined') + user = relationship('User', lazy='joined', back_populates='repositories') fork = relationship('Repository', remote_side=repo_id, lazy='joined') group = relationship('RepoGroup', lazy='joined') - repo_to_perm = relationship( - 'UserRepoToPerm', cascade='all', - order_by='UserRepoToPerm.repo_to_perm_id') - users_group_to_perm = relationship('UserGroupRepoToPerm', cascade='all') + repo_to_perm = relationship('UserRepoToPerm', cascade='all', order_by='UserRepoToPerm.repo_to_perm_id') + users_group_to_perm = relationship('UserGroupRepoToPerm', cascade='all', back_populates='repository') stats = relationship('Statistics', cascade='all', uselist=False) - followers = relationship( - 'UserFollowing', - primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', - cascade='all') - extra_fields = relationship( - 'RepositoryField', cascade="all, delete-orphan") - logs = relationship('UserLog') - comments = relationship( - 'ChangesetComment', cascade="all, delete-orphan") + followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', cascade='all', back_populates='follows_repository') + extra_fields = relationship('RepositoryField', cascade="all, delete-orphan", back_populates='repository') + + logs = relationship('UserLog', back_populates='repository') + + comments = relationship('ChangesetComment', cascade="all, delete-orphan", back_populates='repo') + pull_requests_source = relationship( 'PullRequest', primaryjoin='PullRequest.source_repo_id==Repository.repo_id', - cascade="all, delete-orphan") + cascade="all, delete-orphan", + #back_populates="pr_source" + ) pull_requests_target = relationship( 'PullRequest', primaryjoin='PullRequest.target_repo_id==Repository.repo_id', - cascade="all, delete-orphan") + cascade="all, delete-orphan", + #back_populates="pr_target" + ) + ui = relationship('RepoRhodeCodeUi', cascade="all") settings = relationship('RepoRhodeCodeSetting', cascade="all") - integrations = relationship('Integration', cascade="all, delete-orphan") - - scoped_tokens = relationship('UserApiKeys', cascade="all") + integrations = relationship('Integration', cascade="all, delete-orphan", back_populates='repo') + + scoped_tokens = relationship('UserApiKeys', cascade="all", back_populates='repo') # no cascade, set NULL - artifacts = relationship('FileStore', primaryjoin='FileStore.scope_repo_id==Repository.repo_id') - - - def __str__(self): - return "<%s('%s:%s')>" % (self.__class__.__name__, self.repo_id, self.repo_name) + artifacts = relationship('FileStore', primaryjoin='FileStore.scope_repo_id==Repository.repo_id', viewonly=True) + + review_rules = relationship('RepoReviewRule') + user_branch_perms = relationship('UserToRepoBranchPermission') + user_group_branch_perms = relationship('UserGroupToRepoBranchPermission') + + def __repr__(self): + return "<%s('%s:%s')>" % (self.cls_name, self.repo_id, self.repo_name) @hybrid_property def description_safe(self): @@ -1864,7 +1889,7 @@ class Repository(Base, BaseModel): @repo_name.setter def repo_name(self, value): self._repo_name = value - self.repo_name_hash = hashlib.sha1(safe_str(value)).hexdigest() + self.repo_name_hash = sha1(safe_bytes(value)) @classmethod def normalize_repo_name(cls, repo_name): @@ -1922,10 +1947,8 @@ class Repository(Base, BaseModel): :param cls: """ - q = Session().query(RhodeCodeUi)\ - .filter(RhodeCodeUi.ui_key == cls.NAME_SEP) - q = q.options(FromCache("sql_cache_short", "repository_repo_path")) - return q.one().ui_value + from rhodecode.lib.utils import get_rhodecode_base_path + return get_rhodecode_base_path() @classmethod def get_all_repos(cls, user_id=Optional(None), group_id=Optional(None), @@ -2039,7 +2062,7 @@ class Repository(Base, BaseModel): path = self.repo_full_path return os.path.join( os.path.dirname(path), - '.__shadow_diff_cache_repo_{}'.format(self.repo_id)) + f'.__shadow_diff_cache_repo_{self.repo_id}') def cached_diffs(self): diff_cache_dir = self.cached_diffs_dir @@ -2048,16 +2071,17 @@ class Repository(Base, BaseModel): return [] def shadow_repos(self): - shadow_repos_pattern = '.__shadow_repo_{}'.format(self.repo_id) + shadow_repos_pattern = f'.__shadow_repo_{self.repo_id}' return [ x for x in os.listdir(os.path.dirname(self.repo_full_path)) - if x.startswith(shadow_repos_pattern)] + if x.startswith(shadow_repos_pattern) + ] def get_new_name(self, repo_name): """ returns new full repository name based on assigned group and new new - :param group_name: + :param repo_name: """ path_prefix = self.group.full_path_splitted if self.group else [] return self.NAME_SEP.join(path_prefix + [repo_name]) @@ -2225,15 +2249,6 @@ class Repository(Base, BaseModel): def getlock(cls, repo): return repo.locked - def is_user_lock(self, user_id): - if self.lock[0]: - lock_user_id = safe_int(self.lock[0]) - user_id = safe_int(user_id) - # both are ints, and they are equal - return all([lock_user_id, user_id]) and lock_user_id == user_id - - return False - def get_locking_state(self, action, user_id, only_when_enabled=True): """ Checks locking on this repository, if locking is enabled and lock is @@ -2432,9 +2447,10 @@ class Repository(Base, BaseModel): updated_on """ - from rhodecode.lib.vcs.backends.base import BaseChangeset + from rhodecode.lib.vcs.backends.base import BaseCommit from rhodecode.lib.vcs.utils.helpers import parse_datetime empty_date = datetime.datetime.fromtimestamp(0) + repo_commit_count = 0 if cs_cache is None: # use no-cache version here @@ -2447,10 +2463,11 @@ class Repository(Base, BaseModel): if not empty: cs_cache = scm_repo.get_commit( pre_load=["author", "date", "message", "parents", "branch"]) + repo_commit_count = scm_repo.count() else: cs_cache = EmptyCommit() - if isinstance(cs_cache, BaseChangeset): + if isinstance(cs_cache, BaseCommit): cs_cache = cs_cache.__json__() def is_outdated(new_cs_cache): @@ -2471,6 +2488,9 @@ class Repository(Base, BaseModel): if last_change_timestamp > current_timestamp and not empty: cs_cache['date'] = _current_datetime + # also store size of repo + cs_cache['repo_commit_count'] = repo_commit_count + _date_latest = parse_datetime(cs_cache.get('date') or empty_date) cs_cache['updated_on'] = time.time() self.changeset_cache = cs_cache @@ -2580,7 +2600,7 @@ class Repository(Base, BaseModel): # allows override global config full_cache = vcs_full_cache else: - full_cache = str2bool(rhodecode.CONFIG.get('vcs_full_cache')) + full_cache = rhodecode.ConfigGet().get_bool('vcs_full_cache') # if cache is NOT defined use default global, else we have a full # control over cache behaviour if cache is None and full_cache and not config: @@ -2593,7 +2613,7 @@ class Repository(Base, BaseModel): def _get_instance_cached(self): from rhodecode.lib import rc_cache - cache_namespace_uid = 'cache_repo_instance.{}'.format(self.repo_id) + cache_namespace_uid = f'repo_instance.{self.repo_id}' invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format( repo_id=self.repo_id) region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid) @@ -2655,6 +2675,7 @@ class Repository(Base, BaseModel): result = super(Repository, self).get_dict() result['repo_name'] = result.pop('_repo_name', None) + result.pop('_changeset_cache', '') return result @@ -2683,22 +2704,21 @@ class RepoGroup(Base, BaseModel): personal = Column('personal', Boolean(), nullable=True, unique=None, default=None) _changeset_cache = Column("changeset_cache", LargeBinary(), nullable=True) # JSON data - repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id') - users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all') + repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id', back_populates='group') + users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all', back_populates='group') parent_group = relationship('RepoGroup', remote_side=group_id) - user = relationship('User') - integrations = relationship('Integration', cascade="all, delete-orphan") + user = relationship('User', back_populates='repository_groups') + integrations = relationship('Integration', cascade="all, delete-orphan", back_populates='repo_group') # no cascade, set NULL - scope_artifacts = relationship('FileStore', primaryjoin='FileStore.scope_repo_group_id==RepoGroup.group_id') + scope_artifacts = relationship('FileStore', primaryjoin='FileStore.scope_repo_group_id==RepoGroup.group_id', viewonly=True) def __init__(self, group_name='', parent_group=None): self.group_name = group_name self.parent_group = parent_group - def __str__(self): - return "<%s('id:%s:%s')>" % ( - self.__class__.__name__, self.group_id, self.group_name) + def __repr__(self): + return f"<{self.cls_name}('id:{self.group_id}:{self.group_name}')>" @hybrid_property def group_name(self): @@ -2766,7 +2786,10 @@ class RepoGroup(Base, BaseModel): @classmethod def _generate_choice(cls, repo_group): from webhelpers2.html import literal as _literal - _name = lambda k: _literal(cls.CHOICES_SEPARATOR.join(k)) + + def _name(k): + return _literal(cls.CHOICES_SEPARATOR.join(k)) + return repo_group.group_id, _name(repo_group.full_path_splitted) @classmethod @@ -2776,7 +2799,7 @@ class RepoGroup(Base, BaseModel): repo_groups = [] if show_empty_group: - repo_groups = [(-1, u'-- %s --' % _('No parent'))] + repo_groups = [(-1, '-- %s --' % _('No parent'))] repo_groups.extend([cls._generate_choice(x) for x in groups]) @@ -2798,7 +2821,7 @@ class RepoGroup(Base, BaseModel): if cache: name_key = _hash_key(group_name) gr = gr.options( - FromCache("sql_cache_short", "get_group_%s" % name_key)) + FromCache("sql_cache_short", f"get_group_{name_key}")) return gr.scalar() @classmethod @@ -3107,6 +3130,7 @@ class RepoGroup(Base, BaseModel): # keep compatibility with the code which uses `group_name` field. result = super(RepoGroup, self).get_dict() result['group_name'] = result.pop('_group_name', None) + result.pop('_changeset_cache', '') return result @@ -3230,9 +3254,9 @@ class Permission(Base, BaseModel): permission_name = Column("permission_name", String(255), nullable=True, unique=None, default=None) permission_longname = Column("permission_longname", String(255), nullable=True, unique=None, default=None) - def __str__(self): + def __repr__(self): return "<%s('%s:%s')>" % ( - self.__class__.__name__, self.permission_id, self.permission_name + self.cls_name, self.permission_id, self.permission_name ) @classmethod @@ -3404,11 +3428,11 @@ class UserRepoToPerm(Base, BaseModel): permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) - user = relationship('User') - repository = relationship('Repository') + user = relationship('User', back_populates="repo_to_perm") + repository = relationship('Repository', back_populates="repo_to_perm") permission = relationship('Permission') - branch_perm_entry = relationship('UserToRepoBranchPermission', cascade="all, delete-orphan", lazy='joined') + branch_perm_entry = relationship('UserToRepoBranchPermission', cascade="all, delete-orphan", lazy='joined', back_populates='user_repo_to_perm') @classmethod def create(cls, user, repository, permission): @@ -3419,7 +3443,7 @@ class UserRepoToPerm(Base, BaseModel): Session().add(n) return n - def __str__(self): + def __repr__(self): return f'<{self.user} => {self.repository} >' @@ -3435,8 +3459,8 @@ class UserUserGroupToPerm(Base, BaseMode permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) - user = relationship('User') - user_group = relationship('UserGroup') + user = relationship('User', back_populates='user_group_to_perm') + user_group = relationship('UserGroup', back_populates='user_user_group_to_perm') permission = relationship('Permission') @classmethod @@ -3448,7 +3472,7 @@ class UserUserGroupToPerm(Base, BaseMode Session().add(n) return n - def __str__(self): + def __repr__(self): return f'<{self.user} => {self.user_group} >' @@ -3463,10 +3487,10 @@ class UserToPerm(Base, BaseModel): user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None) permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) - user = relationship('User') + user = relationship('User', back_populates='user_perms') permission = relationship('Permission', lazy='joined') - def __str__(self): + def __repr__(self): return f'<{self.user} => {self.permission} >' @@ -3482,10 +3506,10 @@ class UserGroupRepoToPerm(Base, BaseMode permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) - users_group = relationship('UserGroup') + users_group = relationship('UserGroup', back_populates='users_group_repo_to_perm') permission = relationship('Permission') - repository = relationship('Repository') - user_group_branch_perms = relationship('UserGroupToRepoBranchPermission', cascade='all') + repository = relationship('Repository', back_populates='users_group_to_perm') + user_group_branch_perms = relationship('UserGroupToRepoBranchPermission', cascade='all', back_populates='user_group_repo_to_perm') @classmethod def create(cls, users_group, repository, permission): @@ -3496,8 +3520,8 @@ class UserGroupRepoToPerm(Base, BaseMode Session().add(n) return n - def __str__(self): - return ' %s >' % (self.users_group, self.repository) + def __repr__(self): + return f' {self.repository} >' class UserGroupUserGroupToPerm(Base, BaseModel): @@ -3513,7 +3537,7 @@ class UserGroupUserGroupToPerm(Base, Bas permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) - target_user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id') + target_user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id', back_populates='user_group_user_group_to_perm') user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.user_group_id==UserGroup.users_group_id') permission = relationship('Permission') @@ -3526,8 +3550,8 @@ class UserGroupUserGroupToPerm(Base, Bas Session().add(n) return n - def __str__(self): - return ' %s >' % (self.target_user_group, self.user_group) + def __repr__(self): + return f' {self.user_group} >' class UserGroupToPerm(Base, BaseModel): @@ -3541,7 +3565,7 @@ class UserGroupToPerm(Base, BaseModel): users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None) permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) - users_group = relationship('UserGroup') + users_group = relationship('UserGroup', back_populates='users_group_to_perm') permission = relationship('Permission') @@ -3557,8 +3581,8 @@ class UserRepoGroupToPerm(Base, BaseMode group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None) permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) - user = relationship('User') - group = relationship('RepoGroup') + user = relationship('User', back_populates='repo_group_to_perm') + group = relationship('RepoGroup', back_populates='repo_group_to_perm') permission = relationship('Permission') @classmethod @@ -3583,9 +3607,9 @@ class UserGroupRepoGroupToPerm(Base, Bas group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None) permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) - users_group = relationship('UserGroup') + users_group = relationship('UserGroup', back_populates='users_group_repo_group_to_perm') permission = relationship('Permission') - group = relationship('RepoGroup') + group = relationship('RepoGroup', back_populates='users_group_to_perm') @classmethod def create(cls, user_group, repository_group, permission): @@ -3596,7 +3620,7 @@ class UserGroupRepoGroupToPerm(Base, Bas Session().add(n) return n - def __str__(self): + def __repr__(self): return ' %s >' % (self.users_group, self.group) @@ -3609,11 +3633,11 @@ class Statistics(Base, BaseModel): stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True) repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True, default=None) stat_on_revision = Column("stat_on_revision", Integer(), nullable=False) - commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False)#JSON data - commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data - languages = Column("languages", LargeBinary(1000000), nullable=False)#JSON data - - repository = relationship('Repository', single_parent=True) + commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False) #JSON data + commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False) #JSON data + languages = Column("languages", LargeBinary(1000000), nullable=False) #JSON data + + repository = relationship('Repository', single_parent=True, viewonly=True) class UserFollowing(Base, BaseModel): @@ -3630,10 +3654,10 @@ class UserFollowing(Base, BaseModel): follows_user_id = Column("follows_user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now) - user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id') + user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id', back_populates='followings') follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id') - follows_repository = relationship('Repository', order_by='Repository.repo_name') + follows_repository = relationship('Repository', order_by='Repository.repo_name', back_populates='followers') @classmethod def get_repo_followers(cls, repo_id): @@ -3667,9 +3691,9 @@ class CacheKey(Base, BaseModel): # first key should be same for all entries, since all workers should share it self.cache_state_uid = cache_state_uid or self.generate_new_state_uid() - def __str__(self): + def __repr__(self): return "<%s('%s:%s[%s]')>" % ( - self.__class__.__name__, + self.cls_name, self.cache_id, self.cache_key, self.cache_active) def _cache_key_partition(self): @@ -3756,13 +3780,13 @@ class ChangesetComment(Base, BaseModel): base_table_args, ) - COMMENT_OUTDATED = u'comment_outdated' - COMMENT_TYPE_NOTE = u'note' - COMMENT_TYPE_TODO = u'todo' + COMMENT_OUTDATED = 'comment_outdated' + COMMENT_TYPE_NOTE = 'note' + COMMENT_TYPE_TODO = 'todo' COMMENT_TYPES = [COMMENT_TYPE_NOTE, COMMENT_TYPE_TODO] - OP_IMMUTABLE = u'immutable' - OP_CHANGEABLE = u'changeable' + OP_IMMUTABLE = 'immutable' + OP_CHANGEABLE = 'changeable' comment_id = Column('comment_id', Integer(), nullable=False, primary_key=True) repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False) @@ -3787,12 +3811,12 @@ class ChangesetComment(Base, BaseModel): resolved_comment = relationship('ChangesetComment', remote_side=comment_id, back_populates='resolved_by') resolved_by = relationship('ChangesetComment', back_populates='resolved_comment') - author = relationship('User', lazy='select') - repo = relationship('Repository') - status_change = relationship('ChangesetStatus', cascade="all, delete-orphan", lazy='select') - pull_request = relationship('PullRequest', lazy='select') + author = relationship('User', lazy='select', back_populates='user_comments') + repo = relationship('Repository', back_populates='comments') + status_change = relationship('ChangesetStatus', cascade="all, delete-orphan", lazy='select', back_populates='comment') + pull_request = relationship('PullRequest', lazy='select', back_populates='comments') pull_request_version = relationship('PullRequestVersion', lazy='select') - history = relationship('ChangesetCommentHistory', cascade='all, delete-orphan', lazy='select', order_by='ChangesetCommentHistory.version') + history = relationship('ChangesetCommentHistory', cascade='all, delete-orphan', lazy='select', order_by='ChangesetCommentHistory.version', back_populates="comment") @classmethod def get_users(cls, revision=None, pull_request_id=None): @@ -3803,8 +3827,7 @@ class ChangesetComment(Base, BaseModel): :param cls: :param revision: """ - q = Session().query(User)\ - .join(ChangesetComment.author) + q = Session().query(User).join(ChangesetComment.author) if revision: q = q.filter(cls.revision == revision) elif pull_request_id: @@ -3813,6 +3836,8 @@ class ChangesetComment(Base, BaseModel): @classmethod def get_index_from_version(cls, pr_version, versions=None, num_versions=None): + if pr_version is None: + return 0 if versions is not None: num_versions = [x.pull_request_version_id for x in versions] @@ -3858,10 +3883,14 @@ class ChangesetComment(Base, BaseModel): """ Checks if comment is made from previous version than given """ + cur_ver = 0 + if self.pull_request_version: + cur_ver = self.pull_request_version.pull_request_version_id or cur_ver + if version is None: - return self.pull_request_version != version - - return self.pull_request_version < version + return cur_ver != version + + return cur_ver < version def older_than_version_js(self, version): """ @@ -3909,7 +3938,7 @@ class ChangesetComment(Base, BaseModel): if self.status_change: return self.status_change[0].status_lbl - def __str__(self): + def __repr__(self): if self.comment_id: return f'' else: @@ -3956,7 +3985,7 @@ class ChangesetCommentHistory(Base, Base deleted = Column('deleted', Boolean(), default=False) author = relationship('User', lazy='joined') - comment = relationship('ChangesetComment', cascade="all, delete") + comment = relationship('ChangesetComment', cascade="all, delete", back_populates="history") @classmethod def get_version(cls, comment_id): @@ -3983,7 +4012,7 @@ class ChangesetStatus(Base, BaseModel): STATUS_APPROVED = 'approved' STATUS_REJECTED = 'rejected' STATUS_UNDER_REVIEW = 'under_review' - CheckConstraint, + STATUSES = [ (STATUS_NOT_REVIEWED, _("Not Reviewed")), # (no icon) and default (STATUS_APPROVED, _("Approved")), @@ -4003,12 +4032,11 @@ class ChangesetStatus(Base, BaseModel): author = relationship('User', lazy='select') repo = relationship('Repository', lazy='select') - comment = relationship('ChangesetComment', lazy='select') - pull_request = relationship('PullRequest', lazy='select') - - - def __str__(self): - return f"<{self.__class__.__name__}('{self.status}[v{self.version}]:{self.author}')>" + comment = relationship('ChangesetComment', lazy='select', back_populates='status_change') + pull_request = relationship('PullRequest', lazy='select', back_populates='statuses') + + def __repr__(self): + return f"<{self.cls_name}('{self.status}[v{self.version}]:{self.author}')>" @classmethod def get_status_lbl(cls, value): @@ -4052,7 +4080,7 @@ class _SetState(object): def __exit__(self, exc_type, exc_val, exc_tb): if exc_val is not None or exc_type is not None: - log.error(traceback.format_exc(exc_tb)) + log.error(traceback.format_tb(exc_tb)) return None self.set_pr_state(self._org_state) @@ -4080,15 +4108,15 @@ class _PullRequestBase(BaseModel): """ # .status values - STATUS_NEW = u'new' - STATUS_OPEN = u'open' - STATUS_CLOSED = u'closed' + STATUS_NEW = 'new' + STATUS_OPEN = 'open' + STATUS_CLOSED = 'closed' # available states - STATE_CREATING = u'creating' - STATE_UPDATING = u'updating' - STATE_MERGING = u'merging' - STATE_CREATED = u'created' + STATE_CREATING = 'creating' + STATE_UPDATING = 'updating' + STATE_MERGING = 'merging' + STATE_CREATED = 'created' title = Column('title', Unicode(255), nullable=True) description = Column( @@ -4126,6 +4154,14 @@ class _PullRequestBase(BaseModel): 'org_repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False) + @declared_attr + def pr_source(cls): + return relationship( + 'Repository', + primaryjoin=f'{cls.__name__}.source_repo_id==Repository.repo_id', + overlaps="pull_requests_source" + ) + _source_ref = Column('org_ref', Unicode(255), nullable=False) @hybrid_property @@ -4161,6 +4197,14 @@ class _PullRequestBase(BaseModel): 'other_repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False) + @declared_attr + def pr_target(cls): + return relationship( + 'Repository', + primaryjoin=f'{cls.__name__}.target_repo_id==Repository.repo_id', + overlaps="pull_requests_target" + ) + _shadow_merge_ref = Column('shadow_merge_ref', Unicode(255), nullable=True) # TODO: dan: rename column to last_merge_source_rev @@ -4226,7 +4270,7 @@ class _PullRequestBase(BaseModel): @revisions.setter def revisions(self, val): - self._revisions = u':'.join(val) + self._revisions = ':'.join(val) @hybrid_property def last_merge_status(self): @@ -4238,13 +4282,19 @@ class _PullRequestBase(BaseModel): @declared_attr def author(cls): - return relationship('User', lazy='joined') + return relationship( + 'User', lazy='joined', + #TODO, problem that is somehow :? + #back_populates='user_pull_requests' + ) @declared_attr def source_repo(cls): return relationship( 'Repository', - primaryjoin='%s.source_repo_id==Repository.repo_id' % cls.__name__) + primaryjoin=f'{cls.__name__}.source_repo_id==Repository.repo_id', + #back_populates='' + ) @property def source_ref_parts(self): @@ -4254,7 +4304,8 @@ class _PullRequestBase(BaseModel): def target_repo(cls): return relationship( 'Repository', - primaryjoin='%s.target_repo_id==Repository.repo_id' % cls.__name__) + primaryjoin=f'{cls.__name__}.target_repo_id==Repository.repo_id' + ) @property def target_ref_parts(self): @@ -4294,7 +4345,7 @@ class _PullRequestBase(BaseModel): merge_data = { 'clone_url': PullRequestModel().get_shadow_clone_url(pull_request), 'reference': ( - pull_request.shadow_merge_ref._asdict() + pull_request.shadow_merge_ref.asdict() if pull_request.shadow_merge_ref else None), } @@ -4370,15 +4421,16 @@ class PullRequest(Base, _PullRequestBase pull_request_id = Column( 'pull_request_id', Integer(), nullable=False, primary_key=True) - def __str__(self): + def __repr__(self): if self.pull_request_id: return f'' else: - reviewers = relationship('PullRequestReviewers', cascade="all, delete-orphan") - statuses = relationship('ChangesetStatus', cascade="all, delete-orphan") - comments = relationship('ChangesetComment', cascade="all, delete-orphan") - versions = relationship('PullRequestVersion', cascade="all, delete-orphan", - lazy='dynamic') + return f'' + + reviewers = relationship('PullRequestReviewers', cascade="all, delete-orphan", back_populates='pull_request') + statuses = relationship('ChangesetStatus', cascade="all, delete-orphan", back_populates='pull_request') + comments = relationship('ChangesetComment', cascade="all, delete-orphan", back_populates='pull_request') + versions = relationship('PullRequestVersion', cascade="all, delete-orphan", lazy='dynamic', back_populates='pull_request') @classmethod def get_pr_display_object(cls, pull_request_obj, org_pull_request_obj, @@ -4405,7 +4457,7 @@ class PullRequest(Base, _PullRequestBase raise AttributeError( '%s object has no attribute %s' % (self, item)) - def __str__(self): + def __repr__(self): pr_id = self.attrs.get('pull_request_id') return f'' @@ -4533,14 +4585,11 @@ class PullRequestVersion(Base, _PullRequ base_table_args, ) - pull_request_version_id = Column( - 'pull_request_version_id', Integer(), nullable=False, primary_key=True) - pull_request_id = Column( - 'pull_request_id', Integer(), - ForeignKey('pull_requests.pull_request_id'), nullable=False) - pull_request = relationship('PullRequest') - - def __str__(self): + pull_request_version_id = Column('pull_request_version_id', Integer(), nullable=False, primary_key=True) + pull_request_id = Column('pull_request_id', Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=False) + pull_request = relationship('PullRequest', back_populates='versions') + + def __repr__(self): if self.pull_request_version_id: return f'' else: @@ -4551,10 +4600,6 @@ class PullRequestVersion(Base, _PullRequ return self.pull_request.reviewers @property - def reviewers(self): - return self.pull_request.reviewers - - @property def versions(self): return self.pull_request.versions @@ -4580,8 +4625,8 @@ class PullRequestReviewers(Base, BaseMod __table_args__ = ( base_table_args, ) - ROLE_REVIEWER = u'reviewer' - ROLE_OBSERVER = u'observer' + ROLE_REVIEWER = 'reviewer' + ROLE_OBSERVER = 'observer' ROLES = [ROLE_REVIEWER, ROLE_OBSERVER] @hybrid_property @@ -4613,7 +4658,7 @@ class PullRequestReviewers(Base, BaseMod role = Column('role', Unicode(255), nullable=True, default=ROLE_REVIEWER) user = relationship('User') - pull_request = relationship('PullRequest') + pull_request = relationship('PullRequest', back_populates='reviewers') rule_data = Column( 'rule_data_json', @@ -4643,8 +4688,8 @@ class PullRequestReviewers(Base, BaseMod return qry.all() - def __str__(self): - return f"<{self.__class__.__name__}('id:{self.pull_requests_reviewers_id}')>" + def __repr__(self): + return f"<{self.cls_name}('id:{self.pull_requests_reviewers_id}')>" class Notification(Base, BaseModel): @@ -4654,13 +4699,13 @@ class Notification(Base, BaseModel): base_table_args, ) - TYPE_CHANGESET_COMMENT = u'cs_comment' - TYPE_MESSAGE = u'message' - TYPE_MENTION = u'mention' - TYPE_REGISTRATION = u'registration' - TYPE_PULL_REQUEST = u'pull_request' - TYPE_PULL_REQUEST_COMMENT = u'pull_request_comment' - TYPE_PULL_REQUEST_UPDATE = u'pull_request_update' + TYPE_CHANGESET_COMMENT = 'cs_comment' + TYPE_MESSAGE = 'message' + TYPE_MENTION = 'mention' + TYPE_REGISTRATION = 'registration' + TYPE_PULL_REQUEST = 'pull_request' + TYPE_PULL_REQUEST_COMMENT = 'pull_request_comment' + TYPE_PULL_REQUEST_UPDATE = 'pull_request_update' notification_id = Column('notification_id', Integer(), nullable=False, primary_key=True) subject = Column('subject', Unicode(512), nullable=True) @@ -4669,9 +4714,8 @@ class Notification(Base, BaseModel): created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) type_ = Column('type', Unicode(255)) - created_by_user = relationship('User') - notifications_to_users = relationship('UserNotification', lazy='joined', - cascade="all, delete-orphan") + created_by_user = relationship('User', back_populates='user_created_notifications') + notifications_to_users = relationship('UserNotification', lazy='joined', cascade="all, delete-orphan", back_populates='notification') @property def recipients(self): @@ -4720,9 +4764,8 @@ class UserNotification(Base, BaseModel): read = Column('read', Boolean, default=False) sent_on = Column('sent_on', DateTime(timezone=False), nullable=True, unique=None) - user = relationship('User', lazy="joined") - notification = relationship('Notification', lazy="joined", - order_by=lambda: Notification.created_on.desc(),) + user = relationship('User', lazy="joined", back_populates='notifications') + notification = relationship('Notification', lazy="joined", order_by=lambda: Notification.created_on.desc(), back_populates='notifications_to_users') def mark_as_read(self): self.read = True @@ -4796,12 +4839,12 @@ class Gist(Base, BaseModel): base_table_args ) - GIST_PUBLIC = u'public' - GIST_PRIVATE = u'private' - DEFAULT_FILENAME = u'gistfile1.txt' - - ACL_LEVEL_PUBLIC = u'acl_public' - ACL_LEVEL_PRIVATE = u'acl_private' + GIST_PUBLIC = 'public' + GIST_PRIVATE = 'private' + DEFAULT_FILENAME = 'gistfile1.txt' + + ACL_LEVEL_PUBLIC = 'acl_public' + ACL_LEVEL_PRIVATE = 'acl_private' gist_id = Column('gist_id', Integer(), primary_key=True) gist_access_id = Column('gist_access_id', Unicode(250)) @@ -4813,9 +4856,9 @@ class Gist(Base, BaseModel): modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now) acl_level = Column('acl_level', Unicode(128), nullable=True) - owner = relationship('User') - - def __str__(self): + owner = relationship('User', back_populates='user_gists') + + def __repr__(self): return f'' @hybrid_property @@ -4901,13 +4944,13 @@ class ExternalIdentity(Base, BaseModel): base_table_args ) - external_id = Column('external_id', Unicode(255), default=u'', primary_key=True) - external_username = Column('external_username', Unicode(1024), default=u'') + external_id = Column('external_id', Unicode(255), default='', primary_key=True) + external_username = Column('external_username', Unicode(1024), default='') local_user_id = Column('local_user_id', Integer(), ForeignKey('users.user_id'), primary_key=True) - provider_name = Column('provider_name', Unicode(255), default=u'', primary_key=True) - access_token = Column('access_token', String(1024), default=u'') - alt_token = Column('alt_token', String(1024), default=u'') - token_secret = Column('token_secret', String(1024), default=u'') + provider_name = Column('provider_name', Unicode(255), default='', primary_key=True) + access_token = Column('access_token', String(1024), default='') + alt_token = Column('alt_token', String(1024), default='') + token_secret = Column('token_secret', String(1024), default='') @classmethod def by_external_id_and_provider(cls, external_id, provider_name, local_user_id=None): @@ -4970,21 +5013,16 @@ class Integration(Base, BaseModel): integration_type = Column('integration_type', String(255)) enabled = Column('enabled', Boolean(), nullable=False) name = Column('name', String(255), nullable=False) - child_repos_only = Column('child_repos_only', Boolean(), nullable=False, - default=False) + child_repos_only = Column('child_repos_only', Boolean(), nullable=False, default=False) settings = Column( 'settings_json', MutationObj.as_mutable( JsonType(dialect_map=dict(mysql=UnicodeText(16384))))) - repo_id = Column( - 'repo_id', Integer(), ForeignKey('repositories.repo_id'), - nullable=True, unique=None, default=None) - repo = relationship('Repository', lazy='joined') - - repo_group_id = Column( - 'repo_group_id', Integer(), ForeignKey('groups.group_id'), - nullable=True, unique=None, default=None) - repo_group = relationship('RepoGroup', lazy='joined') + repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None) + repo = relationship('Repository', lazy='joined', back_populates='integrations') + + repo_group_id = Column('repo_group_id', Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None) + repo_group = relationship('RepoGroup', lazy='joined', back_populates='integrations') @property def scope(self): @@ -4999,7 +5037,7 @@ class Integration(Base, BaseModel): return 'root_repos' return 'global' - def __str__(self): + def __repr__(self): return '' % (self.integration_type, self.scope) @@ -5008,8 +5046,8 @@ class RepoReviewRuleUser(Base, BaseModel __table_args__ = ( base_table_args ) - ROLE_REVIEWER = u'reviewer' - ROLE_OBSERVER = u'observer' + ROLE_REVIEWER = 'reviewer' + ROLE_OBSERVER = 'observer' ROLES = [ROLE_REVIEWER, ROLE_OBSERVER] repo_review_rule_user_id = Column('repo_review_rule_user_id', Integer(), primary_key=True) @@ -5017,7 +5055,7 @@ class RepoReviewRuleUser(Base, BaseModel user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False) mandatory = Column("mandatory", Boolean(), nullable=False, default=False) role = Column('role', Unicode(255), nullable=True, default=ROLE_REVIEWER) - user = relationship('User') + user = relationship('User', back_populates='user_review_rules') def rule_data(self): return { @@ -5033,8 +5071,8 @@ class RepoReviewRuleUserGroup(Base, Base ) VOTE_RULE_ALL = -1 - ROLE_REVIEWER = u'reviewer' - ROLE_OBSERVER = u'observer' + ROLE_REVIEWER = 'reviewer' + ROLE_OBSERVER = 'observer' ROLES = [ROLE_REVIEWER, ROLE_OBSERVER] repo_review_rule_users_group_id = Column('repo_review_rule_users_group_id', Integer(), primary_key=True) @@ -5070,12 +5108,12 @@ class RepoReviewRule(Base, BaseModel): 'repo_review_rule_id', Integer(), primary_key=True) repo_id = Column( "repo_id", Integer(), ForeignKey('repositories.repo_id')) - repo = relationship('Repository', backref='review_rules') + repo = relationship('Repository', back_populates='review_rules') review_rule_name = Column('review_rule_name', String(255)) - _branch_pattern = Column("branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob - _target_branch_pattern = Column("target_branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob - _file_pattern = Column("file_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob + _branch_pattern = Column("branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default='*') # glob + _target_branch_pattern = Column("target_branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default='*') # glob + _file_pattern = Column("file_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default='*') # glob use_authors_for_review = Column("use_authors_for_review", Boolean(), nullable=False, default=False) @@ -5242,9 +5280,8 @@ class RepoReviewRule(Base, BaseModel): rules.append(user_group) return rules - def __str__(self): - return '' % ( - self.repo_review_rule_id, self.repo) + def __repr__(self): + return f'' class ScheduleEntry(Base, BaseModel): @@ -5308,7 +5345,7 @@ class ScheduleEntry(Base, BaseModel): dot_notation = obj.task_dot_notation val = '.'.join(map(safe_str, [ sorted(dot_notation), args, sorted(kwargs.items())])) - return hashlib.sha1(val).hexdigest() + return sha1(safe_bytes(val)) @classmethod def get_by_schedule_name(cls, schedule_name): @@ -5346,7 +5383,10 @@ class ScheduleEntry(Base, BaseModel): if hasattr(val, 'de_coerce'): val = val.de_coerce() if val: - val = json.dumps(val, indent=indent, sort_keys=True) + if indent: + ext_json.formatted_json(val) + else: + val = ext_json.json.dumps(val) return val @@ -5360,9 +5400,8 @@ class ScheduleEntry(Base, BaseModel): def kwargs_raw(self, indent=None): return self._as_raw(self.task_kwargs, indent) - def __str__(self): - return ''.format( - self.schedule_entry_id, self.schedule_name) + def __repr__(self): + return f'' @event.listens_for(ScheduleEntry, 'before_update') @@ -5424,19 +5463,19 @@ class UserToRepoBranchPermission(Base, _ branch_rule_id = Column('branch_rule_id', Integer(), primary_key=True) repository_id = Column('repository_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) - repo = relationship('Repository', backref='user_branch_perms') + repo = relationship('Repository', back_populates='user_branch_perms') permission_id = Column('permission_id', Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) permission = relationship('Permission') rule_to_perm_id = Column('rule_to_perm_id', Integer(), ForeignKey('repo_to_perm.repo_to_perm_id'), nullable=False, unique=None, default=None) - user_repo_to_perm = relationship('UserRepoToPerm') + user_repo_to_perm = relationship('UserRepoToPerm', back_populates='branch_perm_entry') rule_order = Column('rule_order', Integer(), nullable=False) - _branch_pattern = Column('branch_pattern', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), default=u'*') # glob + _branch_pattern = Column('branch_pattern', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), default='*') # glob _branch_hash = Column('branch_hash', UnicodeText().with_variant(UnicodeText(2048), 'mysql')) - def __str__(self): + def __repr__(self): return f' {self.branch_pattern!r})>' @@ -5449,19 +5488,19 @@ class UserGroupToRepoBranchPermission(Ba branch_rule_id = Column('branch_rule_id', Integer(), primary_key=True) repository_id = Column('repository_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None) - repo = relationship('Repository', backref='user_group_branch_perms') + repo = relationship('Repository', back_populates='user_group_branch_perms') permission_id = Column('permission_id', Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None) permission = relationship('Permission') rule_to_perm_id = Column('rule_to_perm_id', Integer(), ForeignKey('users_group_repo_to_perm.users_group_to_perm_id'), nullable=False, unique=None, default=None) - user_group_repo_to_perm = relationship('UserGroupRepoToPerm') + user_group_repo_to_perm = relationship('UserGroupRepoToPerm', back_populates='user_group_branch_perms') rule_order = Column('rule_order', Integer(), nullable=False) - _branch_pattern = Column('branch_pattern', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), default=u'*') # glob + _branch_pattern = Column('branch_pattern', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), default='*') # glob _branch_hash = Column('branch_hash', UnicodeText().with_variant(UnicodeText(2048), 'mysql')) - def __str__(self): + def __repr__(self): return f' {self.branch_pattern!r})>' @@ -5510,7 +5549,7 @@ class UserBookmark(Base, BaseModel): return bookmarks.all() - def __str__(self): + def __repr__(self): return f'' @@ -5543,7 +5582,7 @@ class FileStore(Base, BaseModel): hidden = Column('hidden', Boolean(), nullable=False, default=False) user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False) - upload_user = relationship('User', lazy='joined', primaryjoin='User.user_id==FileStore.user_id') + upload_user = relationship('User', lazy='joined', primaryjoin='User.user_id==FileStore.user_id', back_populates='artifacts') file_metadata = relationship('FileStoreMetadata', lazy='joined') @@ -5551,7 +5590,7 @@ class FileStore(Base, BaseModel): scope_user_id = Column( 'scope_user_id', Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None) - user = relationship('User', lazy='joined', primaryjoin='User.user_id==FileStore.scope_user_id') + user = relationship('User', lazy='joined', primaryjoin='User.user_id==FileStore.scope_user_id', back_populates='scope_artifacts') # scope limited to user group, which requester have access to scope_user_group_id = Column( @@ -5664,7 +5703,7 @@ class FileStore(Base, BaseModel): return data - def __str__(self): + def __repr__(self): return f'' @@ -5709,7 +5748,7 @@ class FileStoreMetadata(Base, BaseModel) 'file_store_id', Integer(), ForeignKey('file_store.file_store_id'), nullable=True, unique=None, default=None) - file_store = relationship('FileStore', lazy='joined') + file_store = relationship('FileStore', lazy='joined', viewonly=True) @classmethod def valid_value_type(cls, value): @@ -5781,8 +5820,8 @@ class FileStoreMetadata(Base, BaseModel) return data - def __str__(self): - return '<%s[%s]%s=>%s]>' % (self.__class__.__name__, self.file_store_meta_section, + def __repr__(self): + return '<%s[%s]%s=>%s]>' % (self.cls_name, self.file_store_meta_section, self.file_store_meta_key, self.file_store_meta_value) @@ -5812,7 +5851,7 @@ class DbSession(Base, BaseModel): base_table_args, ) - def __str__(self): + def __repr__(self): return f'' id = Column('id', Integer())