artifacts: added DB store models.

rhodecode/lib/dbmigrate/schema/db_4_16_0_2.py

@@ -0,0 +1,4933
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2019 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+"""
+Database Models for RhodeCode Enterprise
+"""
+
+import re
+import os
+import time
+import hashlib
+import logging
+import datetime
+import warnings
+import ipaddress
+import functools
+import traceback
+import collections
+
+from sqlalchemy import (
+    or_, and_, not_, func, TypeDecorator, event,
+    Index, Sequence, UniqueConstraint, ForeignKey, CheckConstraint, Column,
+    Boolean, String, Unicode, UnicodeText, DateTime, Integer, LargeBinary,
+    Text, Float, PickleType)
+from sqlalchemy.sql.expression import true, false
+from sqlalchemy.sql.functions import coalesce, count  # pragma: no cover
+from sqlalchemy.orm import (
+    relationship, joinedload, class_mapper, validates, aliased)
+from sqlalchemy.ext.declarative import declared_attr
+from sqlalchemy.ext.hybrid import hybrid_property
+from sqlalchemy.exc import IntegrityError  # pragma: no cover
+from sqlalchemy.dialects.mysql import LONGTEXT
+from zope.cachedescriptors.property import Lazy as LazyProperty
+from pyramid import compat
+from pyramid.threadlocal import get_current_request
+
+from rhodecode.translation import _
+from rhodecode.lib.vcs import get_vcs_instance
+from rhodecode.lib.vcs.backends.base import EmptyCommit, Reference
+from rhodecode.lib.utils2 import (
+    str2bool, safe_str, get_commit_safe, safe_unicode, sha1_safe,
+    time_to_datetime, aslist, Optional, safe_int, get_clone_url, AttributeDict,
+    glob2re, StrictAttributeDict, cleaned_uri)
+from rhodecode.lib.jsonalchemy import MutationObj, MutationList, JsonType, \
+    JsonRaw
+from rhodecode.lib.ext_json import json
+from rhodecode.lib.caching_query import FromCache
+from rhodecode.lib.encrypt import AESCipher
+
+from rhodecode.model.meta import Base, Session
+
+URL_SEP = '/'
+log = logging.getLogger(__name__)
+
+# =============================================================================
+# BASE CLASSES
+# =============================================================================
+
+# this is propagated from .ini file rhodecode.encrypted_values.secret or
+# beaker.session.secret if first is not set.
+# and initialized at environment.py
+ENCRYPTION_KEY = None
+
+# used to sort permissions by types, '#' used here is not allowed to be in
+# usernames, and it's very early in sorted string.printable table.
+PERMISSION_TYPE_SORT = {
+    'admin': '####',
+    'write': '###',
+    'read':  '##',
+    'none':  '#',
+}
+
+
+def display_user_sort(obj):
+    """
+    Sort function used to sort permissions in .permissions() function of
+    Repository, RepoGroup, UserGroup. Also it put the default user in front
+    of all other resources
+    """
+
+    if obj.username == User.DEFAULT_USER:
+        return '#####'
+    prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '')
+    return prefix + obj.username
+
+
+def display_user_group_sort(obj):
+    """
+    Sort function used to sort permissions in .permissions() function of
+    Repository, RepoGroup, UserGroup. Also it put the default user in front
+    of all other resources
+    """
+
+    prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '')
+    return prefix + obj.users_group_name
+
+
+def _hash_key(k):
+    return sha1_safe(k)
+
+
+def in_filter_generator(qry, items, limit=500):
+    """
+    Splits IN() into multiple with OR
+    e.g.::
+    cnt = Repository.query().filter(
+        or_(
+            *in_filter_generator(Repository.repo_id, range(100000))
+        )).count()
+    """
+    if not items:
+        # empty list will cause empty query which might cause security issues
+        # this can lead to hidden unpleasant results
+        items = [-1]
+
+    parts = []
+    for chunk in xrange(0, len(items), limit):
+        parts.append(
+            qry.in_(items[chunk: chunk + limit])
+        )
+
+    return parts
+
+
+base_table_args = {
+    'extend_existing': True,
+    'mysql_engine': 'InnoDB',
+    'mysql_charset': 'utf8',
+    'sqlite_autoincrement': True
+}
+
+
+class EncryptedTextValue(TypeDecorator):
+    """
+    Special column for encrypted long text data, use like::
+
+        value = Column("encrypted_value", EncryptedValue(), nullable=False)
+
+    This column is intelligent so if value is in unencrypted form it return
+    unencrypted form, but on save it always encrypts
+    """
+    impl = Text
+
+    def process_bind_param(self, value, dialect):
+        if not value:
+            return value
+        if value.startswith('enc$aes$') or value.startswith('enc$aes_hmac$'):
+            # protect against double encrypting if someone manually starts
+            # doing
+            raise ValueError('value needs to be in unencrypted format, ie. '
+                             'not starting with enc$aes')
+        return 'enc$aes_hmac$%s' % AESCipher(
+            ENCRYPTION_KEY, hmac=True).encrypt(value)
+
+    def process_result_value(self, value, dialect):
+        import rhodecode
+
+        if not value:
+            return value
+
+        parts = value.split('$', 3)
+        if not len(parts) == 3:
+            # probably not encrypted values
+            return value
+        else:
+            if parts[0] != 'enc':
+                # parts ok but without our header ?
+                return value
+            enc_strict_mode = str2bool(rhodecode.CONFIG.get(
+                'rhodecode.encrypted_values.strict') or True)
+            # at that stage we know it's our encryption
+            if parts[1] == 'aes':
+                decrypted_data = AESCipher(ENCRYPTION_KEY).decrypt(parts[2])
+            elif parts[1] == 'aes_hmac':
+                decrypted_data = AESCipher(
+                    ENCRYPTION_KEY, hmac=True,
+                    strict_verification=enc_strict_mode).decrypt(parts[2])
+            else:
+                raise ValueError(
+                    'Encryption type part is wrong, must be `aes` '
+                    'or `aes_hmac`, got `%s` instead' % (parts[1]))
+            return decrypted_data
+
+
+class BaseModel(object):
+    """
+    Base Model for all classes
+    """
+
+    @classmethod
+    def _get_keys(cls):
+        """return column names for this model """
+        return class_mapper(cls).c.keys()
+
+    def get_dict(self):
+        """
+        return dict with keys and values corresponding
+        to this model data """
+
+        d = {}
+        for k in self._get_keys():
+            d[k] = getattr(self, k)
+
+        # also use __json__() if present to get additional fields
+        _json_attr = getattr(self, '__json__', None)
+        if _json_attr:
+            # update with attributes from __json__
+            if callable(_json_attr):
+                _json_attr = _json_attr()
+            for k, val in _json_attr.iteritems():
+                d[k] = val
+        return d
+
+    def get_appstruct(self):
+        """return list with keys and values tuples corresponding
+        to this model data """
+
+        lst = []
+        for k in self._get_keys():
+            lst.append((k, getattr(self, k),))
+        return lst
+
+    def populate_obj(self, populate_dict):
+        """populate model with data from given populate_dict"""
+
+        for k in self._get_keys():
+            if k in populate_dict:
+                setattr(self, k, populate_dict[k])
+
+    @classmethod
+    def query(cls):
+        return Session().query(cls)
+
+    @classmethod
+    def get(cls, id_):
+        if id_:
+            return cls.query().get(id_)
+
+    @classmethod
+    def get_or_404(cls, id_):
+        from pyramid.httpexceptions import HTTPNotFound
+
+        try:
+            id_ = int(id_)
+        except (TypeError, ValueError):
+            raise HTTPNotFound()
+
+        res = cls.query().get(id_)
+        if not res:
+            raise HTTPNotFound()
+        return res
+
+    @classmethod
+    def getAll(cls):
+        # deprecated and left for backward compatibility
+        return cls.get_all()
+
+    @classmethod
+    def get_all(cls):
+        return cls.query().all()
+
+    @classmethod
+    def delete(cls, id_):
+        obj = cls.query().get(id_)
+        Session().delete(obj)
+
+    @classmethod
+    def identity_cache(cls, session, attr_name, value):
+        exist_in_session = []
+        for (item_cls, pkey), instance in session.identity_map.items():
+            if cls == item_cls and getattr(instance, attr_name) == value:
+                exist_in_session.append(instance)
+        if exist_in_session:
+            if len(exist_in_session) == 1:
+                return exist_in_session[0]
+            log.exception(
+                'multiple objects with attr %s and '
+                'value %s found with same name: %r',
+                attr_name, value, exist_in_session)
+
+    def __repr__(self):
+        if hasattr(self, '__unicode__'):
+            # python repr needs to return str
+            try:
+                return safe_str(self.__unicode__())
+            except UnicodeDecodeError:
+                pass
+        return '<DB:%s>' % (self.__class__.__name__)
+
+
+class RhodeCodeSetting(Base, BaseModel):
+    __tablename__ = 'rhodecode_settings'
+    __table_args__ = (
+        UniqueConstraint('app_settings_name'),
+        base_table_args
+    )
+
+    SETTINGS_TYPES = {
+        'str': safe_str,
+        'int': safe_int,
+        'unicode': safe_unicode,
+        'bool': str2bool,
+        'list': functools.partial(aslist, sep=',')
+    }
+    DEFAULT_UPDATE_URL = 'https://rhodecode.com/api/v1/info/versions'
+    GLOBAL_CONF_KEY = 'app_settings'
+
+    app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    app_settings_name = Column("app_settings_name", String(255), nullable=True, unique=None, default=None)
+    _app_settings_value = Column("app_settings_value", String(4096), nullable=True, unique=None, default=None)
+    _app_settings_type = Column("app_settings_type", String(255), nullable=True, unique=None, default=None)
+
+    def __init__(self, key='', val='', type='unicode'):
+        self.app_settings_name = key
+        self.app_settings_type = type
+        self.app_settings_value = val
+
+    @validates('_app_settings_value')
+    def validate_settings_value(self, key, val):
+        assert type(val) == unicode
+        return val
+
+    @hybrid_property
+    def app_settings_value(self):
+        v = self._app_settings_value
+        _type = self.app_settings_type
+        if _type:
+            _type = self.app_settings_type.split('.')[0]
+            # decode the encrypted value
+            if 'encrypted' in self.app_settings_type:
+                cipher = EncryptedTextValue()
+                v = safe_unicode(cipher.process_result_value(v, None))
+
+        converter = self.SETTINGS_TYPES.get(_type) or \
+            self.SETTINGS_TYPES['unicode']
+        return converter(v)
+
+    @app_settings_value.setter
+    def app_settings_value(self, val):
+        """
+        Setter that will always make sure we use unicode in app_settings_value
+
+        :param val:
+        """
+        val = safe_unicode(val)
+        # encode the encrypted value
+        if 'encrypted' in self.app_settings_type:
+            cipher = EncryptedTextValue()
+            val = safe_unicode(cipher.process_bind_param(val, None))
+        self._app_settings_value = val
+
+    @hybrid_property
+    def app_settings_type(self):
+        return self._app_settings_type
+
+    @app_settings_type.setter
+    def app_settings_type(self, val):
+        if val.split('.')[0] not in self.SETTINGS_TYPES:
+            raise Exception('type must be one of %s got %s'
+                            % (self.SETTINGS_TYPES.keys(), val))
+        self._app_settings_type = val
+
+    @classmethod
+    def get_by_prefix(cls, prefix):
+        return RhodeCodeSetting.query()\
+            .filter(RhodeCodeSetting.app_settings_name.startswith(prefix))\
+            .all()
+
+    def __unicode__(self):
+        return u"<%s('%s:%s[%s]')>" % (
+            self.__class__.__name__,
+            self.app_settings_name, self.app_settings_value,
+            self.app_settings_type
+        )
+
+
+class RhodeCodeUi(Base, BaseModel):
+    __tablename__ = 'rhodecode_ui'
+    __table_args__ = (
+        UniqueConstraint('ui_key'),
+        base_table_args
+    )
+
+    HOOK_REPO_SIZE = 'changegroup.repo_size'
+    # HG
+    HOOK_PRE_PULL = 'preoutgoing.pre_pull'
+    HOOK_PULL = 'outgoing.pull_logger'
+    HOOK_PRE_PUSH = 'prechangegroup.pre_push'
+    HOOK_PRETX_PUSH = 'pretxnchangegroup.pre_push'
+    HOOK_PUSH = 'changegroup.push_logger'
+    HOOK_PUSH_KEY = 'pushkey.key_push'
+
+    # TODO: johbo: Unify way how hooks are configured for git and hg,
+    # git part is currently hardcoded.
+
+    # SVN PATTERNS
+    SVN_BRANCH_ID = 'vcs_svn_branch'
+    SVN_TAG_ID = 'vcs_svn_tag'
+
+    ui_id = Column(
+        "ui_id", Integer(), nullable=False, unique=True, default=None,
+        primary_key=True)
+    ui_section = Column(
+        "ui_section", String(255), nullable=True, unique=None, default=None)
+    ui_key = Column(
+        "ui_key", String(255), nullable=True, unique=None, default=None)
+    ui_value = Column(
+        "ui_value", String(255), nullable=True, unique=None, default=None)
+    ui_active = Column(
+        "ui_active", Boolean(), nullable=True, unique=None, default=True)
+
+    def __repr__(self):
+        return '<%s[%s]%s=>%s]>' % (self.__class__.__name__, self.ui_section,
+                                    self.ui_key, self.ui_value)
+
+
+class RepoRhodeCodeSetting(Base, BaseModel):
+    __tablename__ = 'repo_rhodecode_settings'
+    __table_args__ = (
+        UniqueConstraint(
+            'app_settings_name', 'repository_id',
+            name='uq_repo_rhodecode_setting_name_repo_id'),
+        base_table_args
+    )
+
+    repository_id = Column(
+        "repository_id", Integer(), ForeignKey('repositories.repo_id'),
+        nullable=False)
+    app_settings_id = Column(
+        "app_settings_id", Integer(), nullable=False, unique=True,
+        default=None, primary_key=True)
+    app_settings_name = Column(
+        "app_settings_name", String(255), nullable=True, unique=None,
+        default=None)
+    _app_settings_value = Column(
+        "app_settings_value", String(4096), nullable=True, unique=None,
+        default=None)
+    _app_settings_type = Column(
+        "app_settings_type", String(255), nullable=True, unique=None,
+        default=None)
+
+    repository = relationship('Repository')
+
+    def __init__(self, repository_id, key='', val='', type='unicode'):
+        self.repository_id = repository_id
+        self.app_settings_name = key
+        self.app_settings_type = type
+        self.app_settings_value = val
+
+    @validates('_app_settings_value')
+    def validate_settings_value(self, key, val):
+        assert type(val) == unicode
+        return val
+
+    @hybrid_property
+    def app_settings_value(self):
+        v = self._app_settings_value
+        type_ = self.app_settings_type
+        SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES
+        converter = SETTINGS_TYPES.get(type_) or SETTINGS_TYPES['unicode']
+        return converter(v)
+
+    @app_settings_value.setter
+    def app_settings_value(self, val):
+        """
+        Setter that will always make sure we use unicode in app_settings_value
+
+        :param val:
+        """
+        self._app_settings_value = safe_unicode(val)
+
+    @hybrid_property
+    def app_settings_type(self):
+        return self._app_settings_type
+
+    @app_settings_type.setter
+    def app_settings_type(self, val):
+        SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES
+        if val not in SETTINGS_TYPES:
+            raise Exception('type must be one of %s got %s'
+                            % (SETTINGS_TYPES.keys(), val))
+        self._app_settings_type = val
+
+    def __unicode__(self):
+        return u"<%s('%s:%s:%s[%s]')>" % (
+            self.__class__.__name__, self.repository.repo_name,
+            self.app_settings_name, self.app_settings_value,
+            self.app_settings_type
+        )
+
+
+class RepoRhodeCodeUi(Base, BaseModel):
+    __tablename__ = 'repo_rhodecode_ui'
+    __table_args__ = (
+        UniqueConstraint(
+            'repository_id', 'ui_section', 'ui_key',
+            name='uq_repo_rhodecode_ui_repository_id_section_key'),
+        base_table_args
+    )
+
+    repository_id = Column(
+        "repository_id", Integer(), ForeignKey('repositories.repo_id'),
+        nullable=False)
+    ui_id = Column(
+        "ui_id", Integer(), nullable=False, unique=True, default=None,
+        primary_key=True)
+    ui_section = Column(
+        "ui_section", String(255), nullable=True, unique=None, default=None)
+    ui_key = Column(
+        "ui_key", String(255), nullable=True, unique=None, default=None)
+    ui_value = Column(
+        "ui_value", String(255), nullable=True, unique=None, default=None)
+    ui_active = Column(
+        "ui_active", Boolean(), nullable=True, unique=None, default=True)
+
+    repository = relationship('Repository')
+
+    def __repr__(self):
+        return '<%s[%s:%s]%s=>%s]>' % (
+            self.__class__.__name__, self.repository.repo_name,
+            self.ui_section, self.ui_key, self.ui_value)
+
+
+class User(Base, BaseModel):
+    __tablename__ = 'users'
+    __table_args__ = (
+        UniqueConstraint('username'), UniqueConstraint('email'),
+        Index('u_username_idx', 'username'),
+        Index('u_email_idx', 'email'),
+        base_table_args
+    )
+
+    DEFAULT_USER = 'default'
+    DEFAULT_USER_EMAIL = '[e-mail]'
+    DEFAULT_GRAVATAR_URL = 'https://secure.gravatar.com/avatar/{md5email}?d=identicon&s={size}'
+
+    user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
+    username = Column("username", String(255), nullable=True, unique=None, default=None)
+    password = Column("password", String(255), nullable=True, unique=None, default=None)
+    active = Column("active", Boolean(), nullable=True, unique=None, default=True)
+    admin = Column("admin", Boolean(), nullable=True, unique=None, default=False)
+    name = Column("firstname", String(255), nullable=True, unique=None, default=None)
+    lastname = Column("lastname", String(255), nullable=True, unique=None, default=None)
+    _email = Column("email", String(255), nullable=True, unique=None, default=None)
+    last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None)
+    last_activity = Column('last_activity', DateTime(timezone=False), nullable=True, unique=None, default=None)
+
+    extern_type = Column("extern_type", String(255), nullable=True, unique=None, default=None)
+    extern_name = Column("extern_name", String(255), nullable=True, unique=None, default=None)
+    _api_key = Column("api_key", String(255), nullable=True, unique=None, default=None)
+    inherit_default_permissions = Column("inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True)
+    created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
+    _user_data = Column("user_data", LargeBinary(), nullable=True)  # JSON data
+
+    user_log = relationship('UserLog')
+    user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
+
+    repositories = relationship('Repository')
+    repository_groups = relationship('RepoGroup')
+    user_groups = relationship('UserGroup')
+
+    user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
+    followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all')
+
+    repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all')
+    repo_group_to_perm = relationship('UserRepoGroupToPerm', primaryjoin='UserRepoGroupToPerm.user_id==User.user_id', cascade='all')
+    user_group_to_perm = relationship('UserUserGroupToPerm', primaryjoin='UserUserGroupToPerm.user_id==User.user_id', cascade='all')
+
+    group_member = relationship('UserGroupMember', cascade='all')
+
+    notifications = relationship('UserNotification', cascade='all')
+    # notifications assigned to this user
+    user_created_notifications = relationship('Notification', cascade='all')
+    # comments created by this user
+    user_comments = relationship('ChangesetComment', cascade='all')
+    # user profile extra info
+    user_emails = relationship('UserEmailMap', cascade='all')
+    user_ip_map = relationship('UserIpMap', cascade='all')
+    user_auth_tokens = relationship('UserApiKeys', cascade='all')
+    user_ssh_keys = relationship('UserSshKeys', cascade='all')
+
+    # gists
+    user_gists = relationship('Gist', cascade='all')
+    # user pull requests
+    user_pull_requests = relationship('PullRequest', cascade='all')
+    # external identities
+    extenal_identities = relationship(
+        'ExternalIdentity',
+        primaryjoin="User.user_id==ExternalIdentity.local_user_id",
+        cascade='all')
+    # review rules
+    user_review_rules = relationship('RepoReviewRuleUser', cascade='all')
+
+    def __unicode__(self):
+        return u"<%s('id:%s:%s')>" % (self.__class__.__name__,
+                                      self.user_id, self.username)
+
+    @hybrid_property
+    def email(self):
+        return self._email
+
+    @email.setter
+    def email(self, val):
+        self._email = val.lower() if val else None
+
+    @hybrid_property
+    def first_name(self):
+        from rhodecode.lib import helpers as h
+        if self.name:
+            return h.escape(self.name)
+        return self.name
+
+    @hybrid_property
+    def last_name(self):
+        from rhodecode.lib import helpers as h
+        if self.lastname:
+            return h.escape(self.lastname)
+        return self.lastname
+
+    @hybrid_property
+    def api_key(self):
+        """
+        Fetch if exist an auth-token with role ALL connected to this user
+        """
+        user_auth_token = UserApiKeys.query()\
+            .filter(UserApiKeys.user_id == self.user_id)\
+            .filter(or_(UserApiKeys.expires == -1,
+                            UserApiKeys.expires >= time.time()))\
+            .filter(UserApiKeys.role == UserApiKeys.ROLE_ALL).first()
+        if user_auth_token:
+            user_auth_token = user_auth_token.api_key
+
+        return user_auth_token
+
+    @api_key.setter
+    def api_key(self, val):
+        # don't allow to set API key this is deprecated for now
+        self._api_key = None
+
+    @property
+    def reviewer_pull_requests(self):
+        return PullRequestReviewers.query() \
+            .options(joinedload(PullRequestReviewers.pull_request)) \
+            .filter(PullRequestReviewers.user_id == self.user_id) \
+            .all()
+
+    @property
+    def firstname(self):
+        # alias for future
+        return self.name
+
+    @property
+    def emails(self):
+        other = UserEmailMap.query()\
+            .filter(UserEmailMap.user == self) \
+            .order_by(UserEmailMap.email_id.asc()) \
+            .all()
+        return [self.email] + [x.email for x in other]
+
+    @property
+    def auth_tokens(self):
+        auth_tokens = self.get_auth_tokens()
+        return [x.api_key for x in auth_tokens]
+
+    def get_auth_tokens(self):
+        return UserApiKeys.query()\
+            .filter(UserApiKeys.user == self)\
+            .order_by(UserApiKeys.user_api_key_id.asc())\
+            .all()
+
+    @LazyProperty
+    def feed_token(self):
+        return self.get_feed_token()
+
+    def get_feed_token(self, cache=True):