##// END OF EJS Templates
release: Merge default into stable for release preparation
marcink -
r4456:279b3293 merge stable
parent child
Show More
Add another comment
@@ -0,0 +1,74
1 |RCE| 4.20.0 |RNS|
2 ------------------
3
4 Release Date
5 ^^^^^^^^^^^^
6
7 - 2020-07-20
8
9
10 New Features
11 ^^^^^^^^^^^^
12
13 - Comments: users can now edit comments body.
14 Editing is versioned and all older versions are kept for auditing.
15 - Pull requests: changed the order of close-branch after merge,
16 so branch heads are no longer left open after the merge.
17 - Diffs: added diff navigation to improve UX when browsing the full context diffs.
18 - Emails: set the `References` header for threading in emails with different subjects.
19 Only some Email clients supports this.
20 - Emails: added logic to allow overwriting the default email titles via rcextensions.
21 - Markdown: support summary/details tags to allow setting a link with expansion menu.
22 - Integrations: added `store_file` integration. This allows storing
23 selected files from repository on disk on push.
24
25
26 General
27 ^^^^^^^
28
29 - License: individual users can hide license flash messages warning about upcoming
30 license expiration.
31 - Downloads: the default download commit is now the landing revision set in repo settings.
32 - Auth-tokens: expose all roles with explanation to help users understand it better.
33 - Pull requests: make auto generated title for pull requests show also source Ref type
34 eg. branch feature1, instead of just name of the branch.
35 - UI: added secondary action instead of two buttons on files page, and download page.
36 - Emails: reduce excessive warning logs on pre-mailer.
37
38
39 Security
40 ^^^^^^^^
41
42 - Branch permissions: protect from XSS on branch rules forbidden flash message.
43
44
45 Performance
46 ^^^^^^^^^^^
47
48
49
50 Fixes
51 ^^^^^
52
53 - Pull requests: detect missing commits on diffs from new PR ancestor logic. This fixes
54 problem with older PRs opened before 4.19.X that had special ancestor set, which could
55 lead in some cases to crash when viewing older pull requests.
56 - Permissions: fixed a case when a duplicate permission made repository settings active on archived repository.
57 - Permissions: fixed missing user info on global and repository permissions pages.
58 - Permissions: allow users to update settings for repository groups they still own,
59 or have admin perms, when they don't change their name.
60 - Permissions: flush all when running remap and rescan.
61 - Repositories: fixed a bug for repo groups that didn't pre-fill the repo group from GET param.
62 - Repositories: allow updating repository settings for users without
63 store-in-root permissions in case repository name didn't change.
64 - Comments: fixed line display icons.
65 - Summary: fixed summary page total commits count.
66
67
68 Upgrade notes
69 ^^^^^^^^^^^^^
70
71 - Schedule feature update.
72 - On Mercurial repositories we changed the order of commits when the close branch on merge features is used.
73 Before the commits was made after a merge leaving an open head.
74 This backward incompatible change now reverses that order, which is the correct way of doing it.
This diff has been collapsed as it changes many lines, (5625 lines changed) Show them Hide them
@@ -0,0 +1,5625
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2010-2020 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 """
22 Database Models for RhodeCode Enterprise
23 """
24
25 import re
26 import os
27 import time
28 import string
29 import hashlib
30 import logging
31 import datetime
32 import uuid
33 import warnings
34 import ipaddress
35 import functools
36 import traceback
37 import collections
38
39 from sqlalchemy import (
40 or_, and_, not_, func, cast, TypeDecorator, event,
41 Index, Sequence, UniqueConstraint, ForeignKey, CheckConstraint, Column,
42 Boolean, String, Unicode, UnicodeText, DateTime, Integer, LargeBinary,
43 Text, Float, PickleType, BigInteger)
44 from sqlalchemy.sql.expression import true, false, case
45 from sqlalchemy.sql.functions import coalesce, count # pragma: no cover
46 from sqlalchemy.orm import (
47 relationship, joinedload, class_mapper, validates, aliased)
48 from sqlalchemy.ext.declarative import declared_attr
49 from sqlalchemy.ext.hybrid import hybrid_property
50 from sqlalchemy.exc import IntegrityError # pragma: no cover
51 from sqlalchemy.dialects.mysql import LONGTEXT
52 from zope.cachedescriptors.property import Lazy as LazyProperty
53 from pyramid import compat
54 from pyramid.threadlocal import get_current_request
55 from webhelpers2.text import remove_formatting
56
57 from rhodecode.translation import _
58 from rhodecode.lib.vcs import get_vcs_instance, VCSError
59 from rhodecode.lib.vcs.backends.base import EmptyCommit, Reference
60 from rhodecode.lib.utils2 import (
61 str2bool, safe_str, get_commit_safe, safe_unicode, sha1_safe,
62 time_to_datetime, aslist, Optional, safe_int, get_clone_url, AttributeDict,
63 glob2re, StrictAttributeDict, cleaned_uri, datetime_to_time, OrderedDefaultDict)
64 from rhodecode.lib.jsonalchemy import MutationObj, MutationList, JsonType, \
65 JsonRaw
66 from rhodecode.lib.ext_json import json
67 from rhodecode.lib.caching_query import FromCache
68 from rhodecode.lib.encrypt import AESCipher, validate_and_get_enc_data
69 from rhodecode.lib.encrypt2 import Encryptor
70 from rhodecode.lib.exceptions import (
71 ArtifactMetadataDuplicate, ArtifactMetadataBadValueType)
72 from rhodecode.model.meta import Base, Session
73
74 URL_SEP = '/'
75 log = logging.getLogger(__name__)
76
77 # =============================================================================
78 # BASE CLASSES
79 # =============================================================================
80
81 # this is propagated from .ini file rhodecode.encrypted_values.secret or
82 # beaker.session.secret if first is not set.
83 # and initialized at environment.py
84 ENCRYPTION_KEY = None
85
86 # used to sort permissions by types, '#' used here is not allowed to be in
87 # usernames, and it's very early in sorted string.printable table.
88 PERMISSION_TYPE_SORT = {
89 'admin': '####',
90 'write': '###',
91 'read': '##',
92 'none': '#',
93 }
94
95
96 def display_user_sort(obj):
97 """
98 Sort function used to sort permissions in .permissions() function of
99 Repository, RepoGroup, UserGroup. Also it put the default user in front
100 of all other resources
101 """
102
103 if obj.username == User.DEFAULT_USER:
104 return '#####'
105 prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '')
106 return prefix + obj.username
107
108
109 def display_user_group_sort(obj):
110 """
111 Sort function used to sort permissions in .permissions() function of
112 Repository, RepoGroup, UserGroup. Also it put the default user in front
113 of all other resources
114 """
115
116 prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '')
117 return prefix + obj.users_group_name
118
119
120 def _hash_key(k):
121 return sha1_safe(k)
122
123
124 def in_filter_generator(qry, items, limit=500):
125 """
126 Splits IN() into multiple with OR
127 e.g.::
128 cnt = Repository.query().filter(
129 or_(
130 *in_filter_generator(Repository.repo_id, range(100000))
131 )).count()
132 """
133 if not items:
134 # empty list will cause empty query which might cause security issues
135 # this can lead to hidden unpleasant results
136 items = [-1]
137
138 parts = []
139 for chunk in xrange(0, len(items), limit):
140 parts.append(
141 qry.in_(items[chunk: chunk + limit])
142 )
143
144 return parts
145
146
147 base_table_args = {
148 'extend_existing': True,
149 'mysql_engine': 'InnoDB',
150 'mysql_charset': 'utf8',
151 'sqlite_autoincrement': True
152 }
153
154
155 class EncryptedTextValue(TypeDecorator):
156 """
157 Special column for encrypted long text data, use like::
158
159 value = Column("encrypted_value", EncryptedValue(), nullable=False)
160
161 This column is intelligent so if value is in unencrypted form it return
162 unencrypted form, but on save it always encrypts
163 """
164 impl = Text
165
166 def process_bind_param(self, value, dialect):
167 """
168 Setter for storing value
169 """
170 import rhodecode
171 if not value:
172 return value
173
174 # protect against double encrypting if values is already encrypted
175 if value.startswith('enc$aes$') \
176 or value.startswith('enc$aes_hmac$') \
177 or value.startswith('enc2$'):
178 raise ValueError('value needs to be in unencrypted format, '
179 'ie. not starting with enc$ or enc2$')
180
181 algo = rhodecode.CONFIG.get('rhodecode.encrypted_values.algorithm') or 'aes'
182 if algo == 'aes':
183 return 'enc$aes_hmac$%s' % AESCipher(ENCRYPTION_KEY, hmac=True).encrypt(value)
184 elif algo == 'fernet':
185 return Encryptor(ENCRYPTION_KEY).encrypt(value)
186 else:
187 ValueError('Bad encryption algorithm, should be fernet or aes, got: {}'.format(algo))
188
189 def process_result_value(self, value, dialect):
190 """
191 Getter for retrieving value
192 """
193
194 import rhodecode
195 if not value:
196 return value
197
198 algo = rhodecode.CONFIG.get('rhodecode.encrypted_values.algorithm') or 'aes'
199 enc_strict_mode = str2bool(rhodecode.CONFIG.get('rhodecode.encrypted_values.strict') or True)
200 if algo == 'aes':
201 decrypted_data = validate_and_get_enc_data(value, ENCRYPTION_KEY, enc_strict_mode)
202 elif algo == 'fernet':
203 return Encryptor(ENCRYPTION_KEY).decrypt(value)
204 else:
205 ValueError('Bad encryption algorithm, should be fernet or aes, got: {}'.format(algo))
206 return decrypted_data
207
208
209 class BaseModel(object):
210 """
211 Base Model for all classes
212 """
213
214 @classmethod
215 def _get_keys(cls):
216 """return column names for this model """
217 return class_mapper(cls).c.keys()
218
219 def get_dict(self):
220 """
221 return dict with keys and values corresponding
222 to this model data """
223
224 d = {}
225 for k in self._get_keys():
226 d[k] = getattr(self, k)
227
228 # also use __json__() if present to get additional fields
229 _json_attr = getattr(self, '__json__', None)
230 if _json_attr:
231 # update with attributes from __json__
232 if callable(_json_attr):
233 _json_attr = _json_attr()
234 for k, val in _json_attr.iteritems():
235 d[k] = val
236 return d
237
238 def get_appstruct(self):
239 """return list with keys and values tuples corresponding
240 to this model data """
241
242 lst = []
243 for k in self._get_keys():
244 lst.append((k, getattr(self, k),))
245 return lst
246
247 def populate_obj(self, populate_dict):
248 """populate model with data from given populate_dict"""
249
250 for k in self._get_keys():
251 if k in populate_dict:
252 setattr(self, k, populate_dict[k])
253
254 @classmethod
255 def query(cls):
256 return Session().query(cls)
257
258 @classmethod
259 def get(cls, id_):
260 if id_:
261 return cls.query().get(id_)
262
263 @classmethod
264 def get_or_404(cls, id_):
265 from pyramid.httpexceptions import HTTPNotFound
266
267 try:
268 id_ = int(id_)
269 except (TypeError, ValueError):
270 raise HTTPNotFound()
271
272 res = cls.query().get(id_)
273 if not res:
274 raise HTTPNotFound()
275 return res
276
277 @classmethod
278 def getAll(cls):
279 # deprecated and left for backward compatibility
280 return cls.get_all()
281
282 @classmethod
283 def get_all(cls):
284 return cls.query().all()
285
286 @classmethod
287 def delete(cls, id_):
288 obj = cls.query().get(id_)
289 Session().delete(obj)
290
291 @classmethod
292 def identity_cache(cls, session, attr_name, value):
293 exist_in_session = []
294 for (item_cls, pkey), instance in session.identity_map.items():
295 if cls == item_cls and getattr(instance, attr_name) == value:
296 exist_in_session.append(instance)
297 if exist_in_session:
298 if len(exist_in_session) == 1:
299 return exist_in_session[0]
300 log.exception(
301 'multiple objects with attr %s and '
302 'value %s found with same name: %r',
303 attr_name, value, exist_in_session)
304
305 def __repr__(self):
306 if hasattr(self, '__unicode__'):
307 # python repr needs to return str
308 try:
309 return safe_str(self.__unicode__())
310 except UnicodeDecodeError:
311 pass
312 return '<DB:%s>' % (self.__class__.__name__)
313
314
315 class RhodeCodeSetting(Base, BaseModel):
316 __tablename__ = 'rhodecode_settings'
317 __table_args__ = (
318 UniqueConstraint('app_settings_name'),
319 base_table_args
320 )
321
322 SETTINGS_TYPES = {
323 'str': safe_str,
324 'int': safe_int,
325 'unicode': safe_unicode,
326 'bool': str2bool,
327 'list': functools.partial(aslist, sep=',')
328 }
329 DEFAULT_UPDATE_URL = 'https://rhodecode.com/api/v1/info/versions'
330 GLOBAL_CONF_KEY = 'app_settings'
331
332 app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
333 app_settings_name = Column("app_settings_name", String(255), nullable=True, unique=None, default=None)
334 _app_settings_value = Column("app_settings_value", String(4096), nullable=True, unique=None, default=None)
335 _app_settings_type = Column("app_settings_type", String(255), nullable=True, unique=None, default=None)
336
337 def __init__(self, key='', val='', type='unicode'):
338 self.app_settings_name = key
339 self.app_settings_type = type
340 self.app_settings_value = val
341
342 @validates('_app_settings_value')
343 def validate_settings_value(self, key, val):
344 assert type(val) == unicode
345 return val
346
347 @hybrid_property
348 def app_settings_value(self):
349 v = self._app_settings_value
350 _type = self.app_settings_type
351 if _type:
352 _type = self.app_settings_type.split('.')[0]
353 # decode the encrypted value
354 if 'encrypted' in self.app_settings_type:
355 cipher = EncryptedTextValue()
356 v = safe_unicode(cipher.process_result_value(v, None))
357
358 converter = self.SETTINGS_TYPES.get(_type) or \
359 self.SETTINGS_TYPES['unicode']
360 return converter(v)
361
362 @app_settings_value.setter
363 def app_settings_value(self, val):
364 """
365 Setter that will always make sure we use unicode in app_settings_value
366
367 :param val:
368 """
369 val = safe_unicode(val)
370 # encode the encrypted value
371 if 'encrypted' in self.app_settings_type:
372 cipher = EncryptedTextValue()
373 val = safe_unicode(cipher.process_bind_param(val, None))
374 self._app_settings_value = val
375
376 @hybrid_property
377 def app_settings_type(self):
378 return self._app_settings_type
379
380 @app_settings_type.setter
381 def app_settings_type(self, val):
382 if val.split('.')[0] not in self.SETTINGS_TYPES:
383 raise Exception('type must be one of %s got %s'
384 % (self.SETTINGS_TYPES.keys(), val))
385 self._app_settings_type = val
386
387 @classmethod
388 def get_by_prefix(cls, prefix):
389 return RhodeCodeSetting.query()\
390 .filter(RhodeCodeSetting.app_settings_name.startswith(prefix))\
391 .all()
392
393 def __unicode__(self):
394 return u"<%s('%s:%s[%s]')>" % (
395 self.__class__.__name__,
396 self.app_settings_name, self.app_settings_value,
397 self.app_settings_type
398 )
399
400
401 class RhodeCodeUi(Base, BaseModel):
402 __tablename__ = 'rhodecode_ui'
403 __table_args__ = (
404 UniqueConstraint('ui_key'),
405 base_table_args
406 )
407
408 HOOK_REPO_SIZE = 'changegroup.repo_size'
409 # HG
410 HOOK_PRE_PULL = 'preoutgoing.pre_pull'
411 HOOK_PULL = 'outgoing.pull_logger'
412 HOOK_PRE_PUSH = 'prechangegroup.pre_push'
413 HOOK_PRETX_PUSH = 'pretxnchangegroup.pre_push'
414 HOOK_PUSH = 'changegroup.push_logger'
415 HOOK_PUSH_KEY = 'pushkey.key_push'
416
417 HOOKS_BUILTIN = [
418 HOOK_PRE_PULL,
419 HOOK_PULL,
420 HOOK_PRE_PUSH,
421 HOOK_PRETX_PUSH,
422 HOOK_PUSH,
423 HOOK_PUSH_KEY,
424 ]
425
426 # TODO: johbo: Unify way how hooks are configured for git and hg,
427 # git part is currently hardcoded.
428
429 # SVN PATTERNS
430 SVN_BRANCH_ID = 'vcs_svn_branch'
431 SVN_TAG_ID = 'vcs_svn_tag'
432
433 ui_id = Column(
434 "ui_id", Integer(), nullable=False, unique=True, default=None,
435 primary_key=True)
436 ui_section = Column(
437 "ui_section", String(255), nullable=True, unique=None, default=None)
438 ui_key = Column(
439 "ui_key", String(255), nullable=True, unique=None, default=None)
440 ui_value = Column(
441 "ui_value", String(255), nullable=True, unique=None, default=None)
442 ui_active = Column(
443 "ui_active", Boolean(), nullable=True, unique=None, default=True)
444
445 def __repr__(self):
446 return '<%s[%s]%s=>%s]>' % (self.__class__.__name__, self.ui_section,
447 self.ui_key, self.ui_value)
448
449
450 class RepoRhodeCodeSetting(Base, BaseModel):
451 __tablename__ = 'repo_rhodecode_settings'
452 __table_args__ = (
453 UniqueConstraint(
454 'app_settings_name', 'repository_id',
455 name='uq_repo_rhodecode_setting_name_repo_id'),
456 base_table_args
457 )
458
459 repository_id = Column(
460 "repository_id", Integer(), ForeignKey('repositories.repo_id'),
461 nullable=False)
462 app_settings_id = Column(
463 "app_settings_id", Integer(), nullable=False, unique=True,
464 default=None, primary_key=True)
465 app_settings_name = Column(
466 "app_settings_name", String(255), nullable=True, unique=None,
467 default=None)
468 _app_settings_value = Column(
469 "app_settings_value", String(4096), nullable=True, unique=None,
470 default=None)
471 _app_settings_type = Column(
472 "app_settings_type", String(255), nullable=True, unique=None,
473 default=None)
474
475 repository = relationship('Repository')
476
477 def __init__(self, repository_id, key='', val='', type='unicode'):
478 self.repository_id = repository_id
479 self.app_settings_name = key
480 self.app_settings_type = type
481 self.app_settings_value = val
482
483 @validates('_app_settings_value')
484 def validate_settings_value(self, key, val):
485 assert type(val) == unicode
486 return val
487
488 @hybrid_property
489 def app_settings_value(self):
490 v = self._app_settings_value
491 type_ = self.app_settings_type
492 SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES
493 converter = SETTINGS_TYPES.get(type_) or SETTINGS_TYPES['unicode']
494 return converter(v)
495
496 @app_settings_value.setter
497 def app_settings_value(self, val):
498 """
499 Setter that will always make sure we use unicode in app_settings_value
500
501 :param val:
502 """
503 self._app_settings_value = safe_unicode(val)
504
505 @hybrid_property
506 def app_settings_type(self):
507 return self._app_settings_type
508
509 @app_settings_type.setter
510 def app_settings_type(self, val):
511 SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES
512 if val not in SETTINGS_TYPES:
513 raise Exception('type must be one of %s got %s'
514 % (SETTINGS_TYPES.keys(), val))
515 self._app_settings_type = val
516
517 def __unicode__(self):
518 return u"<%s('%s:%s:%s[%s]')>" % (
519 self.__class__.__name__, self.repository.repo_name,
520 self.app_settings_name, self.app_settings_value,
521 self.app_settings_type
522 )
523
524
525 class RepoRhodeCodeUi(Base, BaseModel):
526 __tablename__ = 'repo_rhodecode_ui'
527 __table_args__ = (
528 UniqueConstraint(
529 'repository_id', 'ui_section', 'ui_key',
530 name='uq_repo_rhodecode_ui_repository_id_section_key'),
531 base_table_args
532 )
533
534 repository_id = Column(
535 "repository_id", Integer(), ForeignKey('repositories.repo_id'),
536 nullable=False)
537 ui_id = Column(
538 "ui_id", Integer(), nullable=False, unique=True, default=None,
539 primary_key=True)
540 ui_section = Column(
541 "ui_section", String(255), nullable=True, unique=None, default=None)
542 ui_key = Column(
543 "ui_key", String(255), nullable=True, unique=None, default=None)
544 ui_value = Column(
545 "ui_value", String(255), nullable=True, unique=None, default=None)
546 ui_active = Column(
547 "ui_active", Boolean(), nullable=True, unique=None, default=True)
548
549 repository = relationship('Repository')
550
551 def __repr__(self):
552 return '<%s[%s:%s]%s=>%s]>' % (
553 self.__class__.__name__, self.repository.repo_name,
554 self.ui_section, self.ui_key, self.ui_value)
555
556
557 class User(Base, BaseModel):
558 __tablename__ = 'users'
559 __table_args__ = (
560 UniqueConstraint('username'), UniqueConstraint('email'),
561 Index('u_username_idx', 'username'),
562 Index('u_email_idx', 'email'),
563 base_table_args
564 )
565
566 DEFAULT_USER = 'default'
567 DEFAULT_USER_EMAIL = 'anonymous@rhodecode.org'
568 DEFAULT_GRAVATAR_URL = 'https://secure.gravatar.com/avatar/{md5email}?d=identicon&s={size}'
569
570 user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
571 username = Column("username", String(255), nullable=True, unique=None, default=None)
572 password = Column("password", String(255), nullable=True, unique=None, default=None)
573 active = Column("active", Boolean(), nullable=True, unique=None, default=True)
574 admin = Column("admin", Boolean(), nullable=True, unique=None, default=False)
575 name = Column("firstname", String(255), nullable=True, unique=None, default=None)
576 lastname = Column("lastname", String(255), nullable=True, unique=None, default=None)
577 _email = Column("email", String(255), nullable=True, unique=None, default=None)
578 last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None)
579 last_activity = Column('last_activity', DateTime(timezone=False), nullable=True, unique=None, default=None)
580 description = Column('description', UnicodeText().with_variant(UnicodeText(1024), 'mysql'))
581
582 extern_type = Column("extern_type", String(255), nullable=True, unique=None, default=None)
583 extern_name = Column("extern_name", String(255), nullable=True, unique=None, default=None)
584 _api_key = Column("api_key", String(255), nullable=True, unique=None, default=None)
585 inherit_default_permissions = Column("inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True)
586 created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
587 _user_data = Column("user_data", LargeBinary(), nullable=True) # JSON data
588
589 user_log = relationship('UserLog')
590 user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all, delete-orphan')
591
592 repositories = relationship('Repository')
593 repository_groups = relationship('RepoGroup')
594 user_groups = relationship('UserGroup')
595
596 user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
597 followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all')