release: merge back stable branch into default

docs/release-notes/release-notes-4.24.1.rst

@@ -0,0 +1,46
+|RCE| 4.24.1 |RNS|
+------------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2021-02-04
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+- Core: added statsd client for statistics usage.
+- Clone urls: allow custom clone by id template so users can set clone-by-id as default.
+- Automation: enable check for new version for EE edition as automation task that will send notifications when new RhodeCode version is available
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+- Core: bumped git to 2.30.0
+
+
+Fixes
+^^^^^
+
+- Comments: add ability to resolve todos from the side-bar. This should prevent situations
+  when a TODO was left over in outdated/removed code pieces, and users needs to search to resolve them.
+- Pull requests: fixed a case when template marker was used in description field causing 500 errors on commenting.
+- Merges: fixed excessive data saved in merge metadata that could not fit inside the DB table.
+- Exceptions: fixed problem with exceptions formatting resulting in limited exception data reporting.
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- Un-scheduled release addressing problems in 4.24.X releases.

pkgs/patches/pytest/setuptools.patch

@@ -0,0 +1,12
+diff -rup pytest-4.6.5-orig/setup.py pytest-4.6.5/setup.py
+--- pytest-4.6.5-orig/setup.py	2018-04-10 10:23:04.000000000 +0200
++++  pytest-4.6.5/setup.py	2018-04-10 10:23:34.000000000 +0200
+@@ -24,7 +24,7 @@ INSTALL_REQUIRES = [
+ def main():
+     setup(
+         use_scm_version={"write_to": "src/_pytest/_version.py"},
+-        setup_requires=["setuptools-scm", "setuptools>=40.0"],
++        setup_requires=["setuptools-scm", "setuptools<=42.0"],
+         package_dir={"": "src"},
+         # fmt: off
+         extras_require={
  No newline at end of file

rhodecode/lib/_vendor/statsd/__init__.py

@@ -0,0 +1,46
+from __future__ import absolute_import, division, unicode_literals
+
+import logging
+
+from .stream import TCPStatsClient, UnixSocketStatsClient  # noqa
+from .udp import StatsClient  # noqa
+
+HOST = 'localhost'
+PORT = 8125
+IPV6 = False
+PREFIX = None
+MAXUDPSIZE = 512
+
+log = logging.getLogger('rhodecode.statsd')
+
+
+def statsd_config(config, prefix='statsd.'):
+    _config = {}
+    for key in config.keys():
+        if key.startswith(prefix):
+            _config[key[len(prefix):]] = config[key]
+    return _config
+
+
+def client_from_config(configuration, prefix='statsd.', **kwargs):
+    from pyramid.settings import asbool
+
+    _config = statsd_config(configuration, prefix)
+    statsd_enabled = asbool(_config.pop('enabled', False))
+    if not statsd_enabled:
+        log.debug('statsd client not enabled by statsd.enabled =  flag, skipping...')
+        return
+
+    host = _config.pop('statsd_host', HOST)
+    port = _config.pop('statsd_port', PORT)
+    prefix = _config.pop('statsd_prefix', PREFIX)
+    maxudpsize = _config.pop('statsd_maxudpsize', MAXUDPSIZE)
+    ipv6 = asbool(_config.pop('statsd_ipv6', IPV6))
+    log.debug('configured statsd client %s:%s', host, port)
+
+    return StatsClient(
+        host=host, port=port, prefix=prefix, maxudpsize=maxudpsize, ipv6=ipv6)
+
+
+def get_statsd_client(request):
+    return client_from_config(request.registry.settings)

rhodecode/lib/_vendor/statsd/base.py

@@ -0,0 +1,107
+from __future__ import absolute_import, division, unicode_literals
+
+import random
+from collections import deque
+from datetime import timedelta
+
+from .timer import Timer
+
+
+class StatsClientBase(object):
+    """A Base class for various statsd clients."""
+
+    def close(self):
+        """Used to close and clean up any underlying resources."""
+        raise NotImplementedError()
+
+    def _send(self):
+        raise NotImplementedError()
+
+    def pipeline(self):
+        raise NotImplementedError()
+
+    def timer(self, stat, rate=1):
+        return Timer(self, stat, rate)
+
+    def timing(self, stat, delta, rate=1):
+        """
+        Send new timing information.
+
+        `delta` can be either a number of milliseconds or a timedelta.
+        """
+        if isinstance(delta, timedelta):
+            # Convert timedelta to number of milliseconds.
+            delta = delta.total_seconds() * 1000.
+        self._send_stat(stat, '%0.6f|ms' % delta, rate)
+
+    def incr(self, stat, count=1, rate=1):
+        """Increment a stat by `count`."""
+        self._send_stat(stat, '%s|c' % count, rate)
+
+    def decr(self, stat, count=1, rate=1):
+        """Decrement a stat by `count`."""
+        self.incr(stat, -count, rate)
+
+    def gauge(self, stat, value, rate=1, delta=False):
+        """Set a gauge value."""
+        if value < 0 and not delta:
+            if rate < 1:
+                if random.random() > rate:
+                    return
+            with self.pipeline() as pipe:
+                pipe._send_stat(stat, '0|g', 1)
+                pipe._send_stat(stat, '%s|g' % value, 1)
+        else:
+            prefix = '+' if delta and value >= 0 else ''
+            self._send_stat(stat, '%s%s|g' % (prefix, value), rate)
+
+    def set(self, stat, value, rate=1):
+        """Set a set value."""
+        self._send_stat(stat, '%s|s' % value, rate)
+
+    def _send_stat(self, stat, value, rate):
+        self._after(self._prepare(stat, value, rate))
+
+    def _prepare(self, stat, value, rate):
+        if rate < 1:
+            if random.random() > rate:
+                return
+            value = '%s|@%s' % (value, rate)
+
+        if self._prefix:
+            stat = '%s.%s' % (self._prefix, stat)
+
+        return '%s:%s' % (stat, value)
+
+    def _after(self, data):
+        if data:
+            self._send(data)
+
+
+class PipelineBase(StatsClientBase):
+
+    def __init__(self, client):
+        self._client = client
+        self._prefix = client._prefix
+        self._stats = deque()
+
+    def _send(self):
+        raise NotImplementedError()
+
+    def _after(self, data):
+        if data is not None:
+            self._stats.append(data)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, typ, value, tb):
+        self.send()
+
+    def send(self):
+        if not self._stats:
+            return
+        self._send()
+
+    def pipeline(self):
+        return self.__class__(self)

rhodecode/lib/_vendor/statsd/stream.py

@@ -0,0 +1,75
+from __future__ import absolute_import, division, unicode_literals
+
+import socket
+
+from .base import StatsClientBase, PipelineBase
+
+
+class StreamPipeline(PipelineBase):
+    def _send(self):
+        self._client._after('\n'.join(self._stats))
+        self._stats.clear()
+
+
+class StreamClientBase(StatsClientBase):
+    def connect(self):
+        raise NotImplementedError()
+
+    def close(self):
+        if self._sock and hasattr(self._sock, 'close'):
+            self._sock.close()
+        self._sock = None
+
+    def reconnect(self):
+        self.close()
+        self.connect()
+
+    def pipeline(self):
+        return StreamPipeline(self)
+
+    def _send(self, data):
+        """Send data to statsd."""
+        if not self._sock:
+            self.connect()
+        self._do_send(data)
+
+    def _do_send(self, data):
+        self._sock.sendall(data.encode('ascii') + b'\n')
+
+
+class TCPStatsClient(StreamClientBase):
+    """TCP version of StatsClient."""
+
+    def __init__(self, host='localhost', port=8125, prefix=None,
+                 timeout=None, ipv6=False):
+        """Create a new client."""
+        self._host = host
+        self._port = port
+        self._ipv6 = ipv6
+        self._timeout = timeout
+        self._prefix = prefix
+        self._sock = None
+
+    def connect(self):
+        fam = socket.AF_INET6 if self._ipv6 else socket.AF_INET
+        family, _, _, _, addr = socket.getaddrinfo(
+            self._host, self._port, fam, socket.SOCK_STREAM)[0]
+        self._sock = socket.socket(family, socket.SOCK_STREAM)
+        self._sock.settimeout(self._timeout)
+        self._sock.connect(addr)
+
+
+class UnixSocketStatsClient(StreamClientBase):
+    """Unix domain socket version of StatsClient."""
+
+    def __init__(self, socket_path, prefix=None, timeout=None):
+        """Create a new client."""
+        self._socket_path = socket_path
+        self._timeout = timeout
+        self._prefix = prefix
+        self._sock = None
+
+    def connect(self):
+        self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self._sock.settimeout(self._timeout)
+        self._sock.connect(self._socket_path)

rhodecode/lib/_vendor/statsd/timer.py

@@ -0,0 +1,71
+from __future__ import absolute_import, division, unicode_literals
+
+import functools
+
+# Use timer that's not susceptible to time of day adjustments.
+try:
+    # perf_counter is only present on Py3.3+
+    from time import perf_counter as time_now
+except ImportError:
+    # fall back to using time
+    from time import time as time_now
+
+
+def safe_wraps(wrapper, *args, **kwargs):
+    """Safely wraps partial functions."""
+    while isinstance(wrapper, functools.partial):
+        wrapper = wrapper.func
+    return functools.wraps(wrapper, *args, **kwargs)
+
+
+class Timer(object):
+    """A context manager/decorator for statsd.timing()."""
+
+    def __init__(self, client, stat, rate=1):
+        self.client = client
+        self.stat = stat
+        self.rate = rate
+        self.ms = None
+        self._sent = False
+        self._start_time = None
+
+    def __call__(self, f):
+        """Thread-safe timing function decorator."""
+        @safe_wraps(f)
+        def _wrapped(*args, **kwargs):
+            start_time = time_now()
+            try:
+                return f(*args, **kwargs)
+            finally:
+                elapsed_time_ms = 1000.0 * (time_now() - start_time)
+                self.client.timing(self.stat, elapsed_time_ms, self.rate)
+        return _wrapped
+
+    def __enter__(self):
+        return self.start()
+
+    def __exit__(self, typ, value, tb):
+        self.stop()
+
+    def start(self):
+        self.ms = None
+        self._sent = False
+        self._start_time = time_now()
+        return self
+
+    def stop(self, send=True):
+        if self._start_time is None:
+            raise RuntimeError('Timer has not started.')
+        dt = time_now() - self._start_time
+        self.ms = 1000.0 * dt  # Convert to milliseconds.
+        if send:
+            self.send()
+        return self
+
+    def send(self):
+        if self.ms is None:
+            raise RuntimeError('No data recorded.')
+        if self._sent:
+            raise RuntimeError('Already sent data.')
+        self._sent = True
+        self.client.timing(self.stat, self.ms, self.rate)

rhodecode/lib/_vendor/statsd/udp.py

@@ -0,0 +1,55
+from __future__ import absolute_import, division, unicode_literals
+
+import socket
+
+from .base import StatsClientBase, PipelineBase
+
+
+class Pipeline(PipelineBase):
+
+    def __init__(self, client):
+        super(Pipeline, self).__init__(client)
+        self._maxudpsize = client._maxudpsize
+
+    def _send(self):
+        data = self._stats.popleft()
+        while self._stats:
+            # Use popleft to preserve the order of the stats.
+            stat = self._stats.popleft()
+            if len(stat) + len(data) + 1 >= self._maxudpsize:
+                self._client._after(data)
+                data = stat
+            else:
+                data += '\n' + stat
+        self._client._after(data)
+
+
+class StatsClient(StatsClientBase):
+    """A client for statsd."""
+
+    def __init__(self, host='localhost', port=8125, prefix=None,
+                 maxudpsize=512, ipv6=False):
+        """Create a new client."""
+        fam = socket.AF_INET6 if ipv6 else socket.AF_INET
+        family, _, _, _, addr = socket.getaddrinfo(
+            host, port, fam, socket.SOCK_DGRAM)[0]
+        self._addr = addr
+        self._sock = socket.socket(family, socket.SOCK_DGRAM)
+        self._prefix = prefix
+        self._maxudpsize = maxudpsize
+
+    def _send(self, data):
+        """Send data to statsd."""
+        try:
+            self._sock.sendto(data.encode('ascii'), self._addr)
+        except (socket.error, RuntimeError):
+            # No time for love, Dr. Jones!
+            pass
+
+    def close(self):
+        if self._sock and hasattr(self._sock, 'close'):
+            self._sock.close()
+        self._sock = None
+
+    def pipeline(self):
+        return Pipeline(self)

rhodecode/lib/dbmigrate/versions/113_version_4_24_0.py

@@ -0,0 +1,64
+# -*- coding: utf-8 -*-
+
+import logging
+from sqlalchemy import *
+
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+
+from rhodecode.lib.dbmigrate.versions import _reset_base
+from rhodecode.model import meta, init_model_encryption
+
+
+log = logging.getLogger(__name__)
+
+
+def upgrade(migrate_engine):
+    """
+    Upgrade operations go here.
+    Don't create your own engine; bind migrate_engine to your metadata
+    """
+    _reset_base(migrate_engine)
+    from rhodecode.lib.dbmigrate.schema import db_4_20_0_0 as db
+
+    init_model_encryption(db)
+
+    # issue fixups
+    fixups(db, meta.Session)
+
+
+def downgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+
+def fixups(models, _SESSION):
+    # now create new changed value of clone_url
+    Optional = models.Optional
+
+    def get_by_name(cls, key):
+        return cls.query().filter(cls.app_settings_name == key).scalar()
+
+    def create_or_update(cls, key, val=Optional(''), type_=Optional('unicode')):
+        res = get_by_name(cls, key)
+        if not res:
+            val = Optional.extract(val)
+            type_ = Optional.extract(type_)
+            res = cls(key, val, type_)
+        else:
+            res.app_settings_name = key
+            if not isinstance(val, Optional):
+                # update if set
+                res.app_settings_value = val
+            if not isinstance(type_, Optional):
+                # update if set
+                res.app_settings_type = type_
+        return res
+
+    clone_uri_tmpl = models.Repository.DEFAULT_CLONE_URI_ID
+    print('settings new clone by url template to %s' % clone_uri_tmpl)
+
+    sett = create_or_update(models.RhodeCodeSetting,
+        'clone_uri_id_tmpl', clone_uri_tmpl, 'unicode')
+    _SESSION().add(sett)
+    _SESSION.commit()

rhodecode/templates/email_templates/update_available.mako

@@ -0,0 +1,32
+## -*- coding: utf-8 -*-
+<%inherit file="base.mako"/>
+
+<%def name="subject()" filter="n,trim,whitespace_filter">
+New Version of RhodeCode is available !
+</%def>
+
+## plain text version of the email. Empty by default
+<%def name="body_plaintext()" filter="n,trim">
+A new version of RhodeCode is available!
+
+Your version: ${current_ver}
+New version: ${latest_ver}
+
+Release notes:
+
+https://docs.rhodecode.com/RhodeCode-Enterprise/release-notes/release-notes-${latest_ver}.html
+</%def>
+
+## BODY GOES BELOW
+
+<h3>A new version of RhodeCode is available!</h3>
+<br/>
+Your version: ${current_ver}<br/>
+New version: <strong>${latest_ver}</strong><br/>
+
+<h4>Release notes</h4>
+
+<a href="https://docs.rhodecode.com/RhodeCode-Enterprise/release-notes/release-notes-${latest_ver}.html">
+    https://docs.rhodecode.com/RhodeCode-Enterprise/release-notes/release-notes-${latest_ver}.html
+</a>
+

.hgtags

@@ -73,3 +73,5 90734aac31ee4563bbe665a43ff73190cc762275
 a9655707f7cf4146affc51c12fe5ed8e02898a57 v4.23.0
 56310d93b33b97535908ef9c7b0985b89bb7fad2 v4.23.1
 7637c38528fa38c1eabc1fde6a869c20995a0da7 v4.23.2
+6aeb4ac3ef7f0ac699c914740dad3688c9495e83 v4.24.0
+6eaf953da06e468a4c4e5239d3d0e700bda6b163 v4.24.1

docs/release-notes/release-notes-4.24.0.rst

@@ -1,4 +1,4
-|RCE| 4.23.0 |RNS|
+|RCE| 4.24.0 |RNS|
 ------------------
 
 Release Date
@@ -16,14 +16,16 New Features
   Can be used for backups etc.
 - Pull requests: expose commit versions in the pull-request commit list.
 
+
 General
 ^^^^^^^
 
 - Deps: bumped redis to 3.5.3
-- rcextensions: improve examples
+- Rcextensions: improve examples for some usage.
 - Setup: added optional parameters to apply a default license, or skip re-creation of database at install.
 - Docs: update headers for NGINX
 - Beaker cache: remove no longer used beaker cache init
+- Installation: the installer no longer requires gzip and bzip packages, and works on python 2 and 3
 
 
 Security

docs/release-notes/release-notes.rst

@@ -9,6 +9,7 Release Notes
 .. toctree::
    :maxdepth: 1
 
+   release-notes-4.24.1.rst
    release-notes-4.24.0.rst
    release-notes-4.23.2.rst
    release-notes-4.23.1.rst

pkgs/python-packages-overrides.nix

@@ -274,6 +274,12 self: super: {
     ];
   });
 
+  "pytest" = super."pytest".override (attrs: {
+    patches = [
+      ./patches/pytest/setuptools.patch
+    ];
+  });
+
   # Avoid that base packages screw up the build process
   inherit (basePythonPackages)
     setuptools;

rhodecode/__init__.py

@@ -48,7 +48,7 PYRAMID_SETTINGS = {}
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 112  # defines current db version for migrations
+__dbversion__ = 113  # defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'

rhodecode/apps/admin/views/settings.py

@@ -384,6 +384,7 class AdminSettingsView(BaseAppView):
                 ('markup_renderer', 'rhodecode_markup_renderer', 'unicode'),
                 ('gravatar_url', 'rhodecode_gravatar_url', 'unicode'),
                 ('clone_uri_tmpl', 'rhodecode_clone_uri_tmpl', 'unicode'),
+                ('clone_uri_id_tmpl', 'rhodecode_clone_uri_id_tmpl', 'unicode'),
                 ('clone_uri_ssh_tmpl', 'rhodecode_clone_uri_ssh_tmpl', 'unicode'),
                 ('support_url', 'rhodecode_support_url', 'unicode'),
                 ('show_revision_number', 'rhodecode_show_revision_number', 'bool'),

rhodecode/apps/debug_style/views.py

@@ -102,6 +102,11 Check if we should use full-topic or min
                 'date': datetime.datetime.now(),
             },
 
+            'update_available': {
+                'current_ver': '4.23.0',
+                'latest_ver': '4.24.0',
+            },
+
             'exception': {
                 'email_prefix': '[RHODECODE ERROR]',
                 'exc_id': exc_traceback['exc_id'],

rhodecode/apps/repository/tests/test_repo_pullrequests.py

@@ -420,6 +420,27 class TestPullrequestsView(object):
         assert pull_request.title == 'New title'
         assert pull_request.description == 'New description'
 
+    def test_edit_title_description(self, pr_util, csrf_token):
+        pull_request = pr_util.create_pull_request()
+        pull_request_id = pull_request.pull_request_id
+
+        response = self.app.post(
+            route_path('pullrequest_update',
+                repo_name=pull_request.target_repo.repo_name,
+                pull_request_id=pull_request_id),
+            params={
+                'edit_pull_request': 'true',
+                'title': 'New title {} {2} {foo}',
+                'description': 'New description',
+                'csrf_token': csrf_token})
+
+        assert_session_flash(
+            response, u'Pull request title & description updated.',
+            category='success')
+
+        pull_request = PullRequest.get(pull_request_id)
+        assert pull_request.title_safe == 'New title {{}} {{2}} {{foo}}'
+
     def test_edit_title_description_closed(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id

rhodecode/apps/repository/views/repo_summary.py

@@ -83,14 +83,10 class RepoSummaryView(RepoAppView):
         if self._rhodecode_user.username != User.DEFAULT_USER:
             username = safe_str(self._rhodecode_user.username)
 
-        _def_clone_uri = _def_clone_uri_id = c.clone_uri_tmpl
+        _def_clone_uri = c.clone_uri_tmpl
+        _def_clone_uri_id = c.clone_uri_id_tmpl
         _def_clone_uri_ssh = c.clone_uri_ssh_tmpl
 
-        if '{repo}' in _def_clone_uri:
-            _def_clone_uri_id = _def_clone_uri.replace('{repo}', '_{repoid}')
-        elif '{repoid}' in _def_clone_uri:
-            _def_clone_uri_id = _def_clone_uri.replace('_{repoid}', '{repo}')
-
         c.clone_repo_url = self.db_repo.clone_url(
             user=username, uri_tmpl=_def_clone_uri)
         c.clone_repo_url_id = self.db_repo.clone_url(

rhodecode/config/middleware.py

@@ -340,6 +340,10 def includeme(config, auth_resources=Non
         'rhodecode.lib.request_counter.get_request_counter',
         'request_count')
 
+    config.add_request_method(
+        'rhodecode.lib._vendor.statsd.get_statsd_client',
+        'statsd', reify=True)
+
     # Set the authorization policy.
     authz_policy = ACLAuthorizationPolicy()
     config.set_authorization_policy(authz_policy)

rhodecode/lib/base.py

@@ -342,6 +342,7 def attach_context_attributes(context, r
     if request.GET.get('default_encoding'):
         context.default_encodings.insert(0, request.GET.get('default_encoding'))
     context.clone_uri_tmpl = rc_config.get('rhodecode_clone_uri_tmpl')
+    context.clone_uri_id_tmpl = rc_config.get('rhodecode_clone_uri_id_tmpl')
     context.clone_uri_ssh_tmpl = rc_config.get('rhodecode_clone_uri_ssh_tmpl')
 
     # INI stored

rhodecode/lib/celerylib/tasks.py

@@ -33,9 +33,9 from email.utils import formatdate
 
 import rhodecode
 from rhodecode.lib import audit_logger
-from rhodecode.lib.celerylib import get_logger, async_task, RequestContextTask
+from rhodecode.lib.celerylib import get_logger, async_task, RequestContextTask, run_task
 from rhodecode.lib import hooks_base
-from rhodecode.lib.utils2 import safe_int, str2bool
+from rhodecode.lib.utils2 import safe_int, str2bool, aslist
 from rhodecode.model.db import (
     Session, IntegrityError, true, Repository, RepoGroup, User)
 
@@ -338,15 +338,39 def repo_maintenance(repoid):
 
 
 @async_task(ignore_result=True)
-def check_for_update():
+def check_for_update(send_email_notification=True, email_recipients=None):
     from rhodecode.model.update import UpdateModel
+    from rhodecode.model.notification import EmailNotificationModel
+
+    log = get_logger(check_for_update)
     update_url = UpdateModel().get_update_url()
     cur_ver = rhodecode.__version__
 
     try:
         data = UpdateModel().get_update_data(update_url)
-        latest = data['versions'][0]
-        UpdateModel().store_version(latest['version'])
+
+        current_ver = UpdateModel().get_stored_version(fallback=cur_ver)
+        latest_ver = data['versions'][0]['version']
+        UpdateModel().store_version(latest_ver)
+
+        if send_email_notification:
+            log.debug('Send email notification is enabled. '
+                      'Current RhodeCode version: %s, latest known: %s', current_ver, latest_ver)
+            if UpdateModel().is_outdated(current_ver, latest_ver):
+
+                email_kwargs = {
+                    'current_ver': current_ver,
+                    'latest_ver': latest_ver,
+                }
+
+                (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
+                    EmailNotificationModel.TYPE_UPDATE_AVAILABLE, **email_kwargs)
+
+                email_recipients = aslist(email_recipients, sep=',') or \
+