release: merged default into stable

docs/release-notes/release-notes-4.13.1.rst

@@ -0,0 +1,43 b''
+|RCE| 4.13.1 |RNS|
+------------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2018-08-06
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+- core: added option to prefix cache keys for usage in cluster.
+- exception-tracker: store event sending exception for easier event fail debugging.
+- maintenance: add repack and fsck for git maintenance execution list.
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+
+
+Fixes
+^^^^^
+
+- caches: use single default cache dir for all backends.
+- caches: don't use lower in cache settings to support uppercase PATHS
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- Unscheduled release addressing reported problems, and improving stability.

docs/release-notes/release-notes.rst

@@ -9,6 +9,7 b' Release Notes'
 .. toctree::
    :maxdepth: 1
 
+   release-notes-4.13.1.rst
    release-notes-4.13.0.rst
    release-notes-4.12.4.rst
    release-notes-4.12.3.rst

pkgs/python-packages.nix

@@ -1653,6 +1653,8 b' self: super: {'
       self."setuptools-scm"
       self."amqp"
       self."authomatic"
+      self."atomicwrites"
+      self."attrs"
       self."babel"
       self."beaker"
       self."celery"

requirements.txt

@@ -4,6 +4,8 b' setuptools-scm==2.1.0'
 amqp==2.3.1
 # not released authomatic that has updated some oauth providers
 https://code.rhodecode.com/upstream/authomatic/archive/90a9ce60cc405ae8a2bf5c3713acd5d78579a04e.tar.gz?md5=3c68720a1322b25254009518d1ff6801#egg=authomatic==0.1.0.post1
+atomicwrites==1.1.5
+attrs==18.1.0
 babel==1.3
 beaker==1.9.1
 celery==4.1.1

rhodecode/apps/repository/views/repo_maintainance.py

@@ -32,8 +32,6 b' log = logging.getLogger(__name__)'
 class RepoMaintenanceView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-
-
         return c
 
     @LoginRequired()

rhodecode/config/middleware.py

@@ -436,13 +436,23 b' def _sanitize_vcs_settings(settings):'
 
 
 def _sanitize_cache_settings(settings):
-    _string_setting(settings, 'cache_dir',
-                    os.path.join(tempfile.gettempdir(), 'rc_cache'))
+    default_cache_dir = os.path.join(tempfile.gettempdir(), 'rc_cache')
+
+    # save default, cache dir, and use it for all backends later.
+    default_cache_dir = _string_setting(
+        settings,
+        'cache_dir',
+        default_cache_dir, lower=False, default_when_empty=True)
+
+    # ensure we have our dir created
+    if not os.path.isdir(default_cache_dir):
+        os.makedirs(default_cache_dir, mode=0755)
+
     # cache_perms
     _string_setting(
         settings,
         'rc_cache.cache_perms.backend',
-        'dogpile.cache.rc.file_namespace')
+        'dogpile.cache.rc.file_namespace', lower=False)
     _int_setting(
         settings,
         'rc_cache.cache_perms.expiration_time',
@@ -450,13 +460,13 b' def _sanitize_cache_settings(settings):'
     _string_setting(
         settings,
         'rc_cache.cache_perms.arguments.filename',
-        os.path.join(tempfile.gettempdir(), 'rc_cache_1'))
+        os.path.join(default_cache_dir, 'rc_cache_1'), lower=False)
 
     # cache_repo
     _string_setting(
         settings,
         'rc_cache.cache_repo.backend',
-        'dogpile.cache.rc.file_namespace')
+        'dogpile.cache.rc.file_namespace', lower=False)
     _int_setting(
         settings,
         'rc_cache.cache_repo.expiration_time',
@@ -464,13 +474,13 b' def _sanitize_cache_settings(settings):'
     _string_setting(
         settings,
         'rc_cache.cache_repo.arguments.filename',
-        os.path.join(tempfile.gettempdir(), 'rc_cache_2'))
+        os.path.join(default_cache_dir, 'rc_cache_2'), lower=False)
 
     # cache_license
     _string_setting(
         settings,
         'rc_cache.cache_license.backend',
-        'dogpile.cache.rc.file_namespace')
+        'dogpile.cache.rc.file_namespace', lower=False)
     _int_setting(
         settings,
         'rc_cache.cache_license.expiration_time',
@@ -478,13 +488,13 b' def _sanitize_cache_settings(settings):'
     _string_setting(
         settings,
         'rc_cache.cache_license.arguments.filename',
-        os.path.join(tempfile.gettempdir(), 'rc_cache_3'))
+        os.path.join(default_cache_dir, 'rc_cache_3'), lower=False)
 
     # cache_repo_longterm memory, 96H
     _string_setting(
         settings,
         'rc_cache.cache_repo_longterm.backend',
-        'dogpile.cache.rc.memory_lru')
+        'dogpile.cache.rc.memory_lru', lower=False)
     _int_setting(
         settings,
         'rc_cache.cache_repo_longterm.expiration_time',
@@ -498,7 +508,7 b' def _sanitize_cache_settings(settings):'
     _string_setting(
         settings,
         'rc_cache.sql_cache_short.backend',
-        'dogpile.cache.rc.memory_lru')
+        'dogpile.cache.rc.memory_lru', lower=False)
     _int_setting(
         settings,
         'rc_cache.sql_cache_short.expiration_time',
@@ -511,6 +521,7 b' def _sanitize_cache_settings(settings):'
 
 def _int_setting(settings, name, default):
     settings[name] = int(settings.get(name, default))
+    return settings[name]
 
 
 def _bool_setting(settings, name, default):
@@ -518,6 +529,7 b' def _bool_setting(settings, name, defaul'
     if isinstance(input_val, unicode):
         input_val = input_val.encode('utf8')
     settings[name] = asbool(input_val)
+    return settings[name]
 
 
 def _list_setting(settings, name, default):
@@ -530,13 +542,20 b' def _list_setting(settings, name, defaul'
     else:
         # Otherwise we assume it uses pyramids space/newline separation.
         settings[name] = aslist(raw_value)
+    return settings[name]
 
 
-def _string_setting(settings, name, default, lower=True):
+def _string_setting(settings, name, default, lower=True, default_when_empty=False):
     value = settings.get(name, default)
+
+    if default_when_empty and not value:
+        # use default value when value is empty
+        value = default
+
     if lower:
         value = value.lower()
     settings[name] = value
+    return settings[name]
 
 
 def _substitute_values(mapping, substitutions):

rhodecode/config/utils.py

@@ -81,9 +81,14 b' def get_vcs_server_protocol(config):'
 
 
 def set_instance_id(config):
-    """ Sets a dynamic generated config['instance_id'] if missing or '*' """
+    """
+    Sets a dynamic generated config['instance_id'] if missing or '*'
+    E.g instance_id = *cluster-1 or instance_id = *
+    """
 
     config['instance_id'] = config.get('instance_id') or ''
-    if config['instance_id'] == '*' or not config['instance_id']:
+    instance_id = config['instance_id']
+    if instance_id.startswith('*') or not instance_id:
+        prefix = instance_id.lstrip('*')
         _platform_id = platform.uname()[1] or 'instance'
-        config['instance_id'] = '%s-%s' % (_platform_id, os.getpid())
+        config['instance_id'] = '%s%s-%s' % (prefix, _platform_id, os.getpid())

rhodecode/integrations/__init__.py

@@ -17,11 +17,13 b''
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
-
+import sys
 import logging
 
 from rhodecode.integrations.registry import IntegrationTypeRegistry
 from rhodecode.integrations.types import webhook, slack, hipchat, email, base
+from rhodecode.lib.exc_tracking import store_exception
+
 log = logging.getLogger(__name__)
 
 
@@ -61,6 +63,8 b' def integrations_event_handler(event):'
         try:
             integration_model.send_event(integration, event)
         except Exception:
+            exc_info = sys.exc_info()
+            store_exception(id(exc_info), exc_info)
             log.exception(
                 'failure occurred when sending event %s to integration %s' % (
                     event, integration))

rhodecode/lib/exc_tracking.py

@@ -87,6 +87,13 b' def _store_exception(exc_id, exc_info, p'
 
 
 def store_exception(exc_id, exc_info, prefix=global_prefix):
+    """
+    Example usage::
+
+        exc_info = sys.exc_info()
+        store_exception(id(exc_info), exc_info)
+    """
+
     try:
         _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
     except Exception:

rhodecode/lib/repo_maintenance.py

@@ -51,25 +51,81 b' class GitGC(MaintenanceTask):'
         output = []
         instance = self.db_repo.scm_instance()
 
-        objects = self._count_objects(instance)
-        output.append(objects)
-        log.debug('GIT objects:%s', objects)
+        objects_before = self._count_objects(instance)
 
-        stdout, stderr = instance.run_git_command(
-            ['gc', '--aggressive'], fail_on_stderr=False)
+        log.debug('GIT objects:%s', objects_before)
+        cmd = ['gc', '--aggressive']
+        stdout, stderr = instance.run_git_command(cmd, fail_on_stderr=False)
+
+        out = 'executed {}'.format(' '.join(cmd))
+        output.append(out)
 
-        out = 'executed git gc --aggressive'
+        out = ''
         if stderr:
-            out = ''.join(stderr.splitlines())
+            out += ''.join(stderr.splitlines())
 
-        elif stdout:
-            out = ''.join(stdout.splitlines())
+        if stdout:
+            out += ''.join(stdout.splitlines())
 
+        if out:
         output.append(out)
 
-        objects = self._count_objects(instance)
-        log.debug('GIT objects:%s', objects)
-        output.append(objects)
+        objects_after = self._count_objects(instance)
+        log.debug('GIT objects:%s', objects_after)
+        output.append('objects before :' + objects_before)
+        output.append('objects after  :' + objects_after)
+
+        return '\n'.join(output)
+
+
+class GitFSCK(MaintenanceTask):
+    human_name = 'GIT FSCK'
+
+    def run(self):
+        output = []
+        instance = self.db_repo.scm_instance()
+
+        cmd = ['fsck', '--full']
+        stdout, stderr = instance.run_git_command(cmd, fail_on_stderr=False)
+
+        out = 'executed {}'.format(' '.join(cmd))
+        output.append(out)
+
+        out = ''
+        if stderr:
+            out += ''.join(stderr.splitlines())
+
+        if stdout:
+            out += ''.join(stdout.splitlines())
+
+        if out:
+            output.append(out)
+
+        return '\n'.join(output)
+
+
+class GitRepack(MaintenanceTask):
+    human_name = 'GIT Repack'
+
+    def run(self):
+        output = []
+        instance = self.db_repo.scm_instance()
+        cmd = ['repack', '-a', '-d',
+               '--window-memory', '10m', '--max-pack-size', '100m']
+        stdout, stderr = instance.run_git_command(cmd, fail_on_stderr=False)
+
+        out = 'executed {}'.format(' '.join(cmd))
+        output.append(out)
+        out = ''
+
+        if stderr:
+            out += ''.join(stderr.splitlines())
+
+        if stdout:
+            out += ''.join(stdout.splitlines())
+
+        if out:
+            output.append(out)
 
         return '\n'.join(output)
 
@@ -98,7 +154,7 b' class RepoMaintenance(object):'
     """
     tasks = {
         'hg': [HGVerify],
-        'git': [GitGC],
+        'git': [GitFSCK, GitGC, GitRepack],
         'svn': [SVNVerify],
     }
 
@@ -114,5 +170,6 b' class RepoMaintenance(object):'
     def execute(self, db_repo):
         executed_tasks = []
         for task in self.tasks[db_repo.repo_type]:
-            executed_tasks.append(task(db_repo).run())
+            output = task.human_name + ':\n' + task(db_repo).run() + '\n--\n'
+            executed_tasks.append(output)
         return executed_tasks