diff --git a/docs/release-notes/release-notes-4.13.1.rst b/docs/release-notes/release-notes-4.13.1.rst new file mode 100644 --- /dev/null +++ b/docs/release-notes/release-notes-4.13.1.rst @@ -0,0 +1,43 @@ +|RCE| 4.13.1 |RNS| +------------------ + +Release Date +^^^^^^^^^^^^ + +- 2018-08-06 + + +New Features +^^^^^^^^^^^^ + + + +General +^^^^^^^ + +- core: added option to prefix cache keys for usage in cluster. +- exception-tracker: store event sending exception for easier event fail debugging. +- maintenance: add repack and fsck for git maintenance execution list. + + +Security +^^^^^^^^ + + + +Performance +^^^^^^^^^^^ + + + +Fixes +^^^^^ + +- caches: use single default cache dir for all backends. +- caches: don't use lower in cache settings to support uppercase PATHS + + +Upgrade notes +^^^^^^^^^^^^^ + +- Unscheduled release addressing reported problems, and improving stability. diff --git a/docs/release-notes/release-notes.rst b/docs/release-notes/release-notes.rst --- a/docs/release-notes/release-notes.rst +++ b/docs/release-notes/release-notes.rst @@ -9,6 +9,7 @@ Release Notes .. toctree:: :maxdepth: 1 + release-notes-4.13.1.rst release-notes-4.13.0.rst release-notes-4.12.4.rst release-notes-4.12.3.rst diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -1653,6 +1653,8 @@ self: super: { self."setuptools-scm" self."amqp" self."authomatic" + self."atomicwrites" + self."attrs" self."babel" self."beaker" self."celery" diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,8 @@ setuptools-scm==2.1.0 amqp==2.3.1 # not released authomatic that has updated some oauth providers https://code.rhodecode.com/upstream/authomatic/archive/90a9ce60cc405ae8a2bf5c3713acd5d78579a04e.tar.gz?md5=3c68720a1322b25254009518d1ff6801#egg=authomatic==0.1.0.post1 +atomicwrites==1.1.5 +attrs==18.1.0 babel==1.3 beaker==1.9.1 celery==4.1.1 diff --git a/rhodecode/apps/repository/views/repo_maintainance.py b/rhodecode/apps/repository/views/repo_maintainance.py --- a/rhodecode/apps/repository/views/repo_maintainance.py +++ b/rhodecode/apps/repository/views/repo_maintainance.py @@ -32,8 +32,6 @@ log = logging.getLogger(__name__) class RepoMaintenanceView(RepoAppView): def load_default_context(self): c = self._get_local_tmpl_context() - - return c @LoginRequired() diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -436,13 +436,23 @@ def _sanitize_vcs_settings(settings): def _sanitize_cache_settings(settings): - _string_setting(settings, 'cache_dir', - os.path.join(tempfile.gettempdir(), 'rc_cache')) + default_cache_dir = os.path.join(tempfile.gettempdir(), 'rc_cache') + + # save default, cache dir, and use it for all backends later. + default_cache_dir = _string_setting( + settings, + 'cache_dir', + default_cache_dir, lower=False, default_when_empty=True) + + # ensure we have our dir created + if not os.path.isdir(default_cache_dir): + os.makedirs(default_cache_dir, mode=0755) + # cache_perms _string_setting( settings, 'rc_cache.cache_perms.backend', - 'dogpile.cache.rc.file_namespace') + 'dogpile.cache.rc.file_namespace', lower=False) _int_setting( settings, 'rc_cache.cache_perms.expiration_time', @@ -450,13 +460,13 @@ def _sanitize_cache_settings(settings): _string_setting( settings, 'rc_cache.cache_perms.arguments.filename', - os.path.join(tempfile.gettempdir(), 'rc_cache_1')) + os.path.join(default_cache_dir, 'rc_cache_1'), lower=False) # cache_repo _string_setting( settings, 'rc_cache.cache_repo.backend', - 'dogpile.cache.rc.file_namespace') + 'dogpile.cache.rc.file_namespace', lower=False) _int_setting( settings, 'rc_cache.cache_repo.expiration_time', @@ -464,13 +474,13 @@ def _sanitize_cache_settings(settings): _string_setting( settings, 'rc_cache.cache_repo.arguments.filename', - os.path.join(tempfile.gettempdir(), 'rc_cache_2')) + os.path.join(default_cache_dir, 'rc_cache_2'), lower=False) # cache_license _string_setting( settings, 'rc_cache.cache_license.backend', - 'dogpile.cache.rc.file_namespace') + 'dogpile.cache.rc.file_namespace', lower=False) _int_setting( settings, 'rc_cache.cache_license.expiration_time', @@ -478,13 +488,13 @@ def _sanitize_cache_settings(settings): _string_setting( settings, 'rc_cache.cache_license.arguments.filename', - os.path.join(tempfile.gettempdir(), 'rc_cache_3')) + os.path.join(default_cache_dir, 'rc_cache_3'), lower=False) # cache_repo_longterm memory, 96H _string_setting( settings, 'rc_cache.cache_repo_longterm.backend', - 'dogpile.cache.rc.memory_lru') + 'dogpile.cache.rc.memory_lru', lower=False) _int_setting( settings, 'rc_cache.cache_repo_longterm.expiration_time', @@ -498,7 +508,7 @@ def _sanitize_cache_settings(settings): _string_setting( settings, 'rc_cache.sql_cache_short.backend', - 'dogpile.cache.rc.memory_lru') + 'dogpile.cache.rc.memory_lru', lower=False) _int_setting( settings, 'rc_cache.sql_cache_short.expiration_time', @@ -511,6 +521,7 @@ def _sanitize_cache_settings(settings): def _int_setting(settings, name, default): settings[name] = int(settings.get(name, default)) + return settings[name] def _bool_setting(settings, name, default): @@ -518,6 +529,7 @@ def _bool_setting(settings, name, defaul if isinstance(input_val, unicode): input_val = input_val.encode('utf8') settings[name] = asbool(input_val) + return settings[name] def _list_setting(settings, name, default): @@ -530,13 +542,20 @@ def _list_setting(settings, name, defaul else: # Otherwise we assume it uses pyramids space/newline separation. settings[name] = aslist(raw_value) + return settings[name] -def _string_setting(settings, name, default, lower=True): +def _string_setting(settings, name, default, lower=True, default_when_empty=False): value = settings.get(name, default) + + if default_when_empty and not value: + # use default value when value is empty + value = default + if lower: value = value.lower() settings[name] = value + return settings[name] def _substitute_values(mapping, substitutions): diff --git a/rhodecode/config/utils.py b/rhodecode/config/utils.py --- a/rhodecode/config/utils.py +++ b/rhodecode/config/utils.py @@ -81,9 +81,14 @@ def get_vcs_server_protocol(config): def set_instance_id(config): - """ Sets a dynamic generated config['instance_id'] if missing or '*' """ + """ + Sets a dynamic generated config['instance_id'] if missing or '*' + E.g instance_id = *cluster-1 or instance_id = * + """ config['instance_id'] = config.get('instance_id') or '' - if config['instance_id'] == '*' or not config['instance_id']: + instance_id = config['instance_id'] + if instance_id.startswith('*') or not instance_id: + prefix = instance_id.lstrip('*') _platform_id = platform.uname()[1] or 'instance' - config['instance_id'] = '%s-%s' % (_platform_id, os.getpid()) + config['instance_id'] = '%s%s-%s' % (prefix, _platform_id, os.getpid()) diff --git a/rhodecode/integrations/__init__.py b/rhodecode/integrations/__init__.py --- a/rhodecode/integrations/__init__.py +++ b/rhodecode/integrations/__init__.py @@ -17,11 +17,13 @@ # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ - +import sys import logging from rhodecode.integrations.registry import IntegrationTypeRegistry from rhodecode.integrations.types import webhook, slack, hipchat, email, base +from rhodecode.lib.exc_tracking import store_exception + log = logging.getLogger(__name__) @@ -61,6 +63,8 @@ def integrations_event_handler(event): try: integration_model.send_event(integration, event) except Exception: + exc_info = sys.exc_info() + store_exception(id(exc_info), exc_info) log.exception( 'failure occurred when sending event %s to integration %s' % ( event, integration)) diff --git a/rhodecode/lib/exc_tracking.py b/rhodecode/lib/exc_tracking.py --- a/rhodecode/lib/exc_tracking.py +++ b/rhodecode/lib/exc_tracking.py @@ -87,6 +87,13 @@ def _store_exception(exc_id, exc_info, p def store_exception(exc_id, exc_info, prefix=global_prefix): + """ + Example usage:: + + exc_info = sys.exc_info() + store_exception(id(exc_info), exc_info) + """ + try: _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix) except Exception: diff --git a/rhodecode/lib/repo_maintenance.py b/rhodecode/lib/repo_maintenance.py --- a/rhodecode/lib/repo_maintenance.py +++ b/rhodecode/lib/repo_maintenance.py @@ -51,25 +51,81 @@ class GitGC(MaintenanceTask): output = [] instance = self.db_repo.scm_instance() - objects = self._count_objects(instance) - output.append(objects) - log.debug('GIT objects:%s', objects) - - stdout, stderr = instance.run_git_command( - ['gc', '--aggressive'], fail_on_stderr=False) + objects_before = self._count_objects(instance) - out = 'executed git gc --aggressive' - if stderr: - out = ''.join(stderr.splitlines()) + log.debug('GIT objects:%s', objects_before) + cmd = ['gc', '--aggressive'] + stdout, stderr = instance.run_git_command(cmd, fail_on_stderr=False) - elif stdout: - out = ''.join(stdout.splitlines()) - + out = 'executed {}'.format(' '.join(cmd)) output.append(out) - objects = self._count_objects(instance) - log.debug('GIT objects:%s', objects) - output.append(objects) + out = '' + if stderr: + out += ''.join(stderr.splitlines()) + + if stdout: + out += ''.join(stdout.splitlines()) + + if out: + output.append(out) + + objects_after = self._count_objects(instance) + log.debug('GIT objects:%s', objects_after) + output.append('objects before :' + objects_before) + output.append('objects after :' + objects_after) + + return '\n'.join(output) + + +class GitFSCK(MaintenanceTask): + human_name = 'GIT FSCK' + + def run(self): + output = [] + instance = self.db_repo.scm_instance() + + cmd = ['fsck', '--full'] + stdout, stderr = instance.run_git_command(cmd, fail_on_stderr=False) + + out = 'executed {}'.format(' '.join(cmd)) + output.append(out) + + out = '' + if stderr: + out += ''.join(stderr.splitlines()) + + if stdout: + out += ''.join(stdout.splitlines()) + + if out: + output.append(out) + + return '\n'.join(output) + + +class GitRepack(MaintenanceTask): + human_name = 'GIT Repack' + + def run(self): + output = [] + instance = self.db_repo.scm_instance() + cmd = ['repack', '-a', '-d', + '--window-memory', '10m', '--max-pack-size', '100m'] + stdout, stderr = instance.run_git_command(cmd, fail_on_stderr=False) + + out = 'executed {}'.format(' '.join(cmd)) + output.append(out) + out = '' + + if stderr: + out += ''.join(stderr.splitlines()) + + if stdout: + out += ''.join(stdout.splitlines()) + + if out: + output.append(out) return '\n'.join(output) @@ -98,7 +154,7 @@ class RepoMaintenance(object): """ tasks = { 'hg': [HGVerify], - 'git': [GitGC], + 'git': [GitFSCK, GitGC, GitRepack], 'svn': [SVNVerify], } @@ -114,5 +170,6 @@ class RepoMaintenance(object): def execute(self, db_repo): executed_tasks = [] for task in self.tasks[db_repo.repo_type]: - executed_tasks.append(task(db_repo).run()) + output = task.human_name + ':\n' + task(db_repo).run() + '\n--\n' + executed_tasks.append(output) return executed_tasks