update: add new async task to check for updates via scheduler.

- save the data into DB for later evaluation
- show in system info that we know the system is outdated

rhodecode/model/update.py

@@ -0,0 +1,71 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013-2017 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+import urllib2
+from packaging import version
+
+import rhodecode
+from rhodecode.lib.ext_json import json
+from rhodecode.model import BaseModel
+from rhodecode.model.meta import Session
+from rhodecode.model.settings import SettingsModel
+
+
+log = logging.getLogger(__name__)
+
+
+class UpdateModel(BaseModel):
+    UPDATE_SETTINGS_KEY = 'update_version'
+    UPDATE_URL_SETTINGS_KEY = 'rhodecode_update_url'
+
+    @staticmethod
+    def get_update_data(update_url):
+        """Return the JSON update data."""
+        ver = rhodecode.__version__
+        log.debug('Checking for upgrade on `%s` server', update_url)
+        opener = urllib2.build_opener()
+        opener.addheaders = [('User-agent', 'RhodeCode-SCM/%s' % ver)]
+        response = opener.open(update_url)
+        response_data = response.read()
+        data = json.loads(response_data)
+        log.debug('update server returned data')
+        return data
+
+    def get_update_url(self):
+        settings = SettingsModel().get_all_settings()
+        return settings.get(self.UPDATE_URL_SETTINGS_KEY)
+
+    def store_version(self, version):
+        log.debug('Storing version %s into settings', version)
+        setting = SettingsModel().create_or_update_setting(
+            self.UPDATE_SETTINGS_KEY, version)
+        Session().add(setting)
+        Session().commit()
+
+    def get_stored_version(self):
+        obj = SettingsModel().get_setting_by_name(self.UPDATE_SETTINGS_KEY)
+        if obj:
+            return obj.app_settings_value
+        return '0.0.0'
+
+    def is_outdated(self, cur_version, latest_version=None):
+        latest_version = latest_version or self.get_stored_version()
+        return version.Version(latest_version) > version.Version(cur_version)

rhodecode/apps/admin/tests/test_admin_settings.py

@@ -1,730 +1,729 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import mock
 import pytest
 
 import rhodecode
 from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.lib.utils2 import md5
 from rhodecode.model.db import RhodeCodeUi
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import SettingsModel, IssueTrackerSettingsModel
 from rhodecode.tests import assert_session_flash
 from rhodecode.tests.utils import AssertResponse
 
 
-UPDATE_DATA_QUALNAME = (
-    'rhodecode.apps.admin.views.system_info.AdminSystemInfoSettingsView.get_update_data')
+UPDATE_DATA_QUALNAME = 'rhodecode.model.update.UpdateModel.get_update_data'
 
 
 def route_path(name, params=None, **kwargs):
     import urllib
     from rhodecode.apps._base import ADMIN_PREFIX
 
     base_url = {
 
         'admin_settings':
             ADMIN_PREFIX +'/settings',
         'admin_settings_update':
             ADMIN_PREFIX + '/settings/update',
         'admin_settings_global':
             ADMIN_PREFIX + '/settings/global',
         'admin_settings_global_update':
             ADMIN_PREFIX + '/settings/global/update',
         'admin_settings_vcs':
             ADMIN_PREFIX + '/settings/vcs',
         'admin_settings_vcs_update':
             ADMIN_PREFIX + '/settings/vcs/update',
         'admin_settings_vcs_svn_pattern_delete':
             ADMIN_PREFIX + '/settings/vcs/svn_pattern_delete',
         'admin_settings_mapping':
             ADMIN_PREFIX + '/settings/mapping',
         'admin_settings_mapping_update':
             ADMIN_PREFIX + '/settings/mapping/update',
         'admin_settings_visual':
             ADMIN_PREFIX + '/settings/visual',
         'admin_settings_visual_update':
             ADMIN_PREFIX + '/settings/visual/update',
         'admin_settings_issuetracker':
             ADMIN_PREFIX + '/settings/issue-tracker',
         'admin_settings_issuetracker_update':
             ADMIN_PREFIX + '/settings/issue-tracker/update',
         'admin_settings_issuetracker_test':
             ADMIN_PREFIX + '/settings/issue-tracker/test',
         'admin_settings_issuetracker_delete':
             ADMIN_PREFIX + '/settings/issue-tracker/delete',
         'admin_settings_email':
             ADMIN_PREFIX + '/settings/email',
         'admin_settings_email_update':
             ADMIN_PREFIX + '/settings/email/update',
         'admin_settings_hooks':
             ADMIN_PREFIX + '/settings/hooks',
         'admin_settings_hooks_update':
             ADMIN_PREFIX + '/settings/hooks/update',
         'admin_settings_hooks_delete':
             ADMIN_PREFIX + '/settings/hooks/delete',
         'admin_settings_search':
             ADMIN_PREFIX + '/settings/search',
         'admin_settings_labs':
             ADMIN_PREFIX + '/settings/labs',
         'admin_settings_labs_update':
             ADMIN_PREFIX + '/settings/labs/update',
 
         'admin_settings_sessions':
             ADMIN_PREFIX + '/settings/sessions',
         'admin_settings_sessions_cleanup':
             ADMIN_PREFIX + '/settings/sessions/cleanup',
         'admin_settings_system':
             ADMIN_PREFIX + '/settings/system',
         'admin_settings_system_update':
             ADMIN_PREFIX + '/settings/system/updates',
         'admin_settings_open_source':
             ADMIN_PREFIX + '/settings/open_source',
 
 
     }[name].format(**kwargs)
 
     if params:
         base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
     return base_url
 
 
 @pytest.mark.usefixtures('autologin_user', 'app')
 class TestAdminSettingsController(object):
 
     @pytest.mark.parametrize('urlname', [
         'admin_settings_vcs',
         'admin_settings_mapping',
         'admin_settings_global',
         'admin_settings_visual',
         'admin_settings_email',
         'admin_settings_hooks',
         'admin_settings_search',
     ])
     def test_simple_get(self, urlname):
         self.app.get(route_path(urlname))
 
     def test_create_custom_hook(self, csrf_token):
         response = self.app.post(
             route_path('admin_settings_hooks_update'),
             params={
                 'new_hook_ui_key': 'test_hooks_1',
                 'new_hook_ui_value': 'cd /tmp',
                 'csrf_token': csrf_token})
 
         response = response.follow()
         response.mustcontain('test_hooks_1')
         response.mustcontain('cd /tmp')
 
     def test_create_custom_hook_delete(self, csrf_token):
         response = self.app.post(
             route_path('admin_settings_hooks_update'),
             params={
                 'new_hook_ui_key': 'test_hooks_2',
                 'new_hook_ui_value': 'cd /tmp2',
                 'csrf_token': csrf_token})
 
         response = response.follow()
         response.mustcontain('test_hooks_2')
         response.mustcontain('cd /tmp2')
 
         hook_id = SettingsModel().get_ui_by_key('test_hooks_2').ui_id
 
         # delete
         self.app.post(
             route_path('admin_settings_hooks_delete'),
             params={'hook_id': hook_id, 'csrf_token': csrf_token})
         response = self.app.get(route_path('admin_settings_hooks'))
         response.mustcontain(no=['test_hooks_2'])
         response.mustcontain(no=['cd /tmp2'])
 
 
 @pytest.mark.usefixtures('autologin_user', 'app')
 class TestAdminSettingsGlobal(object):
 
     def test_pre_post_code_code_active(self, csrf_token):
         pre_code = 'rc-pre-code-187652122'
         post_code = 'rc-postcode-98165231'
 
         response = self.post_and_verify_settings({
             'rhodecode_pre_code': pre_code,
             'rhodecode_post_code': post_code,
             'csrf_token': csrf_token,
         })
 
         response = response.follow()
         response.mustcontain(pre_code, post_code)
 
     def test_pre_post_code_code_inactive(self, csrf_token):
         pre_code = 'rc-pre-code-187652122'
         post_code = 'rc-postcode-98165231'
         response = self.post_and_verify_settings({
             'rhodecode_pre_code': '',
             'rhodecode_post_code': '',
             'csrf_token': csrf_token,
         })
 
         response = response.follow()
         response.mustcontain(no=[pre_code, post_code])
 
     def test_captcha_activate(self, csrf_token):
         self.post_and_verify_settings({
             'rhodecode_captcha_private_key': '1234567890',
             'rhodecode_captcha_public_key': '1234567890',
             'csrf_token': csrf_token,
         })
 
         response = self.app.get(ADMIN_PREFIX + '/register')
         response.mustcontain('captcha')
 
     def test_captcha_deactivate(self, csrf_token):
         self.post_and_verify_settings({
             'rhodecode_captcha_private_key': '',
             'rhodecode_captcha_public_key': '1234567890',
             'csrf_token': csrf_token,
         })
 
         response = self.app.get(ADMIN_PREFIX + '/register')
         response.mustcontain(no=['captcha'])
 
     def test_title_change(self, csrf_token):
         old_title = 'RhodeCode'
 
         for new_title in ['Changed', 'Żółwik', old_title]:
             response = self.post_and_verify_settings({
                 'rhodecode_title': new_title,
                 'csrf_token': csrf_token,
             })
 
             response = response.follow()
             response.mustcontain(
                 """<div class="branding">- %s</div>""" % new_title)
 
     def post_and_verify_settings(self, settings):
         old_title = 'RhodeCode'
         old_realm = 'RhodeCode authentication'
         params = {
             'rhodecode_title': old_title,
             'rhodecode_realm': old_realm,
             'rhodecode_pre_code': '',
             'rhodecode_post_code': '',
             'rhodecode_captcha_private_key': '',
             'rhodecode_captcha_public_key': '',
             'rhodecode_create_personal_repo_group': False,
             'rhodecode_personal_repo_group_pattern': '${username}',
         }
         params.update(settings)
         response = self.app.post(
             route_path('admin_settings_global_update'), params=params)
 
         assert_session_flash(response, 'Updated application settings')
         app_settings = SettingsModel().get_all_settings()
         del settings['csrf_token']
         for key, value in settings.iteritems():
             assert app_settings[key] == value.decode('utf-8')
 
         return response
 
 
 @pytest.mark.usefixtures('autologin_user', 'app')
 class TestAdminSettingsVcs(object):
 
     def test_contains_svn_default_patterns(self):
         response = self.app.get(route_path('admin_settings_vcs'))
         expected_patterns = [
             '/trunk',
             '/branches/*',
             '/tags/*',
         ]
         for pattern in expected_patterns:
             response.mustcontain(pattern)
 
     def test_add_new_svn_branch_and_tag_pattern(
             self, backend_svn, form_defaults, disable_sql_cache,
             csrf_token):
         form_defaults.update({
             'new_svn_branch': '/exp/branches/*',
             'new_svn_tag': '/important_tags/*',
             'csrf_token': csrf_token,
         })
 
         response = self.app.post(
             route_path('admin_settings_vcs_update'),
             params=form_defaults, status=302)
         response = response.follow()
 
         # Expect to find the new values on the page
         response.mustcontain('/exp/branches/*')
         response.mustcontain('/important_tags/*')
 
         # Expect that those patterns are used to match branches and tags now
         repo = backend_svn['svn-simple-layout'].scm_instance()
         assert 'exp/branches/exp-sphinx-docs' in repo.branches
         assert 'important_tags/v0.5' in repo.tags
 
     def test_add_same_svn_value_twice_shows_an_error_message(
             self, form_defaults, csrf_token, settings_util):
         settings_util.create_rhodecode_ui('vcs_svn_branch', '/test')
         settings_util.create_rhodecode_ui('vcs_svn_tag', '/test')
 
         response = self.app.post(
             route_path('admin_settings_vcs_update'),
             params={
                 'paths_root_path': form_defaults['paths_root_path'],
                 'new_svn_branch': '/test',
                 'new_svn_tag': '/test',
                 'csrf_token': csrf_token,
             },
             status=200)
 
         response.mustcontain("Pattern already exists")
         response.mustcontain("Some form inputs contain invalid data.")
 
     @pytest.mark.parametrize('section', [
         'vcs_svn_branch',
         'vcs_svn_tag',
     ])
     def test_delete_svn_patterns(
             self, section, csrf_token, settings_util):
         setting = settings_util.create_rhodecode_ui(
             section, '/test_delete', cleanup=False)
 
         self.app.post(
             route_path('admin_settings_vcs_svn_pattern_delete'),
             params={
                 'delete_svn_pattern': setting.ui_id,
                 'csrf_token': csrf_token},
             headers={'X-REQUESTED-WITH': 'XMLHttpRequest'})
 
     @pytest.mark.parametrize('section', [
         'vcs_svn_branch',
         'vcs_svn_tag',
     ])
     def test_delete_svn_patterns_raises_404_when_no_xhr(
             self, section, csrf_token, settings_util):
         setting = settings_util.create_rhodecode_ui(section, '/test_delete')
 
         self.app.post(
             route_path('admin_settings_vcs_svn_pattern_delete'),
             params={
                 'delete_svn_pattern': setting.ui_id,
                 'csrf_token': csrf_token},
             status=404)
 
     def test_extensions_hgsubversion(self, form_defaults, csrf_token):
         form_defaults.update({
             'csrf_token': csrf_token,
             'extensions_hgsubversion': 'True',
         })
         response = self.app.post(
             route_path('admin_settings_vcs_update'),
             params=form_defaults,
             status=302)
 
         response = response.follow()
         extensions_input = (
             '<input id="extensions_hgsubversion" '
             'name="extensions_hgsubversion" type="checkbox" '
             'value="True" checked="checked" />')
         response.mustcontain(extensions_input)
 
     def test_extensions_hgevolve(self, form_defaults, csrf_token):
         form_defaults.update({
             'csrf_token': csrf_token,
             'extensions_evolve': 'True',
         })
         response = self.app.post(
             route_path('admin_settings_vcs_update'),
             params=form_defaults,
             status=302)
 
         response = response.follow()
         extensions_input = (
             '<input id="extensions_evolve" '
             'name="extensions_evolve" type="checkbox" '
             'value="True" checked="checked" />')
         response.mustcontain(extensions_input)
 
     def test_has_a_section_for_pull_request_settings(self):
         response = self.app.get(route_path('admin_settings_vcs'))
         response.mustcontain('Pull Request Settings')
 
     def test_has_an_input_for_invalidation_of_inline_comments(self):
         response = self.app.get(route_path('admin_settings_vcs'))
         assert_response = AssertResponse(response)
         assert_response.one_element_exists(
             '[name=rhodecode_use_outdated_comments]')
 
     @pytest.mark.parametrize('new_value', [True, False])
     def test_allows_to_change_invalidation_of_inline_comments(
             self, form_defaults, csrf_token, new_value):
         setting_key = 'use_outdated_comments'
         setting = SettingsModel().create_or_update_setting(
             setting_key, not new_value, 'bool')
         Session().add(setting)
         Session().commit()
 
         form_defaults.update({
             'csrf_token': csrf_token,
             'rhodecode_use_outdated_comments': str(new_value),
         })
         response = self.app.post(
             route_path('admin_settings_vcs_update'),
             params=form_defaults,
             status=302)
         response = response.follow()
         setting = SettingsModel().get_setting_by_name(setting_key)
         assert setting.app_settings_value is new_value
 
     @pytest.mark.parametrize('new_value', [True, False])
     def test_allows_to_change_hg_rebase_merge_strategy(
             self, form_defaults, csrf_token, new_value):
         setting_key = 'hg_use_rebase_for_merging'
 
         form_defaults.update({
             'csrf_token': csrf_token,
             'rhodecode_' + setting_key: str(new_value),
         })
 
         with mock.patch.dict(
                 rhodecode.CONFIG, {'labs_settings_active': 'true'}):
             self.app.post(
                 route_path('admin_settings_vcs_update'),
                 params=form_defaults,
                 status=302)
 
         setting = SettingsModel().get_setting_by_name(setting_key)
         assert setting.app_settings_value is new_value
 
     @pytest.fixture
     def disable_sql_cache(self, request):
         patcher = mock.patch(
             'rhodecode.lib.caching_query.FromCache.process_query')
         request.addfinalizer(patcher.stop)
         patcher.start()
 
     @pytest.fixture
     def form_defaults(self):
         from rhodecode.apps.admin.views.settings import AdminSettingsView
         return AdminSettingsView._form_defaults()
 
     # TODO: johbo: What we really want is to checkpoint before a test run and
     # reset the session afterwards.
     @pytest.fixture(scope='class', autouse=True)
     def cleanup_settings(self, request, baseapp):
         ui_id = RhodeCodeUi.ui_id
         original_ids = list(
             r.ui_id for r in RhodeCodeUi.query().values(ui_id))
 
         @request.addfinalizer
         def cleanup():
             RhodeCodeUi.query().filter(
                 ui_id.notin_(original_ids)).delete(False)
 
 
 @pytest.mark.usefixtures('autologin_user', 'app')
 class TestLabsSettings(object):
     def test_get_settings_page_disabled(self):
         with mock.patch.dict(
                 rhodecode.CONFIG, {'labs_settings_active': 'false'}):
 
             response = self.app.get(
                 route_path('admin_settings_labs'), status=302)
 
         assert response.location.endswith(route_path('admin_settings'))
 
     def test_get_settings_page_enabled(self):
         from rhodecode.apps.admin.views import settings
         lab_settings = [
             settings.LabSetting(
                 key='rhodecode_bool',
                 type='bool',
                 group='bool group',
                 label='bool label',
                 help='bool help'
             ),
             settings.LabSetting(
                 key='rhodecode_text',
                 type='unicode',
                 group='text group',
                 label='text label',
                 help='text help'
             ),
         ]
         with mock.patch.dict(rhodecode.CONFIG,
                              {'labs_settings_active': 'true'}):
             with mock.patch.object(settings, '_LAB_SETTINGS', lab_settings):
                 response = self.app.get(route_path('admin_settings_labs'))
 
         assert '<label>bool group:</label>' in response
         assert '<label for="rhodecode_bool">bool label</label>' in response
         assert '<p class="help-block">bool help</p>' in response
         assert 'name="rhodecode_bool" type="checkbox"' in response
 
         assert '<label>text group:</label>' in response
         assert '<label for="rhodecode_text">text label</label>' in response
         assert '<p class="help-block">text help</p>' in response
         assert 'name="rhodecode_text" size="60" type="text"' in response
 
 
 @pytest.mark.usefixtures('app')
 class TestOpenSourceLicenses(object):
 
     def test_records_are_displayed(self, autologin_user):
         sample_licenses = {
             "python2.7-pytest-2.7.1": {
                 "UNKNOWN": None
             },
             "python2.7-Markdown-2.6.2": {
                 "BSD-3-Clause": "http://spdx.org/licenses/BSD-3-Clause"
             }
         }
         read_licenses_patch = mock.patch(
             'rhodecode.apps.admin.views.open_source_licenses.read_opensource_licenses',
             return_value=sample_licenses)
         with read_licenses_patch:
             response = self.app.get(
                 route_path('admin_settings_open_source'), status=200)
 
         assert_response = AssertResponse(response)
         assert_response.element_contains(
             '.panel-heading', 'Licenses of Third Party Packages')
         for name in sample_licenses:
             response.mustcontain(name)
             for license in sample_licenses[name]:
                 assert_response.element_contains('.panel-body', license)
 
     def test_records_can_be_read(self, autologin_user):
         response = self.app.get(
             route_path('admin_settings_open_source'), status=200)
         assert_response = AssertResponse(response)
         assert_response.element_contains(
             '.panel-heading', 'Licenses of Third Party Packages')
 
     def test_forbidden_when_normal_user(self, autologin_regular_user):
         self.app.get(
             route_path('admin_settings_open_source'), status=404)
 
 
 @pytest.mark.usefixtures('app')
 class TestUserSessions(object):
 
     def test_forbidden_when_normal_user(self, autologin_regular_user):
         self.app.get(route_path('admin_settings_sessions'), status=404)
 
     def test_show_sessions_page(self, autologin_user):
         response = self.app.get(route_path('admin_settings_sessions'), status=200)
         response.mustcontain('file')
 
     def test_cleanup_old_sessions(self, autologin_user, csrf_token):
 
         post_data = {
             'csrf_token': csrf_token,
             'expire_days': '60'
         }
         response = self.app.post(
             route_path('admin_settings_sessions_cleanup'), params=post_data,
             status=302)
         assert_session_flash(response, 'Cleaned up old sessions')
 
 
 @pytest.mark.usefixtures('app')
 class TestAdminSystemInfo(object):
 
     def test_forbidden_when_normal_user(self, autologin_regular_user):
         self.app.get(route_path('admin_settings_system'), status=404)
 
     def test_system_info_page(self, autologin_user):
         response = self.app.get(route_path('admin_settings_system'))
         response.mustcontain('RhodeCode Community Edition, version {}'.format(
             rhodecode.__version__))
 
     def test_system_update_new_version(self, autologin_user):
         update_data = {
             'versions': [
                 {
                     'version': '100.3.1415926535',
                     'general': 'The latest version we are ever going to ship'
                 },
                 {
                     'version': '0.0.0',
                     'general': 'The first version we ever shipped'
                 }
             ]
         }
         with mock.patch(UPDATE_DATA_QUALNAME, return_value=update_data):
             response = self.app.get(route_path('admin_settings_system_update'))
             response.mustcontain('A <b>new version</b> is available')
 
     def test_system_update_nothing_new(self, autologin_user):
         update_data = {
             'versions': [
                 {
                     'version': '0.0.0',
                     'general': 'The first version we ever shipped'
                 }
             ]
         }
         with mock.patch(UPDATE_DATA_QUALNAME, return_value=update_data):
             response = self.app.get(route_path('admin_settings_system_update'))
             response.mustcontain(
-                'You already have the <b>latest</b> stable version.')
+                'This instance is already running the <b>latest</b> stable version')
 
     def test_system_update_bad_response(self, autologin_user):
         with mock.patch(UPDATE_DATA_QUALNAME, side_effect=ValueError('foo')):
             response = self.app.get(route_path('admin_settings_system_update'))
             response.mustcontain(
                 'Bad data sent from update server')
 
 
 @pytest.mark.usefixtures("app")
 class TestAdminSettingsIssueTracker(object):
     RC_PREFIX = 'rhodecode_'
     SHORT_PATTERN_KEY = 'issuetracker_pat_'
     PATTERN_KEY = RC_PREFIX + SHORT_PATTERN_KEY
 
     def test_issuetracker_index(self, autologin_user):
         response = self.app.get(route_path('admin_settings_issuetracker'))
         assert response.status_code == 200
 
     def test_add_empty_issuetracker_pattern(
             self, request, autologin_user, csrf_token):
         post_url = route_path('admin_settings_issuetracker_update')
         post_data = {
             'csrf_token': csrf_token
         }
         self.app.post(post_url, post_data, status=302)
 
     def test_add_issuetracker_pattern(
             self, request, autologin_user, csrf_token):
         pattern = 'issuetracker_pat'
         another_pattern = pattern+'1'
         post_url = route_path('admin_settings_issuetracker_update')
         post_data = {
             'new_pattern_pattern_0': pattern,
             'new_pattern_url_0': 'http://url',
             'new_pattern_prefix_0': 'prefix',
             'new_pattern_description_0': 'description',
             'new_pattern_pattern_1': another_pattern,
             'new_pattern_url_1': 'https://url1',
             'new_pattern_prefix_1': 'prefix1',
             'new_pattern_description_1': 'description1',
             'csrf_token': csrf_token
         }
         self.app.post(post_url, post_data, status=302)
         settings = SettingsModel().get_all_settings()
         self.uid = md5(pattern)
         assert settings[self.PATTERN_KEY+self.uid] == pattern
         self.another_uid = md5(another_pattern)
         assert settings[self.PATTERN_KEY+self.another_uid] == another_pattern
 
         @request.addfinalizer
         def cleanup():
             defaults = SettingsModel().get_all_settings()
 
             entries = [name for name in defaults if (
                 (self.uid in name) or (self.another_uid) in name)]
             start = len(self.RC_PREFIX)
             for del_key in entries:
                 # TODO: anderson: get_by_name needs name without prefix
                 entry = SettingsModel().get_setting_by_name(del_key[start:])
                 Session().delete(entry)
 
             Session().commit()
 
     def test_edit_issuetracker_pattern(
             self, autologin_user, backend, csrf_token, request):
         old_pattern = 'issuetracker_pat'
         old_uid = md5(old_pattern)
         pattern = 'issuetracker_pat_new'
         self.new_uid = md5(pattern)
 
         SettingsModel().create_or_update_setting(
             self.SHORT_PATTERN_KEY+old_uid, old_pattern, 'unicode')
 
         post_url = route_path('admin_settings_issuetracker_update')
         post_data = {
             'new_pattern_pattern_0': pattern,
             'new_pattern_url_0': 'https://url',
             'new_pattern_prefix_0': 'prefix',
             'new_pattern_description_0': 'description',
             'uid': old_uid,
             'csrf_token': csrf_token
         }
         self.app.post(post_url, post_data, status=302)
         settings = SettingsModel().get_all_settings()
         assert settings[self.PATTERN_KEY+self.new_uid] == pattern
         assert self.PATTERN_KEY+old_uid not in settings
 
         @request.addfinalizer
         def cleanup():
             IssueTrackerSettingsModel().delete_entries(self.new_uid)
 
     def test_replace_issuetracker_pattern_description(
             self, autologin_user, csrf_token, request, settings_util):
         prefix = 'issuetracker'
         pattern = 'issuetracker_pat'
         self.uid = md5(pattern)
         pattern_key = '_'.join([prefix, 'pat', self.uid])
         rc_pattern_key = '_'.join(['rhodecode', pattern_key])
         desc_key = '_'.join([prefix, 'desc', self.uid])
         rc_desc_key = '_'.join(['rhodecode', desc_key])
         new_description = 'new_description'
 
         settings_util.create_rhodecode_setting(
             pattern_key, pattern, 'unicode', cleanup=False)
         settings_util.create_rhodecode_setting(
             desc_key, 'old description', 'unicode', cleanup=False)
 
         post_url = route_path('admin_settings_issuetracker_update')
         post_data = {
             'new_pattern_pattern_0': pattern,
             'new_pattern_url_0': 'https://url',
             'new_pattern_prefix_0': 'prefix',
             'new_pattern_description_0': new_description,
             'uid': self.uid,
             'csrf_token': csrf_token
         }
         self.app.post(post_url, post_data, status=302)
         settings = SettingsModel().get_all_settings()
         assert settings[rc_pattern_key] == pattern
         assert settings[rc_desc_key] == new_description
 
         @request.addfinalizer
         def cleanup():
             IssueTrackerSettingsModel().delete_entries(self.uid)
 
     def test_delete_issuetracker_pattern(
             self, autologin_user, backend, csrf_token, settings_util):
         pattern = 'issuetracker_pat'
         uid = md5(pattern)
         settings_util.create_rhodecode_setting(
             self.SHORT_PATTERN_KEY+uid, pattern, 'unicode', cleanup=False)
 
         post_url = route_path('admin_settings_issuetracker_delete')
         post_data = {
             '_method': 'delete',
             'uid': uid,
             'csrf_token': csrf_token
         }
         self.app.post(post_url, post_data, status=302)
         settings = SettingsModel().get_all_settings()
         assert 'rhodecode_%s%s' % (self.SHORT_PATTERN_KEY, uid) not in settings

rhodecode/apps/admin/views/system_info.py

@@ -1,208 +1,197 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2017  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import urllib2
-import packaging.version
 
 from pyramid.view import view_config
 
 import rhodecode
 from rhodecode.apps._base import BaseAppView
 from rhodecode.apps.admin.navigation import navigation_list
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (LoginRequired, HasPermissionAllDecorator)
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.lib import system_info
-from rhodecode.lib.ext_json import json
-from rhodecode.model.settings import SettingsModel
+from rhodecode.model.update import UpdateModel
 
 log = logging.getLogger(__name__)
 
 
 class AdminSystemInfoSettingsView(BaseAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-
         return c
 
-    @staticmethod
-    def get_update_data(update_url):
-        """Return the JSON update data."""
-        ver = rhodecode.__version__
-        log.debug('Checking for upgrade on `%s` server', update_url)
-        opener = urllib2.build_opener()
-        opener.addheaders = [('User-agent', 'RhodeCode-SCM/%s' % ver)]
-        response = opener.open(update_url)
-        response_data = response.read()
-        data = json.loads(response_data)
-
-        return data
-
-    def get_update_url(self):
-        settings = SettingsModel().get_all_settings()
-        return settings.get('rhodecode_update_url')
-
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_system', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_system_info(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         c.active = 'system'
         c.navlist = navigation_list(self.request)
 
         # TODO(marcink), figure out how to allow only selected users to do this
         c.allowed_to_snapshot = self._rhodecode_user.admin
 
         snapshot = str2bool(self.request.params.get('snapshot'))
 
-        c.rhodecode_update_url = self.get_update_url()
+        c.rhodecode_update_url = UpdateModel().get_update_url()
         server_info = system_info.get_system_info(self.request.environ)
 
         for key, val in server_info.items():
             setattr(c, key, val)
 
         def val(name, subkey='human_value'):
             return server_info[name][subkey]
 
         def state(name):
             return server_info[name]['state']
 
         def val2(name):
             val = server_info[name]['human_value']
             state = server_info[name]['state']
             return val, state
 
         update_info_msg = _('Note: please make sure this server can '
                             'access `${url}` for the update link to work',
                             mapping=dict(url=c.rhodecode_update_url))
+        version = UpdateModel().get_stored_version()
+        is_outdated = UpdateModel().is_outdated(
+            rhodecode.__version__, version)
+        update_state = {
+            'type': 'warning',
+            'message': 'New version available: {}'.format(version)
+            } \
+            if is_outdated else {}
         c.data_items = [
             # update info
             (_('Update info'), h.literal(
                 '<span class="link" id="check_for_update" >%s.</span>' % (
                 _('Check for updates')) +
                 '<br/> <span >%s.</span>' % (update_info_msg)
             ), ''),
 
             # RhodeCode specific
             (_('RhodeCode Version'), val('rhodecode_app')['text'], state('rhodecode_app')),
+            (_('Latest version'), version, update_state),
             (_('RhodeCode Server IP'), val('server')['server_ip'], state('server')),
             (_('RhodeCode Server ID'), val('server')['server_id'], state('server')),
             (_('RhodeCode Configuration'), val('rhodecode_config')['path'], state('rhodecode_config')),
             (_('RhodeCode Certificate'), val('rhodecode_config')['cert_path'], state('rhodecode_config')),
             (_('Workers'), val('rhodecode_config')['config']['server:main'].get('workers', '?'), state('rhodecode_config')),
             (_('Worker Type'), val('rhodecode_config')['config']['server:main'].get('worker_class', 'sync'), state('rhodecode_config')),
             ('', '', ''),  # spacer
 
             # Database
             (_('Database'), val('database')['url'], state('database')),
             (_('Database version'), val('database')['version'], state('database')),
             ('', '', ''),  # spacer
 
             # Platform/Python
             (_('Platform'), val('platform')['name'], state('platform')),
             (_('Platform UUID'), val('platform')['uuid'], state('platform')),
             (_('Python version'), val('python')['version'], state('python')),
             (_('Python path'), val('python')['executable'], state('python')),
             ('', '', ''),  # spacer
 
             # Systems stats
             (_('CPU'), val('cpu')['text'], state('cpu')),
             (_('Load'), val('load')['text'], state('load')),
             (_('Memory'), val('memory')['text'], state('memory')),
             (_('Uptime'), val('uptime')['text'], state('uptime')),
             ('', '', ''),  # spacer
 
             # Repo storage
             (_('Storage location'), val('storage')['path'], state('storage')),
             (_('Storage info'), val('storage')['text'], state('storage')),
             (_('Storage inodes'), val('storage_inodes')['text'], state('storage_inodes')),
 
             (_('Gist storage location'), val('storage_gist')['path'], state('storage_gist')),
             (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
 
             (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
             (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
 
             (_('Temp storage location'), val('storage_temp')['path'], state('storage_temp')),
             (_('Temp storage info'), val('storage_temp')['text'], state('storage_temp')),
 
             (_('Search info'), val('search')['text'], state('search')),
             (_('Search location'), val('search')['location'], state('search')),
             ('', '', ''),  # spacer
 
             # VCS specific
             (_('VCS Backends'), val('vcs_backends'), state('vcs_backends')),
             (_('VCS Server'), val('vcs_server')['text'], state('vcs_server')),
             (_('GIT'), val('git'), state('git')),
             (_('HG'), val('hg'), state('hg')),
             (_('SVN'), val('svn'), state('svn')),
 
         ]
 
         if snapshot:
             if c.allowed_to_snapshot:
                 c.data_items.pop(0)  # remove server info
                 self.request.override_renderer = 'admin/settings/settings_system_snapshot.mako'
             else:
                 h.flash('You are not allowed to do this', category='warning')
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_system_update', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings_system_update.mako')
     def settings_system_info_check_update(self):
         _ = self.request.translate
         c = self.load_default_context()
 
-        update_url = self.get_update_url()
+        update_url = UpdateModel().get_update_url()
 
         _err = lambda s: '<div style="color:#ff8888; padding:4px 0px">{}</div>'.format(s)
         try:
-            data = self.get_update_data(update_url)
+            data = UpdateModel().get_update_data(update_url)
         except urllib2.URLError as e:
             log.exception("Exception contacting upgrade server")
             self.request.override_renderer = 'string'
             return _err('Failed to contact upgrade server: %r' % e)
         except ValueError as e:
             log.exception("Bad data sent from update server")
             self.request.override_renderer = 'string'
             return _err('Bad data sent from update server')
 
         latest = data['versions'][0]
 
         c.update_url = update_url
         c.latest_data = latest
         c.latest_ver = latest['version']
         c.cur_ver = rhodecode.__version__
         c.should_upgrade = False
 
-        if (packaging.version.Version(c.latest_ver) >
-                packaging.version.Version(c.cur_ver)):
+        is_oudated = UpdateModel().is_outdated(c.cur_ver, c.latest_ver)
+        if is_oudated:
             c.should_upgrade = True
         c.important_notices = latest['general']
-
+        UpdateModel().store_version(latest['version'])
         return self._get_template_context(c)

rhodecode/lib/celerylib/tasks.py

@@ -1,285 +1,299 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode task modules, containing all task that suppose to be run
 by celery daemon
 """
 
 import os
 import time
 
 import rhodecode
 from rhodecode.lib import audit_logger
 from rhodecode.lib.celerylib import get_logger, async_task, RequestContextTask
 from rhodecode.lib.hooks_base import log_create_repository
 from rhodecode.lib.rcmail.smtp_mailer import SmtpMailer
 from rhodecode.lib.utils2 import safe_int, str2bool
 from rhodecode.model.db import Session, IntegrityError, Repository, User
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def send_email(recipients, subject, body='', html_body='', email_config=None):
     """
     Sends an email with defined parameters from the .ini files.
 
     :param recipients: list of recipients, it this is empty the defined email
         address from field 'email_to' is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     :param html_body: html version of body
     """
     log = get_logger(send_email)
 
     email_config = email_config or rhodecode.CONFIG
     subject = "%s %s" % (email_config.get('email_prefix', ''), subject)
     if not recipients:
         # if recipients are not defined we send to email_config + all admins
         admins = [
             u.email for u in User.query().filter(User.admin == True).all()]
         recipients = [email_config.get('email_to')] + admins
 
     mail_server = email_config.get('smtp_server') or None
     if mail_server is None:
         log.error("SMTP server information missing. Sending email failed. "
                   "Make sure that `smtp_server` variable is configured "
                   "inside the .ini file")
         return False
 
     mail_from = email_config.get('app_email_from', 'RhodeCode')
     user = email_config.get('smtp_username')
     passwd = email_config.get('smtp_password')
     mail_port = email_config.get('smtp_port')
     tls = str2bool(email_config.get('smtp_use_tls'))
     ssl = str2bool(email_config.get('smtp_use_ssl'))
     debug = str2bool(email_config.get('debug'))
     smtp_auth = email_config.get('smtp_auth')
 
     try:
         m = SmtpMailer(mail_from, user, passwd, mail_server, smtp_auth,
                        mail_port, ssl, tls, debug=debug)
         m.send(recipients, subject, body, html_body)
     except Exception:
         log.exception('Mail sending failed')
         return False
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def create_repo(form_data, cur_user):
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.user import UserModel
     from rhodecode.model.settings import SettingsModel
 
     log = get_logger(create_repo)
 
     cur_user = UserModel()._get_user(cur_user)
     owner = cur_user
 
     repo_name = form_data['repo_name']
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['repo_description']
     private = form_data['repo_private']
     clone_uri = form_data.get('clone_uri')
     repo_group = safe_int(form_data['repo_group'])
     landing_rev = form_data['repo_landing_rev']
     copy_fork_permissions = form_data.get('copy_permissions')
     copy_group_permissions = form_data.get('repo_copy_permissions')
     fork_of = form_data.get('fork_parent_id')
     state = form_data.get('repo_state', Repository.STATE_PENDING)
 
     # repo creation defaults, private and repo_type are filled in form
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = form_data.get(
         'enable_statistics', defs.get('repo_enable_statistics'))
     enable_locking = form_data.get(
         'enable_locking', defs.get('repo_enable_locking'))
     enable_downloads = form_data.get(
         'enable_downloads', defs.get('repo_enable_downloads'))
 
     try:
         repo = RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_rev,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions,
             copy_group_permissions=copy_group_permissions,
             enable_statistics=enable_statistics,
             enable_locking=enable_locking,
             enable_downloads=enable_downloads,
             state=state
         )
         Session().commit()
 
         # now create this repo on Filesystem
         RepoModel()._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel()._get_repo_group(repo_group),
             clone_uri=clone_uri,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         log_create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
         repo.update_commit_cache()
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
         repo_id = repo.repo_id
         repo_data = repo.get_api_data()
 
         audit_logger.store(
             'repo.create', action_data={'data': repo_data},
             user=cur_user,
             repo=audit_logger.RepoWrap(repo_name=repo_name, repo_id=repo_id))
 
         Session().commit()
     except Exception as e:
         log.warning('Exception occurred when creating repository, '
                     'doing cleanup...', exc_info=True)
         if isinstance(e, IntegrityError):
             Session().rollback()
 
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             Session().commit()
             RepoModel()._delete_filesystem_repo(repo)
         log.info('Cleanup of repo %s finished', repo_name_full)
         raise
 
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def create_repo_fork(form_data, cur_user):
     """
     Creates a fork of repository using internal VCS methods
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.user import UserModel
 
     log = get_logger(create_repo_fork)
 
     cur_user = UserModel()._get_user(cur_user)
     owner = cur_user
 
     repo_name = form_data['repo_name']  # fork in this case
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['description']
     private = form_data['private']
     clone_uri = form_data.get('clone_uri')
     repo_group = safe_int(form_data['repo_group'])
     landing_rev = form_data['landing_rev']
     copy_fork_permissions = form_data.get('copy_permissions')
     fork_id = safe_int(form_data.get('fork_parent_id'))
 
     try:
         fork_of = RepoModel()._get_repo(fork_id)
         RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_rev,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions
         )
 
         Session().commit()
 
         base_path = Repository.base_path()
         source_repo_path = os.path.join(base_path, fork_of.repo_name)
 
         # now create this repo on Filesystem
         RepoModel()._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel()._get_repo_group(repo_group),
             clone_uri=source_repo_path,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         log_create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
         config = repo._config
         config.set('extensions', 'largefiles', '')
         repo.update_commit_cache(config=config)
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
 
         repo_id = repo.repo_id
         repo_data = repo.get_api_data()
         audit_logger.store(
             'repo.fork', action_data={'data': repo_data},
             user=cur_user,
             repo=audit_logger.RepoWrap(repo_name=repo_name, repo_id=repo_id))
 
         Session().commit()
     except Exception as e:
         log.warning('Exception %s occurred when forking repository, '
                     'doing cleanup...', exc_info=True)
         if isinstance(e, IntegrityError):
             Session().rollback()
 
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             Session().commit()
             RepoModel()._delete_filesystem_repo(repo)
         log.info('Cleanup of repo %s finished', repo_name_full)
         raise
 
     return True
 
 
 @async_task(ignore_result=True)
 def sync_repo(*args, **kwargs):
     from rhodecode.model.scm import ScmModel
     log = get_logger(sync_repo)
     repo_name = kwargs['repo_name']
     log.info('Pulling from %s', repo_name)
     dbrepo = Repository.get_by_repo_name(repo_name)
     if dbrepo and dbrepo.clone_uri:
         ScmModel().pull_changes(kwargs['repo_name'], kwargs['username'])
     else:
         log.debug('Repo `%s` not found or without a clone_url', repo_name)
 
 
+@async_task(ignore_result=True)
+def check_for_update():
+    from rhodecode.model.update import UpdateModel
+    update_url = UpdateModel().get_update_url()
+    cur_ver = rhodecode.__version__
+
+    try:
+        data = UpdateModel().get_update_data(update_url)
+        latest = data['versions'][0]
+        UpdateModel().store_version(latest['version'])
+    except Exception:
+        pass
+
+
 @async_task(ignore_result=False)
 def beat_check(*args, **kwargs):
     log = get_logger(beat_check)
     log.info('Got args: %r and kwargs %r', args, kwargs)
     return time.time()

rhodecode/templates/admin/settings/settings_system_update.mako

@@ -1,27 +1,26 b''
 ## -*- coding: utf-8 -*-
 ## upgrade block rendered afte on-click check
 
 <div class="alert ${'alert-warning' if c.should_upgrade else 'alert-success'}">
-<p >
-
+<p>
 %if c.should_upgrade:
     A <b>new version</b> is available:
     %if c.latest_data.get('title'):
         <b>${h.literal(c.latest_data['title'])}</b>
     %else:
         <b>${c.latest_ver}</b>
     %endif
 %else:
-    You already have the <b>latest</b> stable version.
+    This instance is already running the <b>latest</b> stable version ${c.latest_ver}.
 %endif
 </p>
 
 % if c.should_upgrade and c.important_notices:
 <div>Important notes for this release:</div>
     <ul>
     % for notice in c.important_notices:
         <li>- ${notice}</li>
     % endfor
     </ul>
 % endif
 </div>