maintainance: add repack and fsck for git maintainance execution list.

rhodecode/apps/repository/views/repo_maintainance.py

@@ -1,64 +1,62 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 from pyramid.view import view_config
 
 from rhodecode.apps._base import RepoAppView
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib import repo_maintenance
 
 log = logging.getLogger(__name__)
 
 
 class RepoMaintenanceView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-
-
         return c
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @view_config(
         route_name='edit_repo_maintenance', request_method='GET',
         renderer='rhodecode:templates/admin/repos/repo_edit.mako')
     def repo_maintenance(self):
         c = self.load_default_context()
         c.active = 'maintenance'
         maintenance = repo_maintenance.RepoMaintenance()
         c.executable_tasks = maintenance.get_tasks_for_repo(self.db_repo)
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @view_config(
         route_name='edit_repo_maintenance_execute', request_method='GET',
         renderer='json', xhr=True)
     def repo_maintenance_execute(self):
         c = self.load_default_context()
         c.active = 'maintenance'
         _ = self.request.translate
 
         maintenance = repo_maintenance.RepoMaintenance()
         executed_types = maintenance.execute(self.db_repo)
 
         return executed_types

rhodecode/lib/repo_maintenance.py

@@ -1,118 +1,175 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2017-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import logging
 
 log = logging.getLogger(__name__)
 
 
 class MaintenanceTask(object):
     human_name = 'undefined'
 
     def __init__(self, db_repo):
         self.db_repo = db_repo
 
     def run(self):
         """Execute task and return task human value"""
         raise NotImplementedError()
 
 
 class GitGC(MaintenanceTask):
     human_name = 'GIT Garbage collect'
 
     def _count_objects(self, repo):
         stdout, stderr = repo.run_git_command(
             ['count-objects', '-v'], fail_on_stderr=False)
 
         errors = ' '
         objects = ' '.join(stdout.splitlines())
 
         if stderr:
             errors = '\nSTD ERR:' + '\n'.join(stderr.splitlines())
         return objects + errors
 
     def run(self):
         output = []
         instance = self.db_repo.scm_instance()
 
-        objects = self._count_objects(instance)
-        output.append(objects)
-        log.debug('GIT objects:%s', objects)
-
-        stdout, stderr = instance.run_git_command(
-            ['gc', '--aggressive'], fail_on_stderr=False)
+        objects_before = self._count_objects(instance)
 
-        out = 'executed git gc --aggressive'
-        if stderr:
-            out = ''.join(stderr.splitlines())
+        log.debug('GIT objects:%s', objects_before)
+        cmd = ['gc', '--aggressive']
+        stdout, stderr = instance.run_git_command(cmd, fail_on_stderr=False)
 
-        elif stdout:
-            out = ''.join(stdout.splitlines())
-
+        out = 'executed {}'.format(' '.join(cmd))
         output.append(out)
 
-        objects = self._count_objects(instance)
-        log.debug('GIT objects:%s', objects)
-        output.append(objects)
+        out = ''
+        if stderr:
+            out += ''.join(stderr.splitlines())
+
+        if stdout:
+            out += ''.join(stdout.splitlines())
+
+        if out:
+            output.append(out)
+
+        objects_after = self._count_objects(instance)
+        log.debug('GIT objects:%s', objects_after)
+        output.append('objects before :' + objects_before)
+        output.append('objects after  :' + objects_after)
+
+        return '\n'.join(output)
+
+
+class GitFSCK(MaintenanceTask):
+    human_name = 'GIT FSCK'
+
+    def run(self):
+        output = []
+        instance = self.db_repo.scm_instance()
+
+        cmd = ['fsck', '--full']
+        stdout, stderr = instance.run_git_command(cmd, fail_on_stderr=False)
+
+        out = 'executed {}'.format(' '.join(cmd))
+        output.append(out)
+
+        out = ''
+        if stderr:
+            out += ''.join(stderr.splitlines())
+
+        if stdout:
+            out += ''.join(stdout.splitlines())
+
+        if out:
+            output.append(out)
+
+        return '\n'.join(output)
+
+
+class GitRepack(MaintenanceTask):
+    human_name = 'GIT Repack'
+
+    def run(self):
+        output = []
+        instance = self.db_repo.scm_instance()
+        cmd = ['repack', '-a', '-d',
+               '--window-memory', '10m', '--max-pack-size', '100m']
+        stdout, stderr = instance.run_git_command(cmd, fail_on_stderr=False)
+
+        out = 'executed {}'.format(' '.join(cmd))
+        output.append(out)
+        out = ''
+
+        if stderr:
+            out += ''.join(stderr.splitlines())
+
+        if stdout:
+            out += ''.join(stdout.splitlines())
+
+        if out:
+            output.append(out)
 
         return '\n'.join(output)
 
 
 class HGVerify(MaintenanceTask):
     human_name = 'HG Verify repo'
 
     def run(self):
         instance = self.db_repo.scm_instance()
         res = instance.verify()
         return res
 
 
 class SVNVerify(MaintenanceTask):
     human_name = 'SVN Verify repo'
 
     def run(self):
         instance = self.db_repo.scm_instance()
         res = instance.verify()
         return res
 
 
 class RepoMaintenance(object):
     """
     Performs maintenance of repository based on it's type
     """
     tasks = {
         'hg': [HGVerify],
-        'git': [GitGC],
+        'git': [GitFSCK, GitGC, GitRepack],
         'svn': [SVNVerify],
     }
 
     def get_tasks_for_repo(self, db_repo):
         """
         fetches human names of tasks pending for execution for given type of repo
         """
         tasks = []
         for task in self.tasks[db_repo.repo_type]:
             tasks.append(task.human_name)
         return tasks
 
     def execute(self, db_repo):
         executed_tasks = []
         for task in self.tasks[db_repo.repo_type]:
-            executed_tasks.append(task(db_repo).run())
+            output = task.human_name + ':\n' + task(db_repo).run() + '\n--\n'
+            executed_tasks.append(output)
         return executed_tasks