rcextensions: added example how to validate file size and name using rcextensions

rhodecode/config/rcextensions/examples/validate_pushed_files_name_and_size.py

@@ -0,0 +1,110 b''
+# Example to validate pushed files names and size using some sort of rules
+
+
+
+@has_kwargs({
+    'server_url': 'url of instance that triggered this hook',
+    'config': 'path to .ini config used',
+    'scm': 'type of version control "git", "hg", "svn"',
+    'username': 'username of actor who triggered this event',
+    'ip': 'ip address of actor who triggered this hook',
+    'action': '',
+    'repository': 'repository name',
+    'repo_store_path': 'full path to where repositories are stored',
+    'commit_ids': 'pre transaction metadata for commit ids',
+    'hook_type': '',
+    'user_agent': 'Client user agent, e.g git or mercurial CLI version',
+})
+def _pre_push_hook(*args, **kwargs):
+    """
+    Post push hook
+    To stop version control from storing the transaction and send a message to user
+    use non-zero HookResponse with a message, e.g return HookResponse(1, 'Not allowed')
+
+    This message will be shown back to client during PUSH operation
+
+    Commit ids might look like that::
+
+    [{u'hg_env|git_env': ...,
+      u'multiple_heads': [],
+      u'name': u'default',
+      u'new_rev': u'd0befe0692e722e01d5677f27a104631cf798b69',
+      u'old_rev': u'd0befe0692e722e01d5677f27a104631cf798b69',
+      u'ref': u'',
+      u'total_commits': 2,
+      u'type': u'branch'}]
+    """
+    import fnmatch
+    from .helpers import extra_fields, extract_pre_files
+    from .utils import str2bool, aslist
+    from rhodecode.lib.helpers import format_byte_size_binary
+
+    # returns list of dicts with key-val fetched from extra fields
+    repo_extra_fields = extra_fields.run(**kwargs)
+
+    # optionally use 'extra fields' to control the logic per repo
+    # e.g store a list of patterns to be forbidden e.g `*.exe, *.dump`
+    forbid_files = repo_extra_fields.get('forbid_files_glob', {}).get('field_value')
+    forbid_files = aslist(forbid_files)
+
+    # optionally get bytes limit for a single file, e.g 1024 for 1KB
+    forbid_size_over = repo_extra_fields.get('forbid_size_over', {}).get('field_value')
+    forbid_size_over = int(forbid_size_over or 0)
+
+    def validate_file_name_and_size(file_data, forbidden_files=None, size_limit=None):
+        """
+        This function validates commited files against some sort of rules.
+        It should return a valid boolean, and a reason for failure
+
+        file_data =[
+            'raw_diff', 'old_revision', 'stats', 'original_filename', 'is_limited_diff',
+            'chunks', 'new_revision', 'operation', 'exceeds_limit', 'filename'
+            ]
+        file_data['ops'] = {
+        # is file binary
+        'binary': False,
+
+        # lines
+        'added': 32,
+        'deleted': 0
+
+        'ops': {3: 'modified file'},
+        'new_mode': '100644',
+        'old_mode': None
+        }
+        """
+        file_name = file_data['filename']
+        operation = file_data['operation']  # can be A(dded), M(odified), D(eleted)
+
+        # check files names
+        if forbidden_files:
+            reason = 'File {} is forbidden to be pushed'.format(file_name)
+            for forbidden_pattern in forbid_files:
+                # here we can also filter for operation, e.g if check for only ADDED files
+                # if operation == 'A':
+                if fnmatch.fnmatch(file_name, forbidden_pattern):
+                    return False, reason
+
+        # validate A(dded) files and size
+        if size_limit and operation == 'A':
+            size = len(file_data['raw_diff'])
+
+            reason = 'File {} size of {} bytes exceeds limit {}'.format(
+                file_name, format_byte_size_binary(size),
+                format_byte_size_binary(size_limit))
+            if size > size_limit:
+                return False, reason
+
+        return True, ''
+
+    if forbid_files or forbid_size_over:
+        # returns list of dicts with key-val fetched from extra fields
+        file_list = extract_pre_files.run(**kwargs)
+
+        for file_data in file_list:
+            file_valid, reason = validate_file_name_and_size(
+                file_data, forbid_files, forbid_size_over)
+            if not file_valid:
+                return HookResponse(1, reason)
+
+    return HookResponse(0, '')

rhodecode/config/rcextensions/helpers/extract_pre_files.py

@@ -0,0 +1,96 b''
+# -*- coding: utf-8 -*-
+# Copyright (C) 2016-2019 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+"""
+us in hooks::
+
+    from .helpers import extract_pre_files
+    # returns list of dicts with key-val fetched from extra fields
+    file_list = extract_pre_files.run(**kwargs)
+
+"""
+import re
+import collections
+import json
+
+from rhodecode.lib import diffs
+from rhodecode.lib.vcs.backends.hg.diff import MercurialDiff
+from rhodecode.lib.vcs.backends.git.diff import GitDiff
+
+
+def get_hg_files(repo, refs):
+    files = []
+    return files
+
+
+def get_git_files(repo, refs):
+    files = []
+
+    for data in refs:
+        # we should now extract commit data
+        old_rev = data['old_rev']
+        new_rev = data['new_rev']
+
+        if '00000000' in old_rev:
+            # new branch, we don't need to extract nothing
+            return files
+
+        git_env = dict(data['git_env'])
+
+        cmd = [
+            'diff', old_rev, new_rev
+        ]
+
+        stdout, stderr = repo.run_git_command(cmd, extra_env=git_env)
+        vcs_diff = GitDiff(stdout)
+
+        diff_processor = diffs.DiffProcessor(vcs_diff, format='newdiff')
+        # this is list of dicts with diff information
+        # _parsed[0].keys()
+        # ['raw_diff', 'old_revision', 'stats', 'original_filename',
+        #  'is_limited_diff', 'chunks', 'new_revision', 'operation',
+        #  'exceeds_limit', 'filename']
+        files = _parsed = diff_processor.prepare()
+
+    return files
+
+
+def run(*args, **kwargs):
+    from rhodecode.model.db import Repository
+
+    vcs_type = kwargs['scm']
+    # use temp name then the main one propagated
+    repo_name = kwargs.pop('REPOSITORY', None) or kwargs['repository']
+
+    repo = Repository.get_by_repo_name(repo_name)
+    vcs_repo = repo.scm_instance(cache=False)
+
+    files = []
+
+    if vcs_type == 'git':
+        for rev_data in kwargs['commit_ids']:
+            new_environ = dict((k, v) for k, v in rev_data['git_env'])
+        files = get_git_files(vcs_repo, kwargs['commit_ids'])
+
+    if vcs_type == 'hg':
+        for rev_data in kwargs['commit_ids']:
+            new_environ = dict((k, v) for k, v in rev_data['hg_env'])
+        files = get_hg_files(vcs_repo, kwargs['commit_ids'])
+
+    return files

rhodecode/config/rcextensions/examples/validate_commit_message_author.py

@@ -1,92 +1,79 b''
 # Example to validate commit message or author using some sort of rules
 
 
 @has_kwargs({
     'server_url': 'url of instance that triggered this hook',
     'config': 'path to .ini config used',
     'scm': 'type of version control "git", "hg", "svn"',
     'username': 'username of actor who triggered this event',
     'ip': 'ip address of actor who triggered this hook',
     'action': '',
     'repository': 'repository name',
     'repo_store_path': 'full path to where repositories are stored',
     'commit_ids': 'pre transaction metadata for commit ids',
     'hook_type': '',
     'user_agent': 'Client user agent, e.g git or mercurial CLI version',
 })
-@has_kwargs({
-    'server_url': 'url of instance that triggered this hook',
-    'config': 'path to .ini config used',
-    'scm': 'type of version control "git", "hg", "svn"',
-    'username': 'username of actor who triggered this event',
-    'ip': 'ip address of actor who triggered this hook',
-    'action': '',
-    'repository': 'repository name',
-    'repo_store_path': 'full path to where repositories are stored',
-    'commit_ids': 'pre transaction metadata for commit ids',
-    'hook_type': '',
-    'user_agent': 'Client user agent, e.g git or mercurial CLI version',
-})
 def _pre_push_hook(*args, **kwargs):
     """
     Post push hook
     To stop version control from storing the transaction and send a message to user
     use non-zero HookResponse with a message, e.g return HookResponse(1, 'Not allowed')
 
     This message will be shown back to client during PUSH operation
 
     Commit ids might look like that::
 
     [{u'hg_env|git_env': ...,
       u'multiple_heads': [],
       u'name': u'default',
       u'new_rev': u'd0befe0692e722e01d5677f27a104631cf798b69',
       u'old_rev': u'd0befe0692e722e01d5677f27a104631cf798b69',
       u'ref': u'',
       u'total_commits': 2,
       u'type': u'branch'}]
     """
     import re
     from .helpers import extra_fields, extract_pre_commits
     from .utils import str2bool
 
     # returns list of dicts with key-val fetched from extra fields
     repo_extra_fields = extra_fields.run(**kwargs)
 
     # optionally use 'extra fields' to control the logic per repo
     validate_author = repo_extra_fields.get('validate_author', {}).get('field_value')
     should_validate = str2bool(validate_author)
 
     # optionally store validation regex into extra fields
     validation_regex = repo_extra_fields.get('validation_regex', {}).get('field_value')
 
     def validate_commit_message(commit_message, message_regex=None):
         """
         This function validates commit_message against some sort of rules.
         It should return a valid boolean, and a reason for failure
         """
 
         if "secret_string" in commit_message:
             msg = "!!Push forbidden: secret string found in commit messages"
             return False, msg
 
         if validation_regex:
             regexp = re.compile(validation_regex)
             if not regexp.match(message):
                 msg = "!!Push forbidden: commit message does not match regexp"
                 return False, msg
 
         return True, ''
 
     if should_validate:
         # returns list of dicts with key-val fetched from extra fields
         commit_list = extract_pre_commits.run(**kwargs)
 
         for commit_data in commit_list:
             message = commit_data['message']
 
             message_valid, reason = validate_commit_message(message, validation_regex)
             if not message_valid:
                 return HookResponse(1, reason)
 
     return HookResponse(0, '')