artifacts: added reading of metadata and define basic audit-log entries for add/delete artifact

rhodecode/apps/file_store/local_store.py

@@ -1,211 +1,224 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import time
 import shutil
 import hashlib
 
 from rhodecode.lib.ext_json import json
 from rhodecode.apps.file_store import utils
 from rhodecode.apps.file_store.extensions import resolve_extensions
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException
 
 METADATA_VER = 'v1'
 
 
 class LocalFileStorage(object):
 
     @classmethod
     def resolve_name(cls, name, directory):
         """
         Resolves a unique name and the correct path. If a filename
         for that path already exists then a numeric prefix with values > 0 will be
         added, for example test.jpg -> test-1.jpg etc. initially file would have 0 prefix.
 
         :param name: base name of file
         :param directory: absolute directory path
         """
 
         basename, ext = os.path.splitext(name)
         counter = 0
         while True:
             name = '%s-%d%s' % (basename, counter, ext)
 
             # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
             sub_store = cls._sub_store_from_filename(basename)
             sub_store_path = os.path.join(directory, sub_store)
             if not os.path.exists(sub_store_path):
                 os.makedirs(sub_store_path)
 
             path = os.path.join(sub_store_path, name)
             if not os.path.exists(path):
                 return name, path
             counter += 1
 
     @classmethod
     def _sub_store_from_filename(cls, filename):
         return filename[:2]
 
     @classmethod
     def calculate_path_hash(cls, file_path):
         """
         Efficient calculation of file_path sha256 sum
 
         :param file_path:
         :return: sha256sum
         """
         digest = hashlib.sha256()
         with open(file_path, 'rb') as f:
             for chunk in iter(lambda: f.read(1024 * 100), b""):
                 digest.update(chunk)
 
         return digest.hexdigest()
 
     def __init__(self, base_path, extension_groups=None):
 
         """
         Local file storage
 
         :param base_path: the absolute base path where uploads are stored
         :param extension_groups: extensions string
         """
 
         extension_groups = extension_groups or ['any']
         self.base_path = base_path
         self.extensions = resolve_extensions([], groups=extension_groups)
 
     def store_path(self, filename):
         """
         Returns absolute file path of the filename, joined to the
         base_path.
 
         :param filename: base name of file
         """
         sub_store = self._sub_store_from_filename(filename)
         return os.path.join(self.base_path, sub_store, filename)
 
     def delete(self, filename):
         """
         Deletes the filename. Filename is resolved with the
         absolute path based on base_path. If file does not exist,
         returns **False**, otherwise **True**
 
         :param filename: base name of file
         """
         if self.exists(filename):
             os.remove(self.store_path(filename))
             return True
         return False
 
     def exists(self, filename):
         """
         Checks if file exists. Resolves filename's absolute
         path based on base_path.
 
         :param filename: base name of file
         """
         return os.path.exists(self.store_path(filename))
 
     def filename_allowed(self, filename, extensions=None):
         """Checks if a filename has an allowed extension
 
         :param filename: base name of file
         :param extensions: iterable of extensions (or self.extensions)
         """
         _, ext = os.path.splitext(filename)
         return self.extension_allowed(ext, extensions)
 
     def extension_allowed(self, ext, extensions=None):
         """
         Checks if an extension is permitted. Both e.g. ".jpg" and
         "jpg" can be passed in. Extension lookup is case-insensitive.
 
         :param ext: extension to check
         :param extensions: iterable of extensions to validate against (or self.extensions)
         """
 
         extensions = extensions or self.extensions
         if not extensions:
             return True
         if ext.startswith('.'):
             ext = ext[1:]
         return ext.lower() in extensions
 
     def save_file(self, file_obj, filename, directory=None, extensions=None,
                   extra_metadata=None, **kwargs):
         """
         Saves a file object to the uploads location.
         Returns the resolved filename, i.e. the directory +
         the (randomized/incremented) base name.
 
         :param file_obj: **cgi.FieldStorage** object (or similar)
         :param filename: original filename
         :param directory: relative path of sub-directory
         :param extensions: iterable of allowed extensions, if not default
         :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
         """
 
         extensions = extensions or self.extensions
 
         if not self.filename_allowed(filename, extensions):
             raise FileNotAllowedException()
 
         if directory:
             dest_directory = os.path.join(self.base_path, directory)
         else:
             dest_directory = self.base_path
 
         if not os.path.exists(dest_directory):
             os.makedirs(dest_directory)
 
         filename = utils.uid_filename(filename)
 
         # resolve also produces special sub-dir for file optimized store
         filename, path = self.resolve_name(filename, dest_directory)
         stored_file_dir = os.path.dirname(path)
 
         file_obj.seek(0)
 
         with open(path, "wb") as dest:
             shutil.copyfileobj(file_obj, dest)
 
         metadata = {}
         if extra_metadata:
             metadata = extra_metadata
 
         size = os.stat(path).st_size
         file_hash = self.calculate_path_hash(path)
 
         metadata.update(
             {"filename": filename,
              "size": size,
              "time": time.time(),
              "sha256": file_hash,
              "meta_ver": METADATA_VER})
 
         filename_meta = filename + '.meta'
         with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
             dest_meta.write(json.dumps(metadata))
 
         if directory:
             filename = os.path.join(directory, filename)
 
         return filename, metadata
+
+    def get_metadata(self, filename):
+        """
+        Reads JSON stored metadata for a file
+
+        :param filename:
+        :return:
+        """
+        filename = self.store_path(filename)
+        filename_meta = filename + '.meta'
+
+        with open(filename_meta, "rb") as source_meta:
+            return json.loads(source_meta.read())

rhodecode/apps/file_store/views.py

@@ -1,144 +1,144 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import logging
 
 from pyramid.view import view_config
 from pyramid.response import FileResponse
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.apps.file_store import utils
 from rhodecode.apps.file_store.exceptions import (
     FileNotAllowedException, FileOverSizeException)
 
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.auth import (CSRFRequired, NotAnonymous, HasRepoPermissionAny, HasRepoGroupPermissionAny)
 from rhodecode.model.db import Session, FileStore
 
 log = logging.getLogger(__name__)
 
 
 class FileStoreView(BaseAppView):
     upload_key = 'store_file'
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         self.storage = utils.get_file_storage(self.request.registry.settings)
         return c
 
     @NotAnonymous()
     @CSRFRequired()
     @view_config(route_name='upload_file', request_method='POST', renderer='json_ext')
     def upload_file(self):
         self.load_default_context()
         file_obj = self.request.POST.get(self.upload_key)
 
         if file_obj is None:
             return {'store_fid': None,
                     'access_path': None,
                     'error': '{} data field is missing'.format(self.upload_key)}
 
         if not hasattr(file_obj, 'filename'):
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'filename cannot be read from the data field'}
 
         filename = file_obj.filename
 
         metadata = {
             'user_uploaded': {'username': self._rhodecode_user.username,
                               'user_id': self._rhodecode_user.user_id,
                               'ip': self._rhodecode_user.ip_addr}}
         try:
-            store_fid, metadata = self.storage.save_file(
+            store_uid, metadata = self.storage.save_file(
                 file_obj.file, filename, extra_metadata=metadata)
         except FileNotAllowedException:
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} is not allowed.'.format(filename)}
 
         except FileOverSizeException:
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} is exceeding allowed limit.'.format(filename)}
 
         try:
             entry = FileStore.create(
-                file_uid=store_fid, filename=metadata["filename"],
+                file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
                 file_description='upload attachment',
                 check_acl=False, user_id=self._rhodecode_user.user_id
             )
             Session().add(entry)
             Session().commit()
             log.debug('Stored upload in DB as %s', entry)
         except Exception:
             log.exception('Failed to store file %s', filename)
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} failed to store in DB.'.format(filename)}
 
-        return {'store_fid': store_fid,
-                'access_path': h.route_path('download_file', fid=store_fid)}
+        return {'store_fid': store_uid,
+                'access_path': h.route_path('download_file', fid=store_uid)}
 
     @view_config(route_name='download_file')
     def download_file(self):
         self.load_default_context()
         file_uid = self.request.matchdict['fid']
         log.debug('Requesting FID:%s from store %s', file_uid, self.storage)
 
         if not self.storage.exists(file_uid):
             log.debug('File with FID:%s not found in the store', file_uid)
             raise HTTPNotFound()
 
         db_obj = FileStore().query().filter(FileStore.file_uid == file_uid).scalar()
         if not db_obj:
             raise HTTPNotFound()
 
         # private upload for user
         if db_obj.check_acl and db_obj.scope_user_id:
             user = db_obj.user
             if self._rhodecode_db_user.user_id != user.user_id:
                 log.warning('Access to file store object forbidden')
                 raise HTTPNotFound()
 
         # scoped to repository permissions
         if db_obj.check_acl and db_obj.scope_repo_id:
             repo = db_obj.repo
             perm_set = ['repository.read', 'repository.write', 'repository.admin']
             has_perm = HasRepoPermissionAny(*perm_set)(repo.repo_name, 'FileStore check')
             if not has_perm:
                 log.warning('Access to file store object forbidden')
                 raise HTTPNotFound()
 
         # scoped to repository group permissions
         if db_obj.check_acl and db_obj.scope_repo_group_id:
             repo_group = db_obj.repo_group
             perm_set = ['group.read', 'group.write', 'group.admin']
             has_perm = HasRepoGroupPermissionAny(*perm_set)(repo_group.group_name, 'FileStore check')
             if not has_perm:
                 log.warning('Access to file store object forbidden')
                 raise HTTPNotFound()
 
         FileStore.bump_access_counter(file_uid)
 
         file_path = self.storage.store_path(file_uid)
         return FileResponse(file_path)

rhodecode/lib/audit_logger.py

@@ -1,288 +1,291 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2017-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import datetime
 
 from rhodecode.lib.jsonalchemy import JsonRaw
 from rhodecode.model import meta
 from rhodecode.model.db import User, UserLog, Repository
 
 
 log = logging.getLogger(__name__)
 
 # action as key, and expected action_data as value
 ACTIONS_V1 = {
     'user.login.success': {'user_agent': ''},
     'user.login.failure': {'user_agent': ''},
     'user.logout': {'user_agent': ''},
     'user.register': {},
     'user.password.reset_request': {},
     'user.push': {'user_agent': '', 'commit_ids': []},
     'user.pull': {'user_agent': ''},
 
     'user.create': {'data': {}},
     'user.delete': {'old_data': {}},
     'user.edit': {'old_data': {}},
     'user.edit.permissions': {},
     'user.edit.ip.add': {'ip': {}, 'user': {}},
     'user.edit.ip.delete': {'ip': {}, 'user': {}},
     'user.edit.token.add': {'token': {}, 'user': {}},
     'user.edit.token.delete': {'token': {}, 'user': {}},
     'user.edit.email.add': {'email': ''},
     'user.edit.email.delete': {'email': ''},
     'user.edit.ssh_key.add': {'token': {}, 'user': {}},
     'user.edit.ssh_key.delete': {'token': {}, 'user': {}},
     'user.edit.password_reset.enabled': {},
     'user.edit.password_reset.disabled': {},
 
     'user_group.create': {'data': {}},
     'user_group.delete': {'old_data': {}},
     'user_group.edit': {'old_data': {}},
     'user_group.edit.permissions': {},
     'user_group.edit.member.add': {'user': {}},
     'user_group.edit.member.delete': {'user': {}},
 
     'repo.create': {'data': {}},
     'repo.fork': {'data': {}},
     'repo.edit': {'old_data': {}},
     'repo.edit.permissions': {},
     'repo.edit.permissions.branch': {},
     'repo.archive': {'old_data': {}},
     'repo.delete': {'old_data': {}},
 
     'repo.archive.download': {'user_agent': '', 'archive_name': '',
                               'archive_spec': '', 'archive_cached': ''},
 
     'repo.permissions.branch_rule.create': {},
     'repo.permissions.branch_rule.edit': {},
     'repo.permissions.branch_rule.delete': {},
 
     'repo.pull_request.create': '',
     'repo.pull_request.edit': '',
     'repo.pull_request.delete': '',
     'repo.pull_request.close': '',
     'repo.pull_request.merge': '',
     'repo.pull_request.vote': '',
     'repo.pull_request.comment.create': '',
     'repo.pull_request.comment.delete': '',
 
     'repo.pull_request.reviewer.add': '',
     'repo.pull_request.reviewer.delete': '',
 
     'repo.commit.strip': {'commit_id': ''},
     'repo.commit.comment.create': {'data': {}},
     'repo.commit.comment.delete': {'data': {}},
     'repo.commit.vote': '',
 
+    'repo.artifact.add': '',
+    'repo.artifact.delete': '',
+
     'repo_group.create': {'data': {}},
     'repo_group.edit': {'old_data': {}},
     'repo_group.edit.permissions': {},
     'repo_group.delete': {'old_data': {}},
 }
 
 ACTIONS = ACTIONS_V1
 
 SOURCE_WEB = 'source_web'
 SOURCE_API = 'source_api'
 
 
 class UserWrap(object):
     """
     Fake object used to imitate AuthUser
     """
 
     def __init__(self, user_id=None, username=None, ip_addr=None):
         self.user_id = user_id
         self.username = username
         self.ip_addr = ip_addr
 
 
 class RepoWrap(object):
     """
     Fake object used to imitate RepoObject that audit logger requires
     """
 
     def __init__(self, repo_id=None, repo_name=None):
         self.repo_id = repo_id
         self.repo_name = repo_name
 
 
 def _store_log(action_name, action_data, user_id, username, user_data,
                ip_address, repository_id, repository_name):
     user_log = UserLog()
     user_log.version = UserLog.VERSION_2
 
     user_log.action = action_name
     user_log.action_data = action_data or JsonRaw(u'{}')
 
     user_log.user_ip = ip_address
 
     user_log.user_id = user_id
     user_log.username = username
     user_log.user_data = user_data or JsonRaw(u'{}')
 
     user_log.repository_id = repository_id
     user_log.repository_name = repository_name
 
     user_log.action_date = datetime.datetime.now()
 
     return user_log
 
 
 def store_web(*args, **kwargs):
     if 'action_data' not in kwargs:
         kwargs['action_data'] = {}
     kwargs['action_data'].update({
         'source': SOURCE_WEB
     })
     return store(*args, **kwargs)
 
 
 def store_api(*args, **kwargs):
     if 'action_data' not in kwargs:
         kwargs['action_data'] = {}
     kwargs['action_data'].update({
         'source': SOURCE_API
     })
     return store(*args, **kwargs)
 
 
 def store(action, user, action_data=None, user_data=None, ip_addr=None,
           repo=None, sa_session=None, commit=False):
     """
     Audit logger for various actions made by users, typically this
     results in a call such::
 
         from rhodecode.lib import audit_logger
 
         audit_logger.store(
             'repo.edit', user=self._rhodecode_user)
         audit_logger.store(
             'repo.delete', action_data={'data': repo_data},
             user=audit_logger.UserWrap(username='itried-login', ip_addr='8.8.8.8'))
 
         # repo action
         audit_logger.store(
             'repo.delete',
             user=audit_logger.UserWrap(username='itried-login', ip_addr='8.8.8.8'),
             repo=audit_logger.RepoWrap(repo_name='some-repo'))
 
         # repo action, when we know and have the repository object already
         audit_logger.store(
             'repo.delete', action_data={'source': audit_logger.SOURCE_WEB, },
             user=self._rhodecode_user,
             repo=repo_object)
 
         # alternative wrapper to the above
         audit_logger.store_web(
             'repo.delete', action_data={},
             user=self._rhodecode_user,
             repo=repo_object)
 
         # without an user ?
         audit_logger.store(
             'user.login.failure',
             user=audit_logger.UserWrap(
                     username=self.request.params.get('username'),
                     ip_addr=self.request.remote_addr))
 
     """
     from rhodecode.lib.utils2 import safe_unicode
     from rhodecode.lib.auth import AuthUser
 
     action_spec = ACTIONS.get(action, None)
     if action_spec is None:
         raise ValueError('Action `{}` is not supported'.format(action))
 
     if not sa_session:
         sa_session = meta.Session()
 
     try:
         username = getattr(user, 'username', None)
         if not username:
             pass
 
         user_id = getattr(user, 'user_id', None)
         if not user_id:
             # maybe we have username ? Try to figure user_id from username
             if username:
                 user_id = getattr(
                     User.get_by_username(username), 'user_id', None)
 
         ip_addr = ip_addr or getattr(user, 'ip_addr', None)
         if not ip_addr:
             pass
 
         if not user_data:
             # try to get this from the auth user
             if isinstance(user, AuthUser):
                 user_data = {
                     'username': user.username,
                     'email': user.email,
                 }
 
         repository_name = getattr(repo, 'repo_name', None)
         repository_id = getattr(repo, 'repo_id', None)
         if not repository_id:
             # maybe we have repo_name ? Try to figure repo_id from repo_name
             if repository_name:
                 repository_id = getattr(
                     Repository.get_by_repo_name(repository_name), 'repo_id', None)
 
         action_name = safe_unicode(action)
         ip_address = safe_unicode(ip_addr)
 
         with sa_session.no_autoflush:
             update_user_last_activity(sa_session, user_id)
 
             user_log = _store_log(
                 action_name=action_name,
                 action_data=action_data or {},
                 user_id=user_id,
                 username=username,
                 user_data=user_data or {},
                 ip_address=ip_address,
                 repository_id=repository_id,
                 repository_name=repository_name
             )
 
             sa_session.add(user_log)
 
             if commit:
                 sa_session.commit()
 
         entry_id = user_log.entry_id or ''
         log.info('AUDIT[%s]: Logging action: `%s` by user:id:%s[%s] ip:%s',
                  entry_id, action_name, user_id, username, ip_address)
 
     except Exception:
         log.exception('AUDIT: failed to store audit log')
 
 
 def update_user_last_activity(sa_session, user_id):
     _last_activity = datetime.datetime.now()
     try:
         sa_session.query(User).filter(User.user_id == user_id).update(
             {"last_activity": _last_activity})
         log.debug(
             'updated user `%s` last activity to:%s', user_id, _last_activity)
     except Exception:
         log.exception("Failed last activity update")