Show More
@@ -0,0 +1,49 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 | import os | |||
|
21 | from rhodecode.apps.upload_store import config_keys | |||
|
22 | from rhodecode.config.middleware import _bool_setting, _string_setting | |||
|
23 | ||||
|
24 | ||||
|
25 | def _sanitize_settings_and_apply_defaults(settings): | |||
|
26 | """ | |||
|
27 | Set defaults, convert to python types and validate settings. | |||
|
28 | """ | |||
|
29 | _bool_setting(settings, config_keys.enabled, 'true') | |||
|
30 | ||||
|
31 | _string_setting(settings, config_keys.backend, 'local') | |||
|
32 | ||||
|
33 | default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store') | |||
|
34 | _string_setting(settings, config_keys.store_path, default_store) | |||
|
35 | ||||
|
36 | ||||
|
37 | def includeme(config): | |||
|
38 | settings = config.registry.settings | |||
|
39 | _sanitize_settings_and_apply_defaults(settings) | |||
|
40 | ||||
|
41 | config.add_route( | |||
|
42 | name='upload_file', | |||
|
43 | pattern='/_file_store/upload') | |||
|
44 | config.add_route( | |||
|
45 | name='download_file', | |||
|
46 | pattern='/_file_store/download/{fid}') | |||
|
47 | ||||
|
48 | # Scan module for configuration decorators. | |||
|
49 | config.scan('.views', ignore='.tests') |
@@ -0,0 +1,27 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 | ||||
|
21 | ||||
|
22 | # Definition of setting keys used to configure this module. Defined here to | |||
|
23 | # avoid repetition of keys throughout the module. | |||
|
24 | ||||
|
25 | enabled = 'file_store.enabled' | |||
|
26 | backend = 'file_store.backend' | |||
|
27 | store_path = 'file_store.storage_path' |
@@ -0,0 +1,31 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 | ||||
|
21 | ||||
|
22 | class FileNotAllowedException(Exception): | |||
|
23 | """ | |||
|
24 | Thrown if file does not have an allowed extension. | |||
|
25 | """ | |||
|
26 | ||||
|
27 | ||||
|
28 | class FileOverSizeException(Exception): | |||
|
29 | """ | |||
|
30 | Thrown if file is over the set limit. | |||
|
31 | """ |
@@ -0,0 +1,66 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 | ||||
|
21 | ||||
|
22 | ANY = [] | |||
|
23 | TEXT_EXT = ['txt', 'md', 'rst', 'log'] | |||
|
24 | DOCUMENTS_EXT = ['pdf', 'rtf', 'odf', 'ods', 'gnumeric', 'abw', 'doc', 'docx', 'xls', 'xlsx'] | |||
|
25 | IMAGES_EXT = ['jpg', 'jpe', 'jpeg', 'png', 'gif', 'svg', 'bmp', 'tiff'] | |||
|
26 | AUDIO_EXT = ['wav', 'mp3', 'aac', 'ogg', 'oga', 'flac'] | |||
|
27 | VIDEO_EXT = ['mpeg', '3gp', 'avi', 'divx', 'dvr', 'flv', 'mp4', 'wmv'] | |||
|
28 | DATA_EXT = ['csv', 'ini', 'json', 'plist', 'xml', 'yaml', 'yml'] | |||
|
29 | SCRIPTS_EXT = ['js', 'php', 'pl', 'py', 'rb', 'sh', 'go', 'c', 'h'] | |||
|
30 | ARCHIVES_EXT = ['gz', 'bz2', 'zip', 'tar', 'tgz', 'txz', '7z'] | |||
|
31 | EXECUTABLES_EXT = ['so', 'exe', 'dll'] | |||
|
32 | ||||
|
33 | ||||
|
34 | DEFAULT = DOCUMENTS_EXT + TEXT_EXT + IMAGES_EXT + DATA_EXT | |||
|
35 | ||||
|
36 | GROUPS = dict(( | |||
|
37 | ('any', ANY), | |||
|
38 | ('text', TEXT_EXT), | |||
|
39 | ('documents', DOCUMENTS_EXT), | |||
|
40 | ('images', IMAGES_EXT), | |||
|
41 | ('audio', AUDIO_EXT), | |||
|
42 | ('video', VIDEO_EXT), | |||
|
43 | ('data', DATA_EXT), | |||
|
44 | ('scripts', SCRIPTS_EXT), | |||
|
45 | ('archives', ARCHIVES_EXT), | |||
|
46 | ('executables', EXECUTABLES_EXT), | |||
|
47 | ('default', DEFAULT), | |||
|
48 | )) | |||
|
49 | ||||
|
50 | ||||
|
51 | def resolve_extensions(extensions, groups=None): | |||
|
52 | """ | |||
|
53 | Calculate allowed extensions based on a list of extensions provided, and optional | |||
|
54 | groups of extensions from the available lists. | |||
|
55 | ||||
|
56 | :param extensions: a list of extensions e.g ['py', 'txt'] | |||
|
57 | :param groups: additionally groups to extend the extensions. | |||
|
58 | """ | |||
|
59 | groups = groups or [] | |||
|
60 | valid_exts = set([x.lower() for x in extensions]) | |||
|
61 | ||||
|
62 | for group in groups: | |||
|
63 | if group in GROUPS: | |||
|
64 | valid_exts.update(GROUPS[group]) | |||
|
65 | ||||
|
66 | return valid_exts |
@@ -0,0 +1,167 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 | ||||
|
21 | import os | |||
|
22 | import shutil | |||
|
23 | ||||
|
24 | from rhodecode.lib.ext_json import json | |||
|
25 | from rhodecode.apps.upload_store import utils | |||
|
26 | from rhodecode.apps.upload_store.extensions import resolve_extensions | |||
|
27 | from rhodecode.apps.upload_store.exceptions import FileNotAllowedException | |||
|
28 | ||||
|
29 | ||||
|
30 | class LocalFileStorage(object): | |||
|
31 | ||||
|
32 | @classmethod | |||
|
33 | def resolve_name(cls, name, directory): | |||
|
34 | """ | |||
|
35 | Resolves a unique name and the correct path. If a filename | |||
|
36 | for that path already exists then a numeric prefix with values > 0 will be | |||
|
37 | added, for example test.jpg -> test-1.jpg etc. initially file would have 0 prefix. | |||
|
38 | ||||
|
39 | :param name: base name of file | |||
|
40 | :param directory: absolute directory path | |||
|
41 | """ | |||
|
42 | ||||
|
43 | basename, ext = os.path.splitext(name) | |||
|
44 | counter = 0 | |||
|
45 | while True: | |||
|
46 | name = '%s-%d%s' % (basename, counter, ext) | |||
|
47 | path = os.path.join(directory, name) | |||
|
48 | if not os.path.exists(path): | |||
|
49 | return name, path | |||
|
50 | counter += 1 | |||
|
51 | ||||
|
52 | def __init__(self, base_path, extension_groups=None): | |||
|
53 | ||||
|
54 | """ | |||
|
55 | Local file storage | |||
|
56 | ||||
|
57 | :param base_path: the absolute base path where uploads are stored | |||
|
58 | :param extension_groups: extensions string | |||
|
59 | """ | |||
|
60 | ||||
|
61 | extension_groups = extension_groups or ['any'] | |||
|
62 | self.base_path = base_path | |||
|
63 | self.extensions = resolve_extensions([], groups=extension_groups) | |||
|
64 | ||||
|
65 | def store_path(self, filename): | |||
|
66 | """ | |||
|
67 | Returns absolute file path of the filename, joined to the | |||
|
68 | base_path. | |||
|
69 | ||||
|
70 | :param filename: base name of file | |||
|
71 | """ | |||
|
72 | return os.path.join(self.base_path, filename) | |||
|
73 | ||||
|
74 | def delete(self, filename): | |||
|
75 | """ | |||
|
76 | Deletes the filename. Filename is resolved with the | |||
|
77 | absolute path based on base_path. If file does not exist, | |||
|
78 | returns **False**, otherwise **True** | |||
|
79 | ||||
|
80 | :param filename: base name of file | |||
|
81 | """ | |||
|
82 | if self.exists(filename): | |||
|
83 | os.remove(self.store_path(filename)) | |||
|
84 | return True | |||
|
85 | return False | |||
|
86 | ||||
|
87 | def exists(self, filename): | |||
|
88 | """ | |||
|
89 | Checks if file exists. Resolves filename's absolute | |||
|
90 | path based on base_path. | |||
|
91 | ||||
|
92 | :param filename: base name of file | |||
|
93 | """ | |||
|
94 | return os.path.exists(self.store_path(filename)) | |||
|
95 | ||||
|
96 | def filename_allowed(self, filename, extensions=None): | |||
|
97 | """Checks if a filename has an allowed extension | |||
|
98 | ||||
|
99 | :param filename: base name of file | |||
|
100 | :param extensions: iterable of extensions (or self.extensions) | |||
|
101 | """ | |||
|
102 | _, ext = os.path.splitext(filename) | |||
|
103 | return self.extension_allowed(ext, extensions) | |||
|
104 | ||||
|
105 | def extension_allowed(self, ext, extensions=None): | |||
|
106 | """ | |||
|
107 | Checks if an extension is permitted. Both e.g. ".jpg" and | |||
|
108 | "jpg" can be passed in. Extension lookup is case-insensitive. | |||
|
109 | ||||
|
110 | :param extensions: iterable of extensions (or self.extensions) | |||
|
111 | """ | |||
|
112 | ||||
|
113 | extensions = extensions or self.extensions | |||
|
114 | if not extensions: | |||
|
115 | return True | |||
|
116 | if ext.startswith('.'): | |||
|
117 | ext = ext[1:] | |||
|
118 | return ext.lower() in extensions | |||
|
119 | ||||
|
120 | def save_file(self, file_obj, filename, directory=None, extensions=None, | |||
|
121 | metadata=None, **kwargs): | |||
|
122 | """ | |||
|
123 | Saves a file object to the uploads location. | |||
|
124 | Returns the resolved filename, i.e. the directory + | |||
|
125 | the (randomized/incremented) base name. | |||
|
126 | ||||
|
127 | :param file_obj: **cgi.FieldStorage** object (or similar) | |||
|
128 | :param filename: original filename | |||
|
129 | :param directory: relative path of sub-directory | |||
|
130 | :param extensions: iterable of allowed extensions, if not default | |||
|
131 | :param metadata: JSON metadata to store next to the file with .meta suffix | |||
|
132 | :returns: modified filename | |||
|
133 | """ | |||
|
134 | ||||
|
135 | extensions = extensions or self.extensions | |||
|
136 | ||||
|
137 | if not self.filename_allowed(filename, extensions): | |||
|
138 | raise FileNotAllowedException() | |||
|
139 | ||||
|
140 | if directory: | |||
|
141 | dest_directory = os.path.join(self.base_path, directory) | |||
|
142 | else: | |||
|
143 | dest_directory = self.base_path | |||
|
144 | ||||
|
145 | if not os.path.exists(dest_directory): | |||
|
146 | os.makedirs(dest_directory) | |||
|
147 | ||||
|
148 | filename = utils.uid_filename(filename) | |||
|
149 | ||||
|
150 | filename, path = self.resolve_name(filename, dest_directory) | |||
|
151 | filename_meta = filename + '.meta' | |||
|
152 | ||||
|
153 | file_obj.seek(0) | |||
|
154 | ||||
|
155 | with open(path, "wb") as dest: | |||
|
156 | shutil.copyfileobj(file_obj, dest) | |||
|
157 | ||||
|
158 | if metadata: | |||
|
159 | size = os.stat(path).st_size | |||
|
160 | metadata.update({'size': size}) | |||
|
161 | with open(os.path.join(dest_directory, filename_meta), "wb") as dest_meta: | |||
|
162 | dest_meta.write(json.dumps(metadata)) | |||
|
163 | ||||
|
164 | if directory: | |||
|
165 | filename = os.path.join(directory, filename) | |||
|
166 | ||||
|
167 | return filename |
@@ -0,0 +1,20 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 |
@@ -0,0 +1,110 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2010-2019 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 | import os | |||
|
21 | import pytest | |||
|
22 | ||||
|
23 | from rhodecode.lib.ext_json import json | |||
|
24 | from rhodecode.tests import TestController | |||
|
25 | from rhodecode.apps.upload_store import utils, config_keys | |||
|
26 | ||||
|
27 | ||||
|
28 | def route_path(name, params=None, **kwargs): | |||
|
29 | import urllib | |||
|
30 | ||||
|
31 | base_url = { | |||
|
32 | 'upload_file': '/_file_store/upload', | |||
|
33 | 'download_file': '/_file_store/download/{fid}', | |||
|
34 | ||||
|
35 | }[name].format(**kwargs) | |||
|
36 | ||||
|
37 | if params: | |||
|
38 | base_url = '{}?{}'.format(base_url, urllib.urlencode(params)) | |||
|
39 | return base_url | |||
|
40 | ||||
|
41 | ||||
|
42 | class TestFileStoreViews(TestController): | |||
|
43 | ||||
|
44 | @pytest.mark.parametrize("fid, content, exists", [ | |||
|
45 | ('abcde-0.jpg', "xxxxx", True), | |||
|
46 | ('abcde-0.exe', "1234567", True), | |||
|
47 | ('abcde-0.jpg', "xxxxx", False), | |||
|
48 | ]) | |||
|
49 | def test_get_files_from_store(self, fid, content, exists, tmpdir): | |||
|
50 | self.log_user() | |||
|
51 | store_path = self.app._pyramid_settings[config_keys.store_path] | |||
|
52 | ||||
|
53 | if exists: | |||
|
54 | status = 200 | |||
|
55 | store = utils.get_file_storage({config_keys.store_path: store_path}) | |||
|
56 | filesystem_file = os.path.join(str(tmpdir), fid) | |||
|
57 | with open(filesystem_file, 'wb') as f: | |||
|
58 | f.write(content) | |||
|
59 | ||||
|
60 | with open(filesystem_file, 'rb') as f: | |||
|
61 | fid = store.save_file(f, fid, metadata={'filename': fid}) | |||
|
62 | ||||
|
63 | else: | |||
|
64 | status = 404 | |||
|
65 | ||||
|
66 | response = self.app.get(route_path('download_file', fid=fid), status=status) | |||
|
67 | ||||
|
68 | if exists: | |||
|
69 | assert response.text == content | |||
|
70 | metadata = os.path.join(store_path, fid + '.meta') | |||
|
71 | assert os.path.exists(metadata) | |||
|
72 | with open(metadata, 'rb') as f: | |||
|
73 | json_data = json.loads(f.read()) | |||
|
74 | ||||
|
75 | assert json_data | |||
|
76 | assert 'size' in json_data | |||
|
77 | ||||
|
78 | def test_upload_files_without_content_to_store(self): | |||
|
79 | self.log_user() | |||
|
80 | response = self.app.post( | |||
|
81 | route_path('upload_file'), | |||
|
82 | params={'csrf_token': self.csrf_token}, | |||
|
83 | status=200) | |||
|
84 | ||||
|
85 | assert response.json == { | |||
|
86 | u'error': u'store_file data field is missing', | |||
|
87 | u'access_path': None, | |||
|
88 | u'store_fid': None} | |||
|
89 | ||||
|
90 | def test_upload_files_bogus_content_to_store(self): | |||
|
91 | self.log_user() | |||
|
92 | response = self.app.post( | |||
|
93 | route_path('upload_file'), | |||
|
94 | params={'csrf_token': self.csrf_token, 'store_file': 'bogus'}, | |||
|
95 | status=200) | |||
|
96 | ||||
|
97 | assert response.json == { | |||
|
98 | u'error': u'filename cannot be read from the data field', | |||
|
99 | u'access_path': None, | |||
|
100 | u'store_fid': None} | |||
|
101 | ||||
|
102 | def test_upload_content_to_store(self): | |||
|
103 | self.log_user() | |||
|
104 | response = self.app.post( | |||
|
105 | route_path('upload_file'), | |||
|
106 | upload_files=[('store_file', 'myfile.txt', 'SOME CONTENT')], | |||
|
107 | params={'csrf_token': self.csrf_token}, | |||
|
108 | status=200) | |||
|
109 | ||||
|
110 | assert response.json['store_fid'] |
@@ -0,0 +1,47 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 | ||||
|
21 | ||||
|
22 | import os | |||
|
23 | import uuid | |||
|
24 | ||||
|
25 | ||||
|
26 | def get_file_storage(settings): | |||
|
27 | from rhodecode.apps.upload_store.local_store import LocalFileStorage | |||
|
28 | from rhodecode.apps.upload_store import config_keys | |||
|
29 | store_path = settings.get(config_keys.store_path) | |||
|
30 | return LocalFileStorage(base_path=store_path) | |||
|
31 | ||||
|
32 | ||||
|
33 | def uid_filename(filename, randomized=True): | |||
|
34 | """ | |||
|
35 | Generates a randomized or stable (uuid) filename, | |||
|
36 | preserving the original extension. | |||
|
37 | ||||
|
38 | :param filename: the original filename | |||
|
39 | :param randomized: define if filename should be stable (sha1 based) or randomized | |||
|
40 | """ | |||
|
41 | _, ext = os.path.splitext(filename) | |||
|
42 | if randomized: | |||
|
43 | uid = uuid.uuid4() | |||
|
44 | else: | |||
|
45 | hash_key = '{}.{}'.format(filename, 'store') | |||
|
46 | uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key) | |||
|
47 | return str(uid) + ext.lower() |
@@ -0,0 +1,97 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | ||||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |||
|
4 | # | |||
|
5 | # This program is free software: you can redistribute it and/or modify | |||
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |||
|
7 | # (only), as published by the Free Software Foundation. | |||
|
8 | # | |||
|
9 | # This program is distributed in the hope that it will be useful, | |||
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
|
12 | # GNU General Public License for more details. | |||
|
13 | # | |||
|
14 | # You should have received a copy of the GNU Affero General Public License | |||
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |||
|
16 | # | |||
|
17 | # This program is dual-licensed. If you wish to learn more about the | |||
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |||
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |||
|
20 | import logging | |||
|
21 | ||||
|
22 | from pyramid.view import view_config | |||
|
23 | from pyramid.response import FileResponse | |||
|
24 | from pyramid.httpexceptions import HTTPFound, HTTPNotFound | |||
|
25 | ||||
|
26 | from rhodecode.apps._base import BaseAppView | |||
|
27 | from rhodecode.apps.upload_store import utils | |||
|
28 | from rhodecode.apps.upload_store.exceptions import ( | |||
|
29 | FileNotAllowedException,FileOverSizeException) | |||
|
30 | ||||
|
31 | from rhodecode.lib import helpers as h | |||
|
32 | from rhodecode.lib import audit_logger | |||
|
33 | from rhodecode.lib.auth import (CSRFRequired, NotAnonymous) | |||
|
34 | ||||
|
35 | log = logging.getLogger(__name__) | |||
|
36 | ||||
|
37 | ||||
|
38 | class FileStoreView(BaseAppView): | |||
|
39 | upload_key = 'store_file' | |||
|
40 | ||||
|
41 | def load_default_context(self): | |||
|
42 | c = self._get_local_tmpl_context() | |||
|
43 | self.storage = utils.get_file_storage(self.request.registry.settings) | |||
|
44 | return c | |||
|
45 | ||||
|
46 | @NotAnonymous() | |||
|
47 | @CSRFRequired() | |||
|
48 | @view_config(route_name='upload_file', request_method='POST', renderer='json_ext') | |||
|
49 | def upload_file(self): | |||
|
50 | self.load_default_context() | |||
|
51 | file_obj = self.request.POST.get(self.upload_key) | |||
|
52 | ||||
|
53 | if file_obj is None: | |||
|
54 | return {'store_fid': None, | |||
|
55 | 'access_path': None, | |||
|
56 | 'error': '{} data field is missing'.format(self.upload_key)} | |||
|
57 | ||||
|
58 | if not hasattr(file_obj, 'filename'): | |||
|
59 | return {'store_fid': None, | |||
|
60 | 'access_path': None, | |||
|
61 | 'error': 'filename cannot be read from the data field'} | |||
|
62 | ||||
|
63 | filename = file_obj.filename | |||
|
64 | ||||
|
65 | metadata = { | |||
|
66 | 'filename': filename, | |||
|
67 | 'size': '', # filled by save_file | |||
|
68 | 'user_uploaded': {'username': self._rhodecode_user.username, | |||
|
69 | 'user_id': self._rhodecode_user.user_id, | |||
|
70 | 'ip': self._rhodecode_user.ip_addr}} | |||
|
71 | try: | |||
|
72 | store_fid = self.storage.save_file(file_obj.file, filename, | |||
|
73 | metadata=metadata) | |||
|
74 | except FileNotAllowedException: | |||
|
75 | return {'store_fid': None, | |||
|
76 | 'access_path': None, | |||
|
77 | 'error': 'File {} is not allowed.'.format(filename)} | |||
|
78 | ||||
|
79 | except FileOverSizeException: | |||
|
80 | return {'store_fid': None, | |||
|
81 | 'access_path': None, | |||
|
82 | 'error': 'File {} is exceeding allowed limit.'.format(filename)} | |||
|
83 | ||||
|
84 | return {'store_fid': store_fid, | |||
|
85 | 'access_path': h.route_path('download_file', fid=store_fid)} | |||
|
86 | ||||
|
87 | @view_config(route_name='download_file') | |||
|
88 | def download_file(self): | |||
|
89 | self.load_default_context() | |||
|
90 | file_uid = self.request.matchdict['fid'] | |||
|
91 | log.debug('Requesting FID:%s from store %s', file_uid, self.storage) | |||
|
92 | if not self.storage.exists(file_uid): | |||
|
93 | log.debug('File with FID:%s not found in the store', file_uid) | |||
|
94 | raise HTTPNotFound() | |||
|
95 | ||||
|
96 | file_path = self.storage.store_path(file_uid) | |||
|
97 | return FileResponse(file_path) |
@@ -284,6 +284,13 b' labs_settings_active = true' | |||||
284 | ## This is used to store exception from RhodeCode in shared directory |
|
284 | ## This is used to store exception from RhodeCode in shared directory | |
285 | #exception_tracker.store_path = |
|
285 | #exception_tracker.store_path = | |
286 |
|
286 | |||
|
287 | ## File store configuration. This is used to store and serve uploaded files | |||
|
288 | file_store.enabled = true | |||
|
289 | ## backend, only available one is local | |||
|
290 | file_store.backend = local | |||
|
291 | ## path to store the uploaded binaries | |||
|
292 | file_store.storage_path = %(here)s/data/file_store | |||
|
293 | ||||
287 |
|
294 | |||
288 | #################################### |
|
295 | #################################### | |
289 | ### CELERY CONFIG #### |
|
296 | ### CELERY CONFIG #### |
@@ -259,6 +259,13 b' labs_settings_active = true' | |||||
259 | ## This is used to store exception from RhodeCode in shared directory |
|
259 | ## This is used to store exception from RhodeCode in shared directory | |
260 | #exception_tracker.store_path = |
|
260 | #exception_tracker.store_path = | |
261 |
|
261 | |||
|
262 | ## File store configuration. This is used to store and serve uploaded files | |||
|
263 | file_store.enabled = true | |||
|
264 | ## backend, only available one is local | |||
|
265 | file_store.backend = local | |||
|
266 | ## path to store the uploaded binaries | |||
|
267 | file_store.storage_path = %(here)s/data/file_store | |||
|
268 | ||||
262 |
|
269 | |||
263 | #################################### |
|
270 | #################################### | |
264 | ### CELERY CONFIG #### |
|
271 | ### CELERY CONFIG #### |
@@ -281,6 +281,7 b' def includeme(config):' | |||||
281 | config.include('rhodecode.apps.ops') |
|
281 | config.include('rhodecode.apps.ops') | |
282 | config.include('rhodecode.apps.admin') |
|
282 | config.include('rhodecode.apps.admin') | |
283 | config.include('rhodecode.apps.channelstream') |
|
283 | config.include('rhodecode.apps.channelstream') | |
|
284 | config.include('rhodecode.apps.upload_store') | |||
284 | config.include('rhodecode.apps.login') |
|
285 | config.include('rhodecode.apps.login') | |
285 | config.include('rhodecode.apps.home') |
|
286 | config.include('rhodecode.apps.home') | |
286 | config.include('rhodecode.apps.journal') |
|
287 | config.include('rhodecode.apps.journal') |
@@ -136,6 +136,8 b' function registerRCRoutes() {' | |||||
136 | pyroutes.register('channelstream_connect', '/_admin/channelstream/connect', []); |
|
136 | pyroutes.register('channelstream_connect', '/_admin/channelstream/connect', []); | |
137 | pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []); |
|
137 | pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []); | |
138 | pyroutes.register('channelstream_proxy', '/_channelstream', []); |
|
138 | pyroutes.register('channelstream_proxy', '/_channelstream', []); | |
|
139 | pyroutes.register('upload_file', '/_file_store/upload', []); | |||
|
140 | pyroutes.register('download_file', '/_file_store/download/%(fid)s', ['fid']); | |||
139 | pyroutes.register('logout', '/_admin/logout', []); |
|
141 | pyroutes.register('logout', '/_admin/logout', []); | |
140 | pyroutes.register('reset_password', '/_admin/password_reset', []); |
|
142 | pyroutes.register('reset_password', '/_admin/password_reset', []); | |
141 | pyroutes.register('reset_password_confirmation', '/_admin/password_reset_confirmation', []); |
|
143 | pyroutes.register('reset_password_confirmation', '/_admin/password_reset_confirmation', []); |
@@ -101,8 +101,7 b' class CustomTestResponse(TestResponse):' | |||||
101 | """ |
|
101 | """ | |
102 |
|
102 | |||
103 | from pyramid_beaker import session_factory_from_settings |
|
103 | from pyramid_beaker import session_factory_from_settings | |
104 | session = session_factory_from_settings( |
|
104 | session = session_factory_from_settings(self.test_app._pyramid_settings) | |
105 | self.test_app.app.config.get_settings()) |
|
|||
106 | return session(self.request) |
|
105 | return session(self.request) | |
107 |
|
106 | |||
108 |
|
107 | |||
@@ -140,6 +139,14 b' class CustomTestApp(TestApp):' | |||||
140 | def csrf_token(self): |
|
139 | def csrf_token(self): | |
141 | return self.rc_login_data['csrf_token'] |
|
140 | return self.rc_login_data['csrf_token'] | |
142 |
|
141 | |||
|
142 | @property | |||
|
143 | def _pyramid_registry(self): | |||
|
144 | return self.app.config.registry | |||
|
145 | ||||
|
146 | @property | |||
|
147 | def _pyramid_settings(self): | |||
|
148 | return self._pyramid_registry.settings | |||
|
149 | ||||
143 |
|
150 | |||
144 | def set_anonymous_access(enabled): |
|
151 | def set_anonymous_access(enabled): | |
145 | """(Dis)allows anonymous access depending on parameter `enabled`""" |
|
152 | """(Dis)allows anonymous access depending on parameter `enabled`""" |
General Comments 0
You need to be logged in to leave comments.
Login now