Show More
@@ -0,0 +1,49 b'' | |||
|
1 | # -*- coding: utf-8 -*- | |
|
2 | ||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |
|
4 | # | |
|
5 | # This program is free software: you can redistribute it and/or modify | |
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |
|
7 | # (only), as published by the Free Software Foundation. | |
|
8 | # | |
|
9 | # This program is distributed in the hope that it will be useful, | |
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
|
12 | # GNU General Public License for more details. | |
|
13 | # | |
|
14 | # You should have received a copy of the GNU Affero General Public License | |
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
|
16 | # | |
|
17 | # This program is dual-licensed. If you wish to learn more about the | |
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |
|
20 | import os | |
|
21 | from rhodecode.apps.upload_store import config_keys | |
|
22 | from rhodecode.config.middleware import _bool_setting, _string_setting | |
|
23 | ||
|
24 | ||
|
25 | def _sanitize_settings_and_apply_defaults(settings): | |
|
26 | """ | |
|
27 | Set defaults, convert to python types and validate settings. | |
|
28 | """ | |
|
29 | _bool_setting(settings, config_keys.enabled, 'true') | |
|
30 | ||
|
31 | _string_setting(settings, config_keys.backend, 'local') | |
|
32 | ||
|
33 | default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store') | |
|
34 | _string_setting(settings, config_keys.store_path, default_store) | |
|
35 | ||
|
36 | ||
|
37 | def includeme(config): | |
|
38 | settings = config.registry.settings | |
|
39 | _sanitize_settings_and_apply_defaults(settings) | |
|
40 | ||
|
41 | config.add_route( | |
|
42 | name='upload_file', | |
|
43 | pattern='/_file_store/upload') | |
|
44 | config.add_route( | |
|
45 | name='download_file', | |
|
46 | pattern='/_file_store/download/{fid}') | |
|
47 | ||
|
48 | # Scan module for configuration decorators. | |
|
49 | config.scan('.views', ignore='.tests') |
@@ -0,0 +1,27 b'' | |||
|
1 | # -*- coding: utf-8 -*- | |
|
2 | ||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |
|
4 | # | |
|
5 | # This program is free software: you can redistribute it and/or modify | |
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |
|
7 | # (only), as published by the Free Software Foundation. | |
|
8 | # | |
|
9 | # This program is distributed in the hope that it will be useful, | |
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
|
12 | # GNU General Public License for more details. | |
|
13 | # | |
|
14 | # You should have received a copy of the GNU Affero General Public License | |
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
|
16 | # | |
|
17 | # This program is dual-licensed. If you wish to learn more about the | |
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |
|
20 | ||
|
21 | ||
|
22 | # Definition of setting keys used to configure this module. Defined here to | |
|
23 | # avoid repetition of keys throughout the module. | |
|
24 | ||
|
25 | enabled = 'file_store.enabled' | |
|
26 | backend = 'file_store.backend' | |
|
27 | store_path = 'file_store.storage_path' |
@@ -0,0 +1,31 b'' | |||
|
1 | # -*- coding: utf-8 -*- | |
|
2 | ||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |
|
4 | # | |
|
5 | # This program is free software: you can redistribute it and/or modify | |
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |
|
7 | # (only), as published by the Free Software Foundation. | |
|
8 | # | |
|
9 | # This program is distributed in the hope that it will be useful, | |
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
|
12 | # GNU General Public License for more details. | |
|
13 | # | |
|
14 | # You should have received a copy of the GNU Affero General Public License | |
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
|
16 | # | |
|
17 | # This program is dual-licensed. If you wish to learn more about the | |
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |
|
20 | ||
|
21 | ||
|
22 | class FileNotAllowedException(Exception): | |
|
23 | """ | |
|
24 | Thrown if file does not have an allowed extension. | |
|
25 | """ | |
|
26 | ||
|
27 | ||
|
28 | class FileOverSizeException(Exception): | |
|
29 | """ | |
|
30 | Thrown if file is over the set limit. | |
|
31 | """ |
@@ -0,0 +1,66 b'' | |||
|
1 | # -*- coding: utf-8 -*- | |
|
2 | ||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |
|
4 | # | |
|
5 | # This program is free software: you can redistribute it and/or modify | |
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |
|
7 | # (only), as published by the Free Software Foundation. | |
|
8 | # | |
|
9 | # This program is distributed in the hope that it will be useful, | |
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
|
12 | # GNU General Public License for more details. | |
|
13 | # | |
|
14 | # You should have received a copy of the GNU Affero General Public License | |
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
|
16 | # | |
|
17 | # This program is dual-licensed. If you wish to learn more about the | |
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |
|
20 | ||
|
21 | ||
|
22 | ANY = [] | |
|
23 | TEXT_EXT = ['txt', 'md', 'rst', 'log'] | |
|
24 | DOCUMENTS_EXT = ['pdf', 'rtf', 'odf', 'ods', 'gnumeric', 'abw', 'doc', 'docx', 'xls', 'xlsx'] | |
|
25 | IMAGES_EXT = ['jpg', 'jpe', 'jpeg', 'png', 'gif', 'svg', 'bmp', 'tiff'] | |
|
26 | AUDIO_EXT = ['wav', 'mp3', 'aac', 'ogg', 'oga', 'flac'] | |
|
27 | VIDEO_EXT = ['mpeg', '3gp', 'avi', 'divx', 'dvr', 'flv', 'mp4', 'wmv'] | |
|
28 | DATA_EXT = ['csv', 'ini', 'json', 'plist', 'xml', 'yaml', 'yml'] | |
|
29 | SCRIPTS_EXT = ['js', 'php', 'pl', 'py', 'rb', 'sh', 'go', 'c', 'h'] | |
|
30 | ARCHIVES_EXT = ['gz', 'bz2', 'zip', 'tar', 'tgz', 'txz', '7z'] | |
|
31 | EXECUTABLES_EXT = ['so', 'exe', 'dll'] | |
|
32 | ||
|
33 | ||
|
34 | DEFAULT = DOCUMENTS_EXT + TEXT_EXT + IMAGES_EXT + DATA_EXT | |
|
35 | ||
|
36 | GROUPS = dict(( | |
|
37 | ('any', ANY), | |
|
38 | ('text', TEXT_EXT), | |
|
39 | ('documents', DOCUMENTS_EXT), | |
|
40 | ('images', IMAGES_EXT), | |
|
41 | ('audio', AUDIO_EXT), | |
|
42 | ('video', VIDEO_EXT), | |
|
43 | ('data', DATA_EXT), | |
|
44 | ('scripts', SCRIPTS_EXT), | |
|
45 | ('archives', ARCHIVES_EXT), | |
|
46 | ('executables', EXECUTABLES_EXT), | |
|
47 | ('default', DEFAULT), | |
|
48 | )) | |
|
49 | ||
|
50 | ||
|
51 | def resolve_extensions(extensions, groups=None): | |
|
52 | """ | |
|
53 | Calculate allowed extensions based on a list of extensions provided, and optional | |
|
54 | groups of extensions from the available lists. | |
|
55 | ||
|
56 | :param extensions: a list of extensions e.g ['py', 'txt'] | |
|
57 | :param groups: additionally groups to extend the extensions. | |
|
58 | """ | |
|
59 | groups = groups or [] | |
|
60 | valid_exts = set([x.lower() for x in extensions]) | |
|
61 | ||
|
62 | for group in groups: | |
|
63 | if group in GROUPS: | |
|
64 | valid_exts.update(GROUPS[group]) | |
|
65 | ||
|
66 | return valid_exts |
@@ -0,0 +1,167 b'' | |||
|
1 | # -*- coding: utf-8 -*- | |
|
2 | ||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |
|
4 | # | |
|
5 | # This program is free software: you can redistribute it and/or modify | |
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |
|
7 | # (only), as published by the Free Software Foundation. | |
|
8 | # | |
|
9 | # This program is distributed in the hope that it will be useful, | |
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
|
12 | # GNU General Public License for more details. | |
|
13 | # | |
|
14 | # You should have received a copy of the GNU Affero General Public License | |
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
|
16 | # | |
|
17 | # This program is dual-licensed. If you wish to learn more about the | |
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |
|
20 | ||
|
21 | import os | |
|
22 | import shutil | |
|
23 | ||
|
24 | from rhodecode.lib.ext_json import json | |
|
25 | from rhodecode.apps.upload_store import utils | |
|
26 | from rhodecode.apps.upload_store.extensions import resolve_extensions | |
|
27 | from rhodecode.apps.upload_store.exceptions import FileNotAllowedException | |
|
28 | ||
|
29 | ||
|
30 | class LocalFileStorage(object): | |
|
31 | ||
|
32 | @classmethod | |
|
33 | def resolve_name(cls, name, directory): | |
|
34 | """ | |
|
35 | Resolves a unique name and the correct path. If a filename | |
|
36 | for that path already exists then a numeric prefix with values > 0 will be | |
|
37 | added, for example test.jpg -> test-1.jpg etc. initially file would have 0 prefix. | |
|
38 | ||
|
39 | :param name: base name of file | |
|
40 | :param directory: absolute directory path | |
|
41 | """ | |
|
42 | ||
|
43 | basename, ext = os.path.splitext(name) | |
|
44 | counter = 0 | |
|
45 | while True: | |
|
46 | name = '%s-%d%s' % (basename, counter, ext) | |
|
47 | path = os.path.join(directory, name) | |
|
48 | if not os.path.exists(path): | |
|
49 | return name, path | |
|
50 | counter += 1 | |
|
51 | ||
|
52 | def __init__(self, base_path, extension_groups=None): | |
|
53 | ||
|
54 | """ | |
|
55 | Local file storage | |
|
56 | ||
|
57 | :param base_path: the absolute base path where uploads are stored | |
|
58 | :param extension_groups: extensions string | |
|
59 | """ | |
|
60 | ||
|
61 | extension_groups = extension_groups or ['any'] | |
|
62 | self.base_path = base_path | |
|
63 | self.extensions = resolve_extensions([], groups=extension_groups) | |
|
64 | ||
|
65 | def store_path(self, filename): | |
|
66 | """ | |
|
67 | Returns absolute file path of the filename, joined to the | |
|
68 | base_path. | |
|
69 | ||
|
70 | :param filename: base name of file | |
|
71 | """ | |
|
72 | return os.path.join(self.base_path, filename) | |
|
73 | ||
|
74 | def delete(self, filename): | |
|
75 | """ | |
|
76 | Deletes the filename. Filename is resolved with the | |
|
77 | absolute path based on base_path. If file does not exist, | |
|
78 | returns **False**, otherwise **True** | |
|
79 | ||
|
80 | :param filename: base name of file | |
|
81 | """ | |
|
82 | if self.exists(filename): | |
|
83 | os.remove(self.store_path(filename)) | |
|
84 | return True | |
|
85 | return False | |
|
86 | ||
|
87 | def exists(self, filename): | |
|
88 | """ | |
|
89 | Checks if file exists. Resolves filename's absolute | |
|
90 | path based on base_path. | |
|
91 | ||
|
92 | :param filename: base name of file | |
|
93 | """ | |
|
94 | return os.path.exists(self.store_path(filename)) | |
|
95 | ||
|
96 | def filename_allowed(self, filename, extensions=None): | |
|
97 | """Checks if a filename has an allowed extension | |
|
98 | ||
|
99 | :param filename: base name of file | |
|
100 | :param extensions: iterable of extensions (or self.extensions) | |
|
101 | """ | |
|
102 | _, ext = os.path.splitext(filename) | |
|
103 | return self.extension_allowed(ext, extensions) | |
|
104 | ||
|
105 | def extension_allowed(self, ext, extensions=None): | |
|
106 | """ | |
|
107 | Checks if an extension is permitted. Both e.g. ".jpg" and | |
|
108 | "jpg" can be passed in. Extension lookup is case-insensitive. | |
|
109 | ||
|
110 | :param extensions: iterable of extensions (or self.extensions) | |
|
111 | """ | |
|
112 | ||
|
113 | extensions = extensions or self.extensions | |
|
114 | if not extensions: | |
|
115 | return True | |
|
116 | if ext.startswith('.'): | |
|
117 | ext = ext[1:] | |
|
118 | return ext.lower() in extensions | |
|
119 | ||
|
120 | def save_file(self, file_obj, filename, directory=None, extensions=None, | |
|
121 | metadata=None, **kwargs): | |
|
122 | """ | |
|
123 | Saves a file object to the uploads location. | |
|
124 | Returns the resolved filename, i.e. the directory + | |
|
125 | the (randomized/incremented) base name. | |
|
126 | ||
|
127 | :param file_obj: **cgi.FieldStorage** object (or similar) | |
|
128 | :param filename: original filename | |
|
129 | :param directory: relative path of sub-directory | |
|
130 | :param extensions: iterable of allowed extensions, if not default | |
|
131 | :param metadata: JSON metadata to store next to the file with .meta suffix | |
|
132 | :returns: modified filename | |
|
133 | """ | |
|
134 | ||
|
135 | extensions = extensions or self.extensions | |
|
136 | ||
|
137 | if not self.filename_allowed(filename, extensions): | |
|
138 | raise FileNotAllowedException() | |
|
139 | ||
|
140 | if directory: | |
|
141 | dest_directory = os.path.join(self.base_path, directory) | |
|
142 | else: | |
|
143 | dest_directory = self.base_path | |
|
144 | ||
|
145 | if not os.path.exists(dest_directory): | |
|
146 | os.makedirs(dest_directory) | |
|
147 | ||
|
148 | filename = utils.uid_filename(filename) | |
|
149 | ||
|
150 | filename, path = self.resolve_name(filename, dest_directory) | |
|
151 | filename_meta = filename + '.meta' | |
|
152 | ||
|
153 | file_obj.seek(0) | |
|
154 | ||
|
155 | with open(path, "wb") as dest: | |
|
156 | shutil.copyfileobj(file_obj, dest) | |
|
157 | ||
|
158 | if metadata: | |
|
159 | size = os.stat(path).st_size | |
|
160 | metadata.update({'size': size}) | |
|
161 | with open(os.path.join(dest_directory, filename_meta), "wb") as dest_meta: | |
|
162 | dest_meta.write(json.dumps(metadata)) | |
|
163 | ||
|
164 | if directory: | |
|
165 | filename = os.path.join(directory, filename) | |
|
166 | ||
|
167 | return filename |
@@ -0,0 +1,20 b'' | |||
|
1 | # -*- coding: utf-8 -*- | |
|
2 | ||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |
|
4 | # | |
|
5 | # This program is free software: you can redistribute it and/or modify | |
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |
|
7 | # (only), as published by the Free Software Foundation. | |
|
8 | # | |
|
9 | # This program is distributed in the hope that it will be useful, | |
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
|
12 | # GNU General Public License for more details. | |
|
13 | # | |
|
14 | # You should have received a copy of the GNU Affero General Public License | |
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
|
16 | # | |
|
17 | # This program is dual-licensed. If you wish to learn more about the | |
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |
|
20 |
@@ -0,0 +1,110 b'' | |||
|
1 | # -*- coding: utf-8 -*- | |
|
2 | ||
|
3 | # Copyright (C) 2010-2019 RhodeCode GmbH | |
|
4 | # | |
|
5 | # This program is free software: you can redistribute it and/or modify | |
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |
|
7 | # (only), as published by the Free Software Foundation. | |
|
8 | # | |
|
9 | # This program is distributed in the hope that it will be useful, | |
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
|
12 | # GNU General Public License for more details. | |
|
13 | # | |
|
14 | # You should have received a copy of the GNU Affero General Public License | |
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
|
16 | # | |
|
17 | # This program is dual-licensed. If you wish to learn more about the | |
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |
|
20 | import os | |
|
21 | import pytest | |
|
22 | ||
|
23 | from rhodecode.lib.ext_json import json | |
|
24 | from rhodecode.tests import TestController | |
|
25 | from rhodecode.apps.upload_store import utils, config_keys | |
|
26 | ||
|
27 | ||
|
28 | def route_path(name, params=None, **kwargs): | |
|
29 | import urllib | |
|
30 | ||
|
31 | base_url = { | |
|
32 | 'upload_file': '/_file_store/upload', | |
|
33 | 'download_file': '/_file_store/download/{fid}', | |
|
34 | ||
|
35 | }[name].format(**kwargs) | |
|
36 | ||
|
37 | if params: | |
|
38 | base_url = '{}?{}'.format(base_url, urllib.urlencode(params)) | |
|
39 | return base_url | |
|
40 | ||
|
41 | ||
|
42 | class TestFileStoreViews(TestController): | |
|
43 | ||
|
44 | @pytest.mark.parametrize("fid, content, exists", [ | |
|
45 | ('abcde-0.jpg', "xxxxx", True), | |
|
46 | ('abcde-0.exe', "1234567", True), | |
|
47 | ('abcde-0.jpg', "xxxxx", False), | |
|
48 | ]) | |
|
49 | def test_get_files_from_store(self, fid, content, exists, tmpdir): | |
|
50 | self.log_user() | |
|
51 | store_path = self.app._pyramid_settings[config_keys.store_path] | |
|
52 | ||
|
53 | if exists: | |
|
54 | status = 200 | |
|
55 | store = utils.get_file_storage({config_keys.store_path: store_path}) | |
|
56 | filesystem_file = os.path.join(str(tmpdir), fid) | |
|
57 | with open(filesystem_file, 'wb') as f: | |
|
58 | f.write(content) | |
|
59 | ||
|
60 | with open(filesystem_file, 'rb') as f: | |
|
61 | fid = store.save_file(f, fid, metadata={'filename': fid}) | |
|
62 | ||
|
63 | else: | |
|
64 | status = 404 | |
|
65 | ||
|
66 | response = self.app.get(route_path('download_file', fid=fid), status=status) | |
|
67 | ||
|
68 | if exists: | |
|
69 | assert response.text == content | |
|
70 | metadata = os.path.join(store_path, fid + '.meta') | |
|
71 | assert os.path.exists(metadata) | |
|
72 | with open(metadata, 'rb') as f: | |
|
73 | json_data = json.loads(f.read()) | |
|
74 | ||
|
75 | assert json_data | |
|
76 | assert 'size' in json_data | |
|
77 | ||
|
78 | def test_upload_files_without_content_to_store(self): | |
|
79 | self.log_user() | |
|
80 | response = self.app.post( | |
|
81 | route_path('upload_file'), | |
|
82 | params={'csrf_token': self.csrf_token}, | |
|
83 | status=200) | |
|
84 | ||
|
85 | assert response.json == { | |
|
86 | u'error': u'store_file data field is missing', | |
|
87 | u'access_path': None, | |
|
88 | u'store_fid': None} | |
|
89 | ||
|
90 | def test_upload_files_bogus_content_to_store(self): | |
|
91 | self.log_user() | |
|
92 | response = self.app.post( | |
|
93 | route_path('upload_file'), | |
|
94 | params={'csrf_token': self.csrf_token, 'store_file': 'bogus'}, | |
|
95 | status=200) | |
|
96 | ||
|
97 | assert response.json == { | |
|
98 | u'error': u'filename cannot be read from the data field', | |
|
99 | u'access_path': None, | |
|
100 | u'store_fid': None} | |
|
101 | ||
|
102 | def test_upload_content_to_store(self): | |
|
103 | self.log_user() | |
|
104 | response = self.app.post( | |
|
105 | route_path('upload_file'), | |
|
106 | upload_files=[('store_file', 'myfile.txt', 'SOME CONTENT')], | |
|
107 | params={'csrf_token': self.csrf_token}, | |
|
108 | status=200) | |
|
109 | ||
|
110 | assert response.json['store_fid'] |
@@ -0,0 +1,47 b'' | |||
|
1 | # -*- coding: utf-8 -*- | |
|
2 | ||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |
|
4 | # | |
|
5 | # This program is free software: you can redistribute it and/or modify | |
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |
|
7 | # (only), as published by the Free Software Foundation. | |
|
8 | # | |
|
9 | # This program is distributed in the hope that it will be useful, | |
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
|
12 | # GNU General Public License for more details. | |
|
13 | # | |
|
14 | # You should have received a copy of the GNU Affero General Public License | |
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
|
16 | # | |
|
17 | # This program is dual-licensed. If you wish to learn more about the | |
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |
|
20 | ||
|
21 | ||
|
22 | import os | |
|
23 | import uuid | |
|
24 | ||
|
25 | ||
|
26 | def get_file_storage(settings): | |
|
27 | from rhodecode.apps.upload_store.local_store import LocalFileStorage | |
|
28 | from rhodecode.apps.upload_store import config_keys | |
|
29 | store_path = settings.get(config_keys.store_path) | |
|
30 | return LocalFileStorage(base_path=store_path) | |
|
31 | ||
|
32 | ||
|
33 | def uid_filename(filename, randomized=True): | |
|
34 | """ | |
|
35 | Generates a randomized or stable (uuid) filename, | |
|
36 | preserving the original extension. | |
|
37 | ||
|
38 | :param filename: the original filename | |
|
39 | :param randomized: define if filename should be stable (sha1 based) or randomized | |
|
40 | """ | |
|
41 | _, ext = os.path.splitext(filename) | |
|
42 | if randomized: | |
|
43 | uid = uuid.uuid4() | |
|
44 | else: | |
|
45 | hash_key = '{}.{}'.format(filename, 'store') | |
|
46 | uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key) | |
|
47 | return str(uid) + ext.lower() |
@@ -0,0 +1,97 b'' | |||
|
1 | # -*- coding: utf-8 -*- | |
|
2 | ||
|
3 | # Copyright (C) 2016-2019 RhodeCode GmbH | |
|
4 | # | |
|
5 | # This program is free software: you can redistribute it and/or modify | |
|
6 | # it under the terms of the GNU Affero General Public License, version 3 | |
|
7 | # (only), as published by the Free Software Foundation. | |
|
8 | # | |
|
9 | # This program is distributed in the hope that it will be useful, | |
|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
|
12 | # GNU General Public License for more details. | |
|
13 | # | |
|
14 | # You should have received a copy of the GNU Affero General Public License | |
|
15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
|
16 | # | |
|
17 | # This program is dual-licensed. If you wish to learn more about the | |
|
18 | # RhodeCode Enterprise Edition, including its added features, Support services, | |
|
19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ | |
|
20 | import logging | |
|
21 | ||
|
22 | from pyramid.view import view_config | |
|
23 | from pyramid.response import FileResponse | |
|
24 | from pyramid.httpexceptions import HTTPFound, HTTPNotFound | |
|
25 | ||
|
26 | from rhodecode.apps._base import BaseAppView | |
|
27 | from rhodecode.apps.upload_store import utils | |
|
28 | from rhodecode.apps.upload_store.exceptions import ( | |
|
29 | FileNotAllowedException,FileOverSizeException) | |
|
30 | ||
|
31 | from rhodecode.lib import helpers as h | |
|
32 | from rhodecode.lib import audit_logger | |
|
33 | from rhodecode.lib.auth import (CSRFRequired, NotAnonymous) | |
|
34 | ||
|
35 | log = logging.getLogger(__name__) | |
|
36 | ||
|
37 | ||
|
38 | class FileStoreView(BaseAppView): | |
|
39 | upload_key = 'store_file' | |
|
40 | ||
|
41 | def load_default_context(self): | |
|
42 | c = self._get_local_tmpl_context() | |
|
43 | self.storage = utils.get_file_storage(self.request.registry.settings) | |
|
44 | return c | |
|
45 | ||
|
46 | @NotAnonymous() | |
|
47 | @CSRFRequired() | |
|
48 | @view_config(route_name='upload_file', request_method='POST', renderer='json_ext') | |
|
49 | def upload_file(self): | |
|
50 | self.load_default_context() | |
|
51 | file_obj = self.request.POST.get(self.upload_key) | |
|
52 | ||
|
53 | if file_obj is None: | |
|
54 | return {'store_fid': None, | |
|
55 | 'access_path': None, | |
|
56 | 'error': '{} data field is missing'.format(self.upload_key)} | |
|
57 | ||
|
58 | if not hasattr(file_obj, 'filename'): | |
|
59 | return {'store_fid': None, | |
|
60 | 'access_path': None, | |
|
61 | 'error': 'filename cannot be read from the data field'} | |
|
62 | ||
|
63 | filename = file_obj.filename | |
|
64 | ||
|
65 | metadata = { | |
|
66 | 'filename': filename, | |
|
67 | 'size': '', # filled by save_file | |
|
68 | 'user_uploaded': {'username': self._rhodecode_user.username, | |
|
69 | 'user_id': self._rhodecode_user.user_id, | |
|
70 | 'ip': self._rhodecode_user.ip_addr}} | |
|
71 | try: | |
|
72 | store_fid = self.storage.save_file(file_obj.file, filename, | |
|
73 | metadata=metadata) | |
|
74 | except FileNotAllowedException: | |
|
75 | return {'store_fid': None, | |
|
76 | 'access_path': None, | |
|
77 | 'error': 'File {} is not allowed.'.format(filename)} | |
|
78 | ||
|
79 | except FileOverSizeException: | |
|
80 | return {'store_fid': None, | |
|
81 | 'access_path': None, | |
|
82 | 'error': 'File {} is exceeding allowed limit.'.format(filename)} | |
|
83 | ||
|
84 | return {'store_fid': store_fid, | |
|
85 | 'access_path': h.route_path('download_file', fid=store_fid)} | |
|
86 | ||
|
87 | @view_config(route_name='download_file') | |
|
88 | def download_file(self): | |
|
89 | self.load_default_context() | |
|
90 | file_uid = self.request.matchdict['fid'] | |
|
91 | log.debug('Requesting FID:%s from store %s', file_uid, self.storage) | |
|
92 | if not self.storage.exists(file_uid): | |
|
93 | log.debug('File with FID:%s not found in the store', file_uid) | |
|
94 | raise HTTPNotFound() | |
|
95 | ||
|
96 | file_path = self.storage.store_path(file_uid) | |
|
97 | return FileResponse(file_path) |
@@ -284,6 +284,13 b' labs_settings_active = true' | |||
|
284 | 284 | ## This is used to store exception from RhodeCode in shared directory |
|
285 | 285 | #exception_tracker.store_path = |
|
286 | 286 | |
|
287 | ## File store configuration. This is used to store and serve uploaded files | |
|
288 | file_store.enabled = true | |
|
289 | ## backend, only available one is local | |
|
290 | file_store.backend = local | |
|
291 | ## path to store the uploaded binaries | |
|
292 | file_store.storage_path = %(here)s/data/file_store | |
|
293 | ||
|
287 | 294 | |
|
288 | 295 | #################################### |
|
289 | 296 | ### CELERY CONFIG #### |
@@ -259,6 +259,13 b' labs_settings_active = true' | |||
|
259 | 259 | ## This is used to store exception from RhodeCode in shared directory |
|
260 | 260 | #exception_tracker.store_path = |
|
261 | 261 | |
|
262 | ## File store configuration. This is used to store and serve uploaded files | |
|
263 | file_store.enabled = true | |
|
264 | ## backend, only available one is local | |
|
265 | file_store.backend = local | |
|
266 | ## path to store the uploaded binaries | |
|
267 | file_store.storage_path = %(here)s/data/file_store | |
|
268 | ||
|
262 | 269 | |
|
263 | 270 | #################################### |
|
264 | 271 | ### CELERY CONFIG #### |
@@ -281,6 +281,7 b' def includeme(config):' | |||
|
281 | 281 | config.include('rhodecode.apps.ops') |
|
282 | 282 | config.include('rhodecode.apps.admin') |
|
283 | 283 | config.include('rhodecode.apps.channelstream') |
|
284 | config.include('rhodecode.apps.upload_store') | |
|
284 | 285 | config.include('rhodecode.apps.login') |
|
285 | 286 | config.include('rhodecode.apps.home') |
|
286 | 287 | config.include('rhodecode.apps.journal') |
@@ -136,6 +136,8 b' function registerRCRoutes() {' | |||
|
136 | 136 | pyroutes.register('channelstream_connect', '/_admin/channelstream/connect', []); |
|
137 | 137 | pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []); |
|
138 | 138 | pyroutes.register('channelstream_proxy', '/_channelstream', []); |
|
139 | pyroutes.register('upload_file', '/_file_store/upload', []); | |
|
140 | pyroutes.register('download_file', '/_file_store/download/%(fid)s', ['fid']); | |
|
139 | 141 | pyroutes.register('logout', '/_admin/logout', []); |
|
140 | 142 | pyroutes.register('reset_password', '/_admin/password_reset', []); |
|
141 | 143 | pyroutes.register('reset_password_confirmation', '/_admin/password_reset_confirmation', []); |
@@ -101,8 +101,7 b' class CustomTestResponse(TestResponse):' | |||
|
101 | 101 | """ |
|
102 | 102 | |
|
103 | 103 | from pyramid_beaker import session_factory_from_settings |
|
104 | session = session_factory_from_settings( | |
|
105 | self.test_app.app.config.get_settings()) | |
|
104 | session = session_factory_from_settings(self.test_app._pyramid_settings) | |
|
106 | 105 | return session(self.request) |
|
107 | 106 | |
|
108 | 107 | |
@@ -140,6 +139,14 b' class CustomTestApp(TestApp):' | |||
|
140 | 139 | def csrf_token(self): |
|
141 | 140 | return self.rc_login_data['csrf_token'] |
|
142 | 141 | |
|
142 | @property | |
|
143 | def _pyramid_registry(self): | |
|
144 | return self.app.config.registry | |
|
145 | ||
|
146 | @property | |
|
147 | def _pyramid_settings(self): | |
|
148 | return self._pyramid_registry.settings | |
|
149 | ||
|
143 | 150 | |
|
144 | 151 | def set_anonymous_access(enabled): |
|
145 | 152 | """(Dis)allows anonymous access depending on parameter `enabled`""" |
General Comments 0
You need to be logged in to leave comments.
Login now