##// END OF EJS Templates
file-store: handle custom extensions validation better, e.g lower case and both formats .exe / exe
bart -
r3972:c69ff9e6 default
parent child Browse files
Show More
@@ -1,226 +1,231 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2016-2019 RhodeCode GmbH
3 # Copyright (C) 2016-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import os
21 import os
22 import time
22 import time
23 import shutil
23 import shutil
24 import hashlib
24 import hashlib
25
25
26 from rhodecode.lib.ext_json import json
26 from rhodecode.lib.ext_json import json
27 from rhodecode.apps.file_store import utils
27 from rhodecode.apps.file_store import utils
28 from rhodecode.apps.file_store.extensions import resolve_extensions
28 from rhodecode.apps.file_store.extensions import resolve_extensions
29 from rhodecode.apps.file_store.exceptions import FileNotAllowedException
29 from rhodecode.apps.file_store.exceptions import FileNotAllowedException
30
30
31 METADATA_VER = 'v1'
31 METADATA_VER = 'v1'
32
32
33
33
34 class LocalFileStorage(object):
34 class LocalFileStorage(object):
35
35
36 @classmethod
36 @classmethod
37 def resolve_name(cls, name, directory):
37 def resolve_name(cls, name, directory):
38 """
38 """
39 Resolves a unique name and the correct path. If a filename
39 Resolves a unique name and the correct path. If a filename
40 for that path already exists then a numeric prefix with values > 0 will be
40 for that path already exists then a numeric prefix with values > 0 will be
41 added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
41 added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
42
42
43 :param name: base name of file
43 :param name: base name of file
44 :param directory: absolute directory path
44 :param directory: absolute directory path
45 """
45 """
46
46
47 counter = 0
47 counter = 0
48 while True:
48 while True:
49 name = '%d-%s' % (counter, name)
49 name = '%d-%s' % (counter, name)
50
50
51 # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
51 # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
52 sub_store = cls._sub_store_from_filename(name)
52 sub_store = cls._sub_store_from_filename(name)
53 sub_store_path = os.path.join(directory, sub_store)
53 sub_store_path = os.path.join(directory, sub_store)
54 if not os.path.exists(sub_store_path):
54 if not os.path.exists(sub_store_path):
55 os.makedirs(sub_store_path)
55 os.makedirs(sub_store_path)
56
56
57 path = os.path.join(sub_store_path, name)
57 path = os.path.join(sub_store_path, name)
58 if not os.path.exists(path):
58 if not os.path.exists(path):
59 return name, path
59 return name, path
60 counter += 1
60 counter += 1
61
61
62 @classmethod
62 @classmethod
63 def _sub_store_from_filename(cls, filename):
63 def _sub_store_from_filename(cls, filename):
64 return filename[:2]
64 return filename[:2]
65
65
66 @classmethod
66 @classmethod
67 def calculate_path_hash(cls, file_path):
67 def calculate_path_hash(cls, file_path):
68 """
68 """
69 Efficient calculation of file_path sha256 sum
69 Efficient calculation of file_path sha256 sum
70
70
71 :param file_path:
71 :param file_path:
72 :return: sha256sum
72 :return: sha256sum
73 """
73 """
74 digest = hashlib.sha256()
74 digest = hashlib.sha256()
75 with open(file_path, 'rb') as f:
75 with open(file_path, 'rb') as f:
76 for chunk in iter(lambda: f.read(1024 * 100), b""):
76 for chunk in iter(lambda: f.read(1024 * 100), b""):
77 digest.update(chunk)
77 digest.update(chunk)
78
78
79 return digest.hexdigest()
79 return digest.hexdigest()
80
80
81 def __init__(self, base_path, extension_groups=None):
81 def __init__(self, base_path, extension_groups=None):
82
82
83 """
83 """
84 Local file storage
84 Local file storage
85
85
86 :param base_path: the absolute base path where uploads are stored
86 :param base_path: the absolute base path where uploads are stored
87 :param extension_groups: extensions string
87 :param extension_groups: extensions string
88 """
88 """
89
89
90 extension_groups = extension_groups or ['any']
90 extension_groups = extension_groups or ['any']
91 self.base_path = base_path
91 self.base_path = base_path
92 self.extensions = resolve_extensions([], groups=extension_groups)
92 self.extensions = resolve_extensions([], groups=extension_groups)
93
93
94 def __repr__(self):
94 def __repr__(self):
95 return '{}@{}'.format(self.__class__, self.base_path)
95 return '{}@{}'.format(self.__class__, self.base_path)
96
96
97 def store_path(self, filename):
97 def store_path(self, filename):
98 """
98 """
99 Returns absolute file path of the filename, joined to the
99 Returns absolute file path of the filename, joined to the
100 base_path.
100 base_path.
101
101
102 :param filename: base name of file
102 :param filename: base name of file
103 """
103 """
104 sub_store = self._sub_store_from_filename(filename)
104 sub_store = self._sub_store_from_filename(filename)
105 return os.path.join(self.base_path, sub_store, filename)
105 return os.path.join(self.base_path, sub_store, filename)
106
106
107 def delete(self, filename):
107 def delete(self, filename):
108 """
108 """
109 Deletes the filename. Filename is resolved with the
109 Deletes the filename. Filename is resolved with the
110 absolute path based on base_path. If file does not exist,
110 absolute path based on base_path. If file does not exist,
111 returns **False**, otherwise **True**
111 returns **False**, otherwise **True**
112
112
113 :param filename: base name of file
113 :param filename: base name of file
114 """
114 """
115 if self.exists(filename):
115 if self.exists(filename):
116 os.remove(self.store_path(filename))
116 os.remove(self.store_path(filename))
117 return True
117 return True
118 return False
118 return False
119
119
120 def exists(self, filename):
120 def exists(self, filename):
121 """
121 """
122 Checks if file exists. Resolves filename's absolute
122 Checks if file exists. Resolves filename's absolute
123 path based on base_path.
123 path based on base_path.
124
124
125 :param filename: base name of file
125 :param filename: base name of file
126 """
126 """
127 return os.path.exists(self.store_path(filename))
127 return os.path.exists(self.store_path(filename))
128
128
129 def filename_allowed(self, filename, extensions=None):
129 def filename_allowed(self, filename, extensions=None):
130 """Checks if a filename has an allowed extension
130 """Checks if a filename has an allowed extension
131
131
132 :param filename: base name of file
132 :param filename: base name of file
133 :param extensions: iterable of extensions (or self.extensions)
133 :param extensions: iterable of extensions (or self.extensions)
134 """
134 """
135 _, ext = os.path.splitext(filename)
135 _, ext = os.path.splitext(filename)
136 return self.extension_allowed(ext, extensions)
136 return self.extension_allowed(ext, extensions)
137
137
138 def extension_allowed(self, ext, extensions=None):
138 def extension_allowed(self, ext, extensions=None):
139 """
139 """
140 Checks if an extension is permitted. Both e.g. ".jpg" and
140 Checks if an extension is permitted. Both e.g. ".jpg" and
141 "jpg" can be passed in. Extension lookup is case-insensitive.
141 "jpg" can be passed in. Extension lookup is case-insensitive.
142
142
143 :param ext: extension to check
143 :param ext: extension to check
144 :param extensions: iterable of extensions to validate against (or self.extensions)
144 :param extensions: iterable of extensions to validate against (or self.extensions)
145 """
145 """
146 def normalize_ext(_ext):
147 if _ext.startswith('.'):
148 _ext = _ext[1:]
149 return _ext.lower()
146
150
147 extensions = extensions or self.extensions
151 extensions = extensions or self.extensions
148 if not extensions:
152 if not extensions:
149 return True
153 return True
150 if ext.startswith('.'):
154
151 ext = ext[1:]
155 ext = normalize_ext(ext)
152 return ext.lower() in extensions
156
157 return ext in [normalize_ext(x) for x in extensions]
153
158
154 def save_file(self, file_obj, filename, directory=None, extensions=None,
159 def save_file(self, file_obj, filename, directory=None, extensions=None,
155 extra_metadata=None, **kwargs):
160 extra_metadata=None, **kwargs):
156 """
161 """
157 Saves a file object to the uploads location.
162 Saves a file object to the uploads location.
158 Returns the resolved filename, i.e. the directory +
163 Returns the resolved filename, i.e. the directory +
159 the (randomized/incremented) base name.
164 the (randomized/incremented) base name.
160
165
161 :param file_obj: **cgi.FieldStorage** object (or similar)
166 :param file_obj: **cgi.FieldStorage** object (or similar)
162 :param filename: original filename
167 :param filename: original filename
163 :param directory: relative path of sub-directory
168 :param directory: relative path of sub-directory
164 :param extensions: iterable of allowed extensions, if not default
169 :param extensions: iterable of allowed extensions, if not default
165 :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
170 :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
166 """
171 """
167
172
168 extensions = extensions or self.extensions
173 extensions = extensions or self.extensions
169
174
170 if not self.filename_allowed(filename, extensions):
175 if not self.filename_allowed(filename, extensions):
171 raise FileNotAllowedException()
176 raise FileNotAllowedException()
172
177
173 if directory:
178 if directory:
174 dest_directory = os.path.join(self.base_path, directory)
179 dest_directory = os.path.join(self.base_path, directory)
175 else:
180 else:
176 dest_directory = self.base_path
181 dest_directory = self.base_path
177
182
178 if not os.path.exists(dest_directory):
183 if not os.path.exists(dest_directory):
179 os.makedirs(dest_directory)
184 os.makedirs(dest_directory)
180
185
181 filename = utils.uid_filename(filename)
186 filename = utils.uid_filename(filename)
182
187
183 # resolve also produces special sub-dir for file optimized store
188 # resolve also produces special sub-dir for file optimized store
184 filename, path = self.resolve_name(filename, dest_directory)
189 filename, path = self.resolve_name(filename, dest_directory)
185 stored_file_dir = os.path.dirname(path)
190 stored_file_dir = os.path.dirname(path)
186
191
187 file_obj.seek(0)
192 file_obj.seek(0)
188
193
189 with open(path, "wb") as dest:
194 with open(path, "wb") as dest:
190 shutil.copyfileobj(file_obj, dest)
195 shutil.copyfileobj(file_obj, dest)
191
196
192 metadata = {}
197 metadata = {}
193 if extra_metadata:
198 if extra_metadata:
194 metadata = extra_metadata
199 metadata = extra_metadata
195
200
196 size = os.stat(path).st_size
201 size = os.stat(path).st_size
197 file_hash = self.calculate_path_hash(path)
202 file_hash = self.calculate_path_hash(path)
198
203
199 metadata.update(
204 metadata.update(
200 {"filename": filename,
205 {"filename": filename,
201 "size": size,
206 "size": size,
202 "time": time.time(),
207 "time": time.time(),
203 "sha256": file_hash,
208 "sha256": file_hash,
204 "meta_ver": METADATA_VER})
209 "meta_ver": METADATA_VER})
205
210
206 filename_meta = filename + '.meta'
211 filename_meta = filename + '.meta'
207 with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
212 with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
208 dest_meta.write(json.dumps(metadata))
213 dest_meta.write(json.dumps(metadata))
209
214
210 if directory:
215 if directory:
211 filename = os.path.join(directory, filename)
216 filename = os.path.join(directory, filename)
212
217
213 return filename, metadata
218 return filename, metadata
214
219
215 def get_metadata(self, filename):
220 def get_metadata(self, filename):
216 """
221 """
217 Reads JSON stored metadata for a file
222 Reads JSON stored metadata for a file
218
223
219 :param filename:
224 :param filename:
220 :return:
225 :return:
221 """
226 """
222 filename = self.store_path(filename)
227 filename = self.store_path(filename)
223 filename_meta = filename + '.meta'
228 filename_meta = filename + '.meta'
224
229
225 with open(filename_meta, "rb") as source_meta:
230 with open(filename_meta, "rb") as source_meta:
226 return json.loads(source_meta.read())
231 return json.loads(source_meta.read())
General Comments 0
You need to be logged in to leave comments. Login now