##// END OF EJS Templates
artifacts: few fixes for handling cases of using sub path to store artifacts....
marcink -
r4476:8538fa60 default
parent child
Show More
@@ -147,12 +147,13 Use the following example to configure N
147
147
148 ## Special Cache for file store, make sure you enable this intentionally as
148 ## Special Cache for file store, make sure you enable this intentionally as
149 ## it could bypass upload files permissions
149 ## it could bypass upload files permissions
150 # location /_file_store/download {
150 # location /_file_store/download/gravatars {
151 #
151 #
152 # proxy_cache cache_zone;
152 # proxy_cache cache_zone;
153 # # ignore Set-Cookie
153 # # ignore Set-Cookie
154 # proxy_ignore_headers Set-Cookie;
154 # proxy_ignore_headers Set-Cookie;
155 # proxy_ignore_headers Cookie;
155 # # ignore cache-control
156 # proxy_ignore_headers Cache-Control;
156 #
157 #
157 # proxy_cache_key $host$uri$is_args$args;
158 # proxy_cache_key $host$uri$is_args$args;
158 # proxy_cache_methods GET;
159 # proxy_cache_methods GET;
@@ -43,10 +43,10 def includeme(config):
43 pattern='/_file_store/upload')
43 pattern='/_file_store/upload')
44 config.add_route(
44 config.add_route(
45 name='download_file',
45 name='download_file',
46 pattern='/_file_store/download/{fid}')
46 pattern='/_file_store/download/{fid:.*}')
47 config.add_route(
47 config.add_route(
48 name='download_file_by_token',
48 name='download_file_by_token',
49 pattern='/_file_store/token-download/{_auth_token}/{fid}')
49 pattern='/_file_store/token-download/{_auth_token}/{fid:.*}')
50
50
51 # Scan module for configuration decorators.
51 # Scan module for configuration decorators.
52 config.scan('.views', ignore='.tests')
52 config.scan('.views', ignore='.tests')
@@ -20,6 +20,7
20
20
21 import os
21 import os
22 import time
22 import time
23 import errno
23 import shutil
24 import shutil
24 import hashlib
25 import hashlib
25
26
@@ -32,9 +33,24 from rhodecode.apps.file_store.exception
32 METADATA_VER = 'v1'
33 METADATA_VER = 'v1'
33
34
34
35
36 def safe_make_dirs(dir_path):
37 if not os.path.exists(dir_path):
38 try:
39 os.makedirs(dir_path)
40 except OSError as e:
41 if e.errno != errno.EEXIST:
42 raise
43 return
44
45
35 class LocalFileStorage(object):
46 class LocalFileStorage(object):
36
47
37 @classmethod
48 @classmethod
49 def apply_counter(cls, counter, filename):
50 name_counted = '%d-%s' % (counter, filename)
51 return name_counted
52
53 @classmethod
38 def resolve_name(cls, name, directory):
54 def resolve_name(cls, name, directory):
39 """
55 """
40 Resolves a unique name and the correct path. If a filename
56 Resolves a unique name and the correct path. If a filename
@@ -47,17 +63,16 class LocalFileStorage(object):
47
63
48 counter = 0
64 counter = 0
49 while True:
65 while True:
50 name = '%d-%s' % (counter, name)
66 name_counted = cls.apply_counter(counter, name)
51
67
52 # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
68 # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
53 sub_store = cls._sub_store_from_filename(name)
69 sub_store = cls._sub_store_from_filename(name_counted)
54 sub_store_path = os.path.join(directory, sub_store)
70 sub_store_path = os.path.join(directory, sub_store)
55 if not os.path.exists(sub_store_path):
71 safe_make_dirs(sub_store_path)
56 os.makedirs(sub_store_path)
57
72
58 path = os.path.join(sub_store_path, name)
73 path = os.path.join(sub_store_path, name_counted)
59 if not os.path.exists(path):
74 if not os.path.exists(path):
60 return name, path
75 return name_counted, path
61 counter += 1
76 counter += 1
62
77
63 @classmethod
78 @classmethod
@@ -102,8 +117,13 class LocalFileStorage(object):
102
117
103 :param filename: base name of file
118 :param filename: base name of file
104 """
119 """
105 sub_store = self._sub_store_from_filename(filename)
120 prefix_dir = ''
106 return os.path.join(self.base_path, sub_store, filename)
121 if '/' in filename:
122 prefix_dir, filename = filename.split('/')
123 sub_store = self._sub_store_from_filename(filename)
124 else:
125 sub_store = self._sub_store_from_filename(filename)
126 return os.path.join(self.base_path, prefix_dir, sub_store, filename)
107
127
108 def delete(self, filename):
128 def delete(self, filename):
109 """
129 """
@@ -123,7 +143,7 class LocalFileStorage(object):
123 Checks if file exists. Resolves filename's absolute
143 Checks if file exists. Resolves filename's absolute
124 path based on base_path.
144 path based on base_path.
125
145
126 :param filename: base name of file
146 :param filename: file_uid name of file, e.g 0-f62b2b2d-9708-4079-a071-ec3f958448d4.svg
127 """
147 """
128 return os.path.exists(self.store_path(filename))
148 return os.path.exists(self.store_path(filename))
129
149
@@ -158,7 +178,7 class LocalFileStorage(object):
158 return ext in [normalize_ext(x) for x in extensions]
178 return ext in [normalize_ext(x) for x in extensions]
159
179
160 def save_file(self, file_obj, filename, directory=None, extensions=None,
180 def save_file(self, file_obj, filename, directory=None, extensions=None,
161 extra_metadata=None, max_filesize=None, **kwargs):
181 extra_metadata=None, max_filesize=None, randomized_name=True, **kwargs):
162 """
182 """
163 Saves a file object to the uploads location.
183 Saves a file object to the uploads location.
164 Returns the resolved filename, i.e. the directory +
184 Returns the resolved filename, i.e. the directory +
@@ -169,6 +189,7 class LocalFileStorage(object):
169 :param directory: relative path of sub-directory
189 :param directory: relative path of sub-directory
170 :param extensions: iterable of allowed extensions, if not default
190 :param extensions: iterable of allowed extensions, if not default
171 :param max_filesize: maximum size of file that should be allowed
191 :param max_filesize: maximum size of file that should be allowed
192 :param randomized_name: generate random generated UID or fixed based on the filename
172 :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
193 :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
173
194
174 """
195 """
@@ -183,13 +204,12 class LocalFileStorage(object):
183 else:
204 else:
184 dest_directory = self.base_path
205 dest_directory = self.base_path
185
206
186 if not os.path.exists(dest_directory):
207 safe_make_dirs(dest_directory)
187 os.makedirs(dest_directory)
188
208
189 filename = utils.uid_filename(filename)
209 uid_filename = utils.uid_filename(filename, randomized=randomized_name)
190
210
191 # resolve also produces special sub-dir for file optimized store
211 # resolve also produces special sub-dir for file optimized store
192 filename, path = self.resolve_name(filename, dest_directory)
212 filename, path = self.resolve_name(uid_filename, dest_directory)
193 stored_file_dir = os.path.dirname(path)
213 stored_file_dir = os.path.dirname(path)
194
214
195 file_obj.seek(0)
215 file_obj.seek(0)
@@ -210,12 +230,13 class LocalFileStorage(object):
210
230
211 file_hash = self.calculate_path_hash(path)
231 file_hash = self.calculate_path_hash(path)
212
232
213 metadata.update(
233 metadata.update({
214 {"filename": filename,
234 "filename": filename,
215 "size": size,
235 "size": size,
216 "time": time.time(),
236 "time": time.time(),
217 "sha256": file_hash,
237 "sha256": file_hash,
218 "meta_ver": METADATA_VER})
238 "meta_ver": METADATA_VER
239 })
219
240
220 filename_meta = filename + '.meta'
241 filename_meta = filename + '.meta'
221 with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
242 with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
@@ -20,7 +20,7
20
20
21
21
22 import uuid
22 import uuid
23
23 import StringIO
24 import pathlib2
24 import pathlib2
25
25
26
26
@@ -52,3 +52,7 def uid_filename(filename, randomized=Tr
52 hash_key = '{}.{}'.format(filename, 'store')
52 hash_key = '{}.{}'.format(filename, 'store')
53 uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
53 uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
54 return str(uid) + ext.lower()
54 return str(uid) + ext.lower()
55
56
57 def bytes_to_file_obj(bytes_data):
58 return StringIO.StringIO(bytes_data)
@@ -64,7 +64,7 class FileStoreView(BaseAppView):
64 file_uid, store_path)
64 file_uid, store_path)
65 raise HTTPNotFound()
65 raise HTTPNotFound()
66
66
67 db_obj = FileStore().query().filter(FileStore.file_uid == file_uid).scalar()
67 db_obj = FileStore.get_by_store_uid(file_uid, safe=True)
68 if not db_obj:
68 if not db_obj:
69 raise HTTPNotFound()
69 raise HTTPNotFound()
70
70
@@ -5442,8 +5442,11 class FileStore(Base, BaseModel):
5442 repo_group = relationship('RepoGroup', lazy='joined')
5442 repo_group = relationship('RepoGroup', lazy='joined')
5443
5443
5444 @classmethod
5444 @classmethod
5445 def get_by_store_uid(cls, file_store_uid):
5445 def get_by_store_uid(cls, file_store_uid, safe=False):
5446 return FileStore.query().filter(FileStore.file_uid == file_store_uid).scalar()
5446 if safe:
5447 return FileStore.query().filter(FileStore.file_uid == file_store_uid).first()
5448 else:
5449 return FileStore.query().filter(FileStore.file_uid == file_store_uid).scalar()
5447
5450
5448 @classmethod
5451 @classmethod
5449 def create(cls, file_uid, filename, file_hash, file_size, file_display_name='',
5452 def create(cls, file_uid, filename, file_hash, file_size, file_display_name='',
General Comments 0
You need to be logged in to leave comments. Login now