##// END OF EJS Templates
artifacts: few fixes for handling cases of using sub path to store artifacts....
marcink -
r4476:8538fa60 default
parent child Browse files
Show More
@@ -147,12 +147,13 b' Use the following example to configure N'
147 147
148 148 ## Special Cache for file store, make sure you enable this intentionally as
149 149 ## it could bypass upload files permissions
150 # location /_file_store/download {
150 # location /_file_store/download/gravatars {
151 151 #
152 152 # proxy_cache cache_zone;
153 153 # # ignore Set-Cookie
154 154 # proxy_ignore_headers Set-Cookie;
155 # proxy_ignore_headers Cookie;
155 # # ignore cache-control
156 # proxy_ignore_headers Cache-Control;
156 157 #
157 158 # proxy_cache_key $host$uri$is_args$args;
158 159 # proxy_cache_methods GET;
@@ -43,10 +43,10 b' def includeme(config):'
43 43 pattern='/_file_store/upload')
44 44 config.add_route(
45 45 name='download_file',
46 pattern='/_file_store/download/{fid}')
46 pattern='/_file_store/download/{fid:.*}')
47 47 config.add_route(
48 48 name='download_file_by_token',
49 pattern='/_file_store/token-download/{_auth_token}/{fid}')
49 pattern='/_file_store/token-download/{_auth_token}/{fid:.*}')
50 50
51 51 # Scan module for configuration decorators.
52 52 config.scan('.views', ignore='.tests')
@@ -20,6 +20,7 b''
20 20
21 21 import os
22 22 import time
23 import errno
23 24 import shutil
24 25 import hashlib
25 26
@@ -32,9 +33,24 b' from rhodecode.apps.file_store.exception'
32 33 METADATA_VER = 'v1'
33 34
34 35
36 def safe_make_dirs(dir_path):
37 if not os.path.exists(dir_path):
38 try:
39 os.makedirs(dir_path)
40 except OSError as e:
41 if e.errno != errno.EEXIST:
42 raise
43 return
44
45
35 46 class LocalFileStorage(object):
36 47
37 48 @classmethod
49 def apply_counter(cls, counter, filename):
50 name_counted = '%d-%s' % (counter, filename)
51 return name_counted
52
53 @classmethod
38 54 def resolve_name(cls, name, directory):
39 55 """
40 56 Resolves a unique name and the correct path. If a filename
@@ -47,17 +63,16 b' class LocalFileStorage(object):'
47 63
48 64 counter = 0
49 65 while True:
50 name = '%d-%s' % (counter, name)
66 name_counted = cls.apply_counter(counter, name)
51 67
52 68 # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
53 sub_store = cls._sub_store_from_filename(name)
69 sub_store = cls._sub_store_from_filename(name_counted)
54 70 sub_store_path = os.path.join(directory, sub_store)
55 if not os.path.exists(sub_store_path):
56 os.makedirs(sub_store_path)
71 safe_make_dirs(sub_store_path)
57 72
58 path = os.path.join(sub_store_path, name)
73 path = os.path.join(sub_store_path, name_counted)
59 74 if not os.path.exists(path):
60 return name, path
75 return name_counted, path
61 76 counter += 1
62 77
63 78 @classmethod
@@ -102,8 +117,13 b' class LocalFileStorage(object):'
102 117
103 118 :param filename: base name of file
104 119 """
120 prefix_dir = ''
121 if '/' in filename:
122 prefix_dir, filename = filename.split('/')
105 123 sub_store = self._sub_store_from_filename(filename)
106 return os.path.join(self.base_path, sub_store, filename)
124 else:
125 sub_store = self._sub_store_from_filename(filename)
126 return os.path.join(self.base_path, prefix_dir, sub_store, filename)
107 127
108 128 def delete(self, filename):
109 129 """
@@ -123,7 +143,7 b' class LocalFileStorage(object):'
123 143 Checks if file exists. Resolves filename's absolute
124 144 path based on base_path.
125 145
126 :param filename: base name of file
146 :param filename: file_uid name of file, e.g 0-f62b2b2d-9708-4079-a071-ec3f958448d4.svg
127 147 """
128 148 return os.path.exists(self.store_path(filename))
129 149
@@ -158,7 +178,7 b' class LocalFileStorage(object):'
158 178 return ext in [normalize_ext(x) for x in extensions]
159 179
160 180 def save_file(self, file_obj, filename, directory=None, extensions=None,
161 extra_metadata=None, max_filesize=None, **kwargs):
181 extra_metadata=None, max_filesize=None, randomized_name=True, **kwargs):
162 182 """
163 183 Saves a file object to the uploads location.
164 184 Returns the resolved filename, i.e. the directory +
@@ -169,6 +189,7 b' class LocalFileStorage(object):'
169 189 :param directory: relative path of sub-directory
170 190 :param extensions: iterable of allowed extensions, if not default
171 191 :param max_filesize: maximum size of file that should be allowed
192 :param randomized_name: generate random generated UID or fixed based on the filename
172 193 :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
173 194
174 195 """
@@ -183,13 +204,12 b' class LocalFileStorage(object):'
183 204 else:
184 205 dest_directory = self.base_path
185 206
186 if not os.path.exists(dest_directory):
187 os.makedirs(dest_directory)
207 safe_make_dirs(dest_directory)
188 208
189 filename = utils.uid_filename(filename)
209 uid_filename = utils.uid_filename(filename, randomized=randomized_name)
190 210
191 211 # resolve also produces special sub-dir for file optimized store
192 filename, path = self.resolve_name(filename, dest_directory)
212 filename, path = self.resolve_name(uid_filename, dest_directory)
193 213 stored_file_dir = os.path.dirname(path)
194 214
195 215 file_obj.seek(0)
@@ -210,12 +230,13 b' class LocalFileStorage(object):'
210 230
211 231 file_hash = self.calculate_path_hash(path)
212 232
213 metadata.update(
214 {"filename": filename,
233 metadata.update({
234 "filename": filename,
215 235 "size": size,
216 236 "time": time.time(),
217 237 "sha256": file_hash,
218 "meta_ver": METADATA_VER})
238 "meta_ver": METADATA_VER
239 })
219 240
220 241 filename_meta = filename + '.meta'
221 242 with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
@@ -20,7 +20,7 b''
20 20
21 21
22 22 import uuid
23
23 import StringIO
24 24 import pathlib2
25 25
26 26
@@ -52,3 +52,7 b' def uid_filename(filename, randomized=Tr'
52 52 hash_key = '{}.{}'.format(filename, 'store')
53 53 uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
54 54 return str(uid) + ext.lower()
55
56
57 def bytes_to_file_obj(bytes_data):
58 return StringIO.StringIO(bytes_data)
@@ -64,7 +64,7 b' class FileStoreView(BaseAppView):'
64 64 file_uid, store_path)
65 65 raise HTTPNotFound()
66 66
67 db_obj = FileStore().query().filter(FileStore.file_uid == file_uid).scalar()
67 db_obj = FileStore.get_by_store_uid(file_uid, safe=True)
68 68 if not db_obj:
69 69 raise HTTPNotFound()
70 70
@@ -5442,7 +5442,10 b' class FileStore(Base, BaseModel):'
5442 5442 repo_group = relationship('RepoGroup', lazy='joined')
5443 5443
5444 5444 @classmethod
5445 def get_by_store_uid(cls, file_store_uid):
5445 def get_by_store_uid(cls, file_store_uid, safe=False):
5446 if safe:
5447 return FileStore.query().filter(FileStore.file_uid == file_store_uid).first()
5448 else:
5446 5449 return FileStore.query().filter(FileStore.file_uid == file_store_uid).scalar()
5447 5450
5448 5451 @classmethod
General Comments 0
You need to be logged in to leave comments. Login now