##// END OF EJS Templates
file-store: the file backend now uses directory distribution to optimize file storage.
marcink -
r3454:b3d726a5 default
parent child
Show More
@@ -47,11 +47,22 class LocalFileStorage(object):
47 counter = 0
47 counter = 0
48 while True:
48 while True:
49 name = '%s-%d%s' % (basename, counter, ext)
49 name = '%s-%d%s' % (basename, counter, ext)
50 path = os.path.join(directory, name)
50
51 # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
52 sub_store = cls._sub_store_from_filename(basename)
53 sub_store_path = os.path.join(directory, sub_store)
54 if not os.path.exists(sub_store_path):
55 os.makedirs(sub_store_path)
56
57 path = os.path.join(sub_store_path, name)
51 if not os.path.exists(path):
58 if not os.path.exists(path):
52 return name, path
59 return name, path
53 counter += 1
60 counter += 1
54
61
62 @classmethod
63 def _sub_store_from_filename(cls, filename):
64 return filename[:2]
65
55 def __init__(self, base_path, extension_groups=None):
66 def __init__(self, base_path, extension_groups=None):
56
67
57 """
68 """
@@ -72,7 +83,8 class LocalFileStorage(object):
72
83
73 :param filename: base name of file
84 :param filename: base name of file
74 """
85 """
75 return os.path.join(self.base_path, filename)
86 sub_store = self._sub_store_from_filename(filename)
87 return os.path.join(self.base_path, sub_store, filename)
76
88
77 def delete(self, filename):
89 def delete(self, filename):
78 """
90 """
@@ -152,7 +164,6 class LocalFileStorage(object):
152 filename = utils.uid_filename(filename)
164 filename = utils.uid_filename(filename)
153
165
154 filename, path = self.resolve_name(filename, dest_directory)
166 filename, path = self.resolve_name(filename, dest_directory)
155 filename_meta = filename + '.meta'
156
167
157 file_obj.seek(0)
168 file_obj.seek(0)
158
169
@@ -166,7 +177,9 class LocalFileStorage(object):
166 "time": time.time(),
177 "time": time.time(),
167 "meta_ver": METADATA_VER})
178 "meta_ver": METADATA_VER})
168
179
169 with open(os.path.join(dest_directory, filename_meta), "wb") as dest_meta:
180 stored_file_path = os.path.dirname(path)
181 filename_meta = filename + '.meta'
182 with open(os.path.join(stored_file_path, filename_meta), "wb") as dest_meta:
170 dest_meta.write(json.dumps(metadata))
183 dest_meta.write(json.dumps(metadata))
171
184
172 if directory:
185 if directory:
@@ -67,9 +67,10 class TestFileStoreViews(TestController)
67
67
68 if exists:
68 if exists:
69 assert response.text == content
69 assert response.text == content
70 metadata = os.path.join(store_path, fid + '.meta')
70 file_store_path = os.path.dirname(store.resolve_name(fid, store_path)[1])
71 assert os.path.exists(metadata)
71 metadata_file = os.path.join(file_store_path, fid + '.meta')
72 with open(metadata, 'rb') as f:
72 assert os.path.exists(metadata_file)
73 with open(metadata_file, 'rb') as f:
73 json_data = json.loads(f.read())
74 json_data = json.loads(f.read())
74
75
75 assert json_data
76 assert json_data
General Comments 0
You need to be logged in to leave comments. Login now