##// END OF EJS Templates
archives: optimize performance of repo archive option by delegating all logic to vcsserver....
milka -
r4536:2e292c98 default
parent child Browse files
Show More
@@ -402,8 +402,8 b' class RepoFilesView(RepoAppView):'
402 402 archive_dir_name = response_archive_name[:-len(ext)]
403 403
404 404 use_cached_archive = False
405 archive_cache_enabled = CONFIG.get(
406 'archive_cache_dir') and not self.request.GET.get('no_cache')
405 archive_cache_dir = CONFIG.get('archive_cache_dir')
406 archive_cache_enabled = archive_cache_dir and not self.request.GET.get('no_cache')
407 407 cached_archive_path = None
408 408
409 409 if archive_cache_enabled:
@@ -421,13 +421,13 b' class RepoFilesView(RepoAppView):'
421 421
422 422 # generate new archive, as previous was not found in the cache
423 423 if not use_cached_archive:
424
425 fd, archive = tempfile.mkstemp()
424 _dir = os.path.abspath(archive_cache_dir) if archive_cache_dir else None
425 fd, archive = tempfile.mkstemp(dir=_dir)
426 426 log.debug('Creating new temp archive in %s', archive)
427 427 try:
428 428 commit.archive_repo(archive, archive_dir_name=archive_dir_name,
429 429 kind=fileformat, subrepos=subrepos,
430 archive_at_path=at_path, with_hash=with_hash)
430 archive_at_path=at_path)
431 431 except ImproperArchiveTypeError:
432 432 return _('Unknown archive type')
433 433 if archive_cache_enabled:
@@ -915,8 +915,9 b' class BaseCommit(object):'
915 915 list of parent commits
916 916
917 917 """
918 repository = None
919 branch = None
918 920
919 branch = None
920 921 """
921 922 Depending on the backend this should be set to the branch name of the
922 923 commit. Backends not supporting branches on commits should leave this
@@ -1193,7 +1194,7 b' class BaseCommit(object):'
1193 1194
1194 1195 def archive_repo(self, archive_dest_path, kind='tgz', subrepos=None,
1195 1196 archive_dir_name=None, write_metadata=False, mtime=None,
1196 archive_at_path='/', with_hash=True):
1197 archive_at_path='/'):
1197 1198 """
1198 1199 Creates an archive containing the contents of the repository.
1199 1200
@@ -1216,29 +1217,12 b' class BaseCommit(object):'
1216 1217 (kind, allowed_kinds))
1217 1218
1218 1219 archive_dir_name = self._validate_archive_prefix(archive_dir_name)
1219
1220 1220 mtime = mtime is not None or time.mktime(self.date.timetuple())
1221
1222 file_info = []
1223 cur_rev = self.repository.get_commit(commit_id=self.raw_id)
1224 for _r, _d, files in cur_rev.walk(archive_at_path):
1225 for f in files:
1226 f_path = os.path.join(archive_dir_name, f.path)
1227 file_info.append(
1228 (f_path, f.mode, f.is_link(), f.raw_bytes))
1221 commit_id = self.raw_id
1229 1222
1230 if write_metadata:
1231 metadata = [
1232 ('repo_name', self.repository.name),
1233 ('commit_id', self.raw_id),
1234 ('mtime', mtime),
1235 ('branch', self.branch),
1236 ('tags', ','.join(self.tags)),
1237 ]
1238 meta = ["%s:%s" % (f_name, value) for f_name, value in metadata]
1239 file_info.append(('.archival.txt', 0o644, False, '\n'.join(meta)))
1240
1241 connection.Hg.archive_repo(archive_dest_path, mtime, file_info, kind)
1223 return self.repository._remote.archive_repo(
1224 archive_dest_path, kind, mtime, archive_at_path,
1225 archive_dir_name, commit_id)
1242 1226
1243 1227 def _validate_archive_prefix(self, archive_dir_name):
1244 1228 if archive_dir_name is None:
@@ -264,7 +264,6 b' input[type="button"] {'
264 264 margin-left: -1px;
265 265 padding-left: 2px;
266 266 padding-right: 2px;
267 border-left: 1px solid @grey3;
268 267 }
269 268 }
270 269
@@ -29,7 +29,7 b''
29 29 </a>
30 30
31 31 <div class="btn-action-switcher-container right-align">
32 <ul class="btn-action-switcher" role="menu" style="min-width: 200px">
32 <ul class="btn-action-switcher" role="menu" style="min-width: 200px; width: max-content">
33 33 <li>
34 34 <a class="action_button" href="${h.route_path('repo_files_upload_file',repo_name=c.repo_name,commit_id=c.commit.raw_id,f_path=c.f_path)}">
35 35 <i class="icon-upload"></i>
@@ -44,17 +44,40 b''
44 44 % endif
45 45
46 46 % if c.enable_downloads:
47 <% at_path = '{}'.format(request.GET.get('at') or c.commit.raw_id[:6]) %>
48 <div class="btn btn-default new-file">
49 % if c.f_path == '/':
50 <a href="${h.route_path('repo_archivefile',repo_name=c.repo_name, fname='{}.zip'.format(c.commit.raw_id))}">
51 ${_('Download full tree ZIP')}
47 <%
48 at_path = '{}'.format(request.GET.get('at') or c.commit.raw_id[:6])
49 if c.f_path == '/':
50 label = _('Full tree as {}')
51 _query = {'with_hash': '1'}
52 else:
53 label = _('This tree as {}')
54 _query = {'at_path':c.f_path, 'with_hash': '1'}
55 %>
56
57 <div class="btn-group btn-group-actions new-file">
58 <a class="archive_link btn btn-default" data-ext=".zip" href="${h.route_path('repo_archivefile',repo_name=c.rhodecode_db_repo.repo_name, fname='{}{}'.format(c.commit.raw_id, '.zip'), _query=_query)}">
59 <i class="icon-download"></i>
60 ${label.format('.zip')}
52 61 </a>
53 % else:
54 <a href="${h.route_path('repo_archivefile',repo_name=c.repo_name, fname='{}.zip'.format(c.commit.raw_id), _query={'at_path':c.f_path, 'with_hash': '1'})}">
55 ${_('Download this tree ZIP')}
62
63 <a class="tooltip btn btn-default btn-more-option" data-toggle="dropdown" aria-pressed="false" role="button" title="${_('more download options')}">
64 <i class="icon-down"></i>
56 65 </a>
66
67 <div class="btn-action-switcher-container left-align">
68 <ul class="btn-action-switcher" role="menu" style="min-width: 200px; width: max-content">
69 % for a_type, content_type, extension in h.ARCHIVE_SPECS:
70 % if extension not in ['.zip']:
71 <li>
72 <a class="archive_link" data-ext="${extension}" href="${h.route_path('repo_archivefile',repo_name=c.rhodecode_db_repo.repo_name, fname='{}{}'.format(c.commit.raw_id, extension), _query=_query)}">
73 <i class="icon-download"></i>
74 ${label.format(extension)}
75 </a>
76 </li>
57 77 % endif
78 % endfor
79 </ul>
80 </div>
58 81 </div>
59 82 % endif
60 83
@@ -198,7 +198,7 b''
198 198 </a>
199 199
200 200 <div class="btn-action-switcher-container left-align">
201 <ul class="btn-action-switcher" role="menu" style="min-width: 200px">
201 <ul class="btn-action-switcher" role="menu" style="min-width: 200px; width: max-content">
202 202 % for a_type, content_type, extension in h.ARCHIVE_SPECS:
203 203 % if extension not in ['.zip']:
204 204 <li>
@@ -49,23 +49,32 b' class TestArchives(BackendTestMixin):'
49 49 @classmethod
50 50 def _get_commits(cls):
51 51 start_date = datetime.datetime(2010, 1, 1, 20)
52 yield {
53 'message': 'Initial Commit',
54 'author': 'Joe Doe <joe.doe@example.com>',
55 'date': start_date + datetime.timedelta(hours=12),
56 'added': [
57 FileNode('executable_0o100755', '...', mode=0o100755),
58 FileNode('executable_0o100500', '...', mode=0o100500),
59 FileNode('not_executable', '...', mode=0o100644),
60 ],
61 }
52 62 for x in range(5):
53 63 yield {
54 64 'message': 'Commit %d' % x,
55 65 'author': 'Joe Doe <joe.doe@example.com>',
56 66 'date': start_date + datetime.timedelta(hours=12 * x),
57 67 'added': [
58 FileNode(
59 '%d/file_%d.txt' % (x, x), content='Foobar %d' % x),
68 FileNode('%d/file_%d.txt' % (x, x), content='Foobar %d' % x),
60 69 ],
61 70 }
62 71
63 72 @pytest.mark.parametrize('compressor', ['gz', 'bz2'])
64 73 def test_archive_tar(self, compressor):
65 74 self.tip.archive_repo(
66 self.temp_file, kind='t' + compressor, prefix='repo')
75 self.temp_file, kind='t{}'.format(compressor), archive_dir_name='repo')
67 76 out_dir = tempfile.mkdtemp()
68 out_file = tarfile.open(self.temp_file, 'r|' + compressor)
77 out_file = tarfile.open(self.temp_file, 'r|{}'.format(compressor))
69 78 out_file.extractall(out_dir)
70 79 out_file.close()
71 80
@@ -77,8 +86,24 b' class TestArchives(BackendTestMixin):'
77 86
78 87 shutil.rmtree(out_dir)
79 88
89 @pytest.mark.parametrize('compressor', ['gz', 'bz2'])
90 def test_archive_tar_symlink(self, compressor):
91 return False
92
93 @pytest.mark.parametrize('compressor', ['gz', 'bz2'])
94 def test_archive_tar_file_modes(self, compressor):
95 self.tip.archive_repo(
96 self.temp_file, kind='t{}'.format(compressor), archive_dir_name='repo')
97 out_dir = tempfile.mkdtemp()
98 out_file = tarfile.open(self.temp_file, 'r|{}'.format(compressor))
99 out_file.extractall(out_dir)
100 out_file.close()
101 dest = lambda inp: os.path.join(out_dir, 'repo/' + inp)
102
103 assert oct(os.stat(dest('not_executable')).st_mode) == '0100644'
104
80 105 def test_archive_zip(self):
81 self.tip.archive_repo(self.temp_file, kind='zip', prefix='repo')
106 self.tip.archive_repo(self.temp_file, kind='zip', archive_dir_name='repo')
82 107 out = zipfile.ZipFile(self.temp_file)
83 108
84 109 for x in range(5):
@@ -91,10 +116,10 b' class TestArchives(BackendTestMixin):'
91 116
92 117 def test_archive_zip_with_metadata(self):
93 118 self.tip.archive_repo(self.temp_file, kind='zip',
94 prefix='repo', write_metadata=True)
119 archive_dir_name='repo', write_metadata=True)
95 120
96 121 out = zipfile.ZipFile(self.temp_file)
97 metafile = out.read('.archival.txt')
122 metafile = out.read('repo/.archival.txt')
98 123
99 124 raw_id = self.tip.raw_id
100 125 assert 'commit_id:%s' % raw_id in metafile
General Comments 0
You need to be logged in to leave comments. Login now