archives: optimize performance of repo archive option by delegating all logic to vcsserver.

- sending GBs of data over the wire was slow and very memory consuming

rhodecode/apps/repository/views/repo_files.py

@@ -402,8 +402,8 b' class RepoFilesView(RepoAppView):'
         archive_dir_name = response_archive_name[:-len(ext)]
 
         use_cached_archive = False
-        archive_cache_enabled = CONFIG.get(
-            'archive_cache_dir') and not self.request.GET.get('no_cache')
+        archive_cache_dir = CONFIG.get('archive_cache_dir')
+        archive_cache_enabled = archive_cache_dir and not self.request.GET.get('no_cache')
         cached_archive_path = None
 
         if archive_cache_enabled:
@@ -421,13 +421,13 b' class RepoFilesView(RepoAppView):'
 
         # generate new archive, as previous was not found in the cache
         if not use_cached_archive:
-
-            fd, archive = tempfile.mkstemp()
+            _dir = os.path.abspath(archive_cache_dir) if archive_cache_dir else None
+            fd, archive = tempfile.mkstemp(dir=_dir)
             log.debug('Creating new temp archive in %s', archive)
             try:
                 commit.archive_repo(archive, archive_dir_name=archive_dir_name,
                                     kind=fileformat, subrepos=subrepos,
-                                    archive_at_path=at_path, with_hash=with_hash)
+                                    archive_at_path=at_path)
             except ImproperArchiveTypeError:
                 return _('Unknown archive type')
             if archive_cache_enabled:

rhodecode/lib/vcs/backends/base.py

@@ -915,8 +915,9 b' class BaseCommit(object):'
             list of parent commits
 
     """
+    repository = None
+    branch = None
 
-    branch = None
     """
     Depending on the backend this should be set to the branch name of the
     commit. Backends not supporting branches on commits should leave this
@@ -1193,7 +1194,7 b' class BaseCommit(object):'
 
     def archive_repo(self, archive_dest_path, kind='tgz', subrepos=None,
                      archive_dir_name=None, write_metadata=False, mtime=None,
-                     archive_at_path='/', with_hash=True):
+                     archive_at_path='/'):
         """
         Creates an archive containing the contents of the repository.
 
@@ -1216,29 +1217,12 b' class BaseCommit(object):'
                 (kind, allowed_kinds))
 
         archive_dir_name = self._validate_archive_prefix(archive_dir_name)
-
         mtime = mtime is not None or time.mktime(self.date.timetuple())
-
-        file_info = []
-        cur_rev = self.repository.get_commit(commit_id=self.raw_id)
-        for _r, _d, files in cur_rev.walk(archive_at_path):
-            for f in files:
-                f_path = os.path.join(archive_dir_name, f.path)
-                file_info.append(
-                    (f_path, f.mode, f.is_link(), f.raw_bytes))
+        commit_id = self.raw_id
 
-        if write_metadata:
-            metadata = [
-                ('repo_name', self.repository.name),
-                ('commit_id', self.raw_id),
-                ('mtime', mtime),
-                ('branch', self.branch),
-                ('tags', ','.join(self.tags)),
-            ]
-            meta = ["%s:%s" % (f_name, value) for f_name, value in metadata]
-            file_info.append(('.archival.txt', 0o644, False, '\n'.join(meta)))
-
-        connection.Hg.archive_repo(archive_dest_path, mtime, file_info, kind)
+        return self.repository._remote.archive_repo(
+            archive_dest_path, kind, mtime, archive_at_path,
+            archive_dir_name, commit_id)
 
     def _validate_archive_prefix(self, archive_dir_name):
         if archive_dir_name is None:

rhodecode/public/css/buttons.less

@@ -264,7 +264,6 b' input[type="button"] {'
         margin-left: -1px;
         padding-left: 2px;
         padding-right: 2px;
-        border-left: 1px solid @grey3;
     }
 }
 

rhodecode/templates/files/files_browser.mako

@@ -29,7 +29,7 b''
                     </a>
 
                     <div class="btn-action-switcher-container right-align">
-                        <ul class="btn-action-switcher" role="menu" style="min-width: 200px">
+                        <ul class="btn-action-switcher" role="menu" style="min-width: 200px; width: max-content">
                             <li>
                                 <a class="action_button" href="${h.route_path('repo_files_upload_file',repo_name=c.repo_name,commit_id=c.commit.raw_id,f_path=c.f_path)}">
                                     <i class="icon-upload"></i>
@@ -44,18 +44,41 b''
             % endif
 
             % if c.enable_downloads:
-              <% at_path = '{}'.format(request.GET.get('at') or c.commit.raw_id[:6]) %>
-              <div class="btn btn-default new-file">
-                  % if c.f_path == '/':
-                    <a href="${h.route_path('repo_archivefile',repo_name=c.repo_name, fname='{}.zip'.format(c.commit.raw_id))}">
-                        ${_('Download full tree ZIP')}
+              <%
+                  at_path = '{}'.format(request.GET.get('at') or c.commit.raw_id[:6])
+                  if c.f_path == '/':
+                      label = _('Full tree as {}')
+                      _query = {'with_hash': '1'}
+                  else:
+                      label = _('This tree as {}')
+                      _query = {'at_path':c.f_path, 'with_hash': '1'}
+              %>
+
+                <div class="btn-group btn-group-actions new-file">
+                    <a class="archive_link btn btn-default" data-ext=".zip" href="${h.route_path('repo_archivefile',repo_name=c.rhodecode_db_repo.repo_name, fname='{}{}'.format(c.commit.raw_id, '.zip'), _query=_query)}">
+                        <i class="icon-download"></i>
+                        ${label.format('.zip')}
                     </a>
-                  % else:
-                    <a href="${h.route_path('repo_archivefile',repo_name=c.repo_name, fname='{}.zip'.format(c.commit.raw_id), _query={'at_path':c.f_path, 'with_hash': '1'})}">
-                        ${_('Download this tree ZIP')}
+
+                    <a class="tooltip btn btn-default btn-more-option" data-toggle="dropdown" aria-pressed="false" role="button" title="${_('more download options')}">
+                        <i class="icon-down"></i>
                     </a>
-                  % endif
-              </div>
+
+                    <div class="btn-action-switcher-container left-align">
+                        <ul class="btn-action-switcher" role="menu" style="min-width: 200px; width: max-content">
+                            % for a_type, content_type, extension in h.ARCHIVE_SPECS:
+                            % if extension not in ['.zip']:
+                            <li>
+                                <a class="archive_link" data-ext="${extension}" href="${h.route_path('repo_archivefile',repo_name=c.rhodecode_db_repo.repo_name, fname='{}{}'.format(c.commit.raw_id, extension), _query=_query)}">
+                                    <i class="icon-download"></i>
+                                    ${label.format(extension)}
+                                </a>
+                            </li>
+                            % endif
+                            % endfor
+                        </ul>
+                    </div>
+                </div>
             % endif
 
             <div class="files-quick-filter">

rhodecode/templates/summary/components.mako

@@ -198,7 +198,7 b''
                             </a>
 
                             <div class="btn-action-switcher-container left-align">
-                                <ul class="btn-action-switcher" role="menu" style="min-width: 200px">
+                                <ul class="btn-action-switcher" role="menu" style="min-width: 200px; width: max-content">
                                     % for a_type, content_type, extension in h.ARCHIVE_SPECS:
                                     % if extension not in ['.zip']:
                                     <li>

rhodecode/tests/vcs/test_archives.py

@@ -49,23 +49,32 b' class TestArchives(BackendTestMixin):'
     @classmethod
     def _get_commits(cls):
         start_date = datetime.datetime(2010, 1, 1, 20)
+        yield {
+            'message': 'Initial Commit',
+            'author': 'Joe Doe <[e-mail]>',
+            'date': start_date + datetime.timedelta(hours=12),
+            'added': [
+                FileNode('executable_0o100755', '...', mode=0o100755),
+                FileNode('executable_0o100500', '...', mode=0o100500),
+                FileNode('not_executable', '...', mode=0o100644),
+            ],
+        }
         for x in range(5):
             yield {
                 'message': 'Commit %d' % x,
                 'author': 'Joe Doe <[e-mail]>',
                 'date': start_date + datetime.timedelta(hours=12 * x),
                 'added': [
-                    FileNode(
-                        '%d/file_%d.txt' % (x, x), content='Foobar %d' % x),
+                    FileNode('%d/file_%d.txt' % (x, x), content='Foobar %d' % x),
                 ],
             }
 
     @pytest.mark.parametrize('compressor', ['gz', 'bz2'])
     def test_archive_tar(self, compressor):
         self.tip.archive_repo(
-            self.temp_file, kind='t' + compressor, prefix='repo')
+            self.temp_file, kind='t{}'.format(compressor), archive_dir_name='repo')
         out_dir = tempfile.mkdtemp()
-        out_file = tarfile.open(self.temp_file, 'r|' + compressor)
+        out_file = tarfile.open(self.temp_file, 'r|{}'.format(compressor))
         out_file.extractall(out_dir)
         out_file.close()
 
@@ -77,8 +86,24 b' class TestArchives(BackendTestMixin):'
 
         shutil.rmtree(out_dir)
 
+    @pytest.mark.parametrize('compressor', ['gz', 'bz2'])
+    def test_archive_tar_symlink(self, compressor):
+        return False
+
+    @pytest.mark.parametrize('compressor', ['gz', 'bz2'])
+    def test_archive_tar_file_modes(self, compressor):
+        self.tip.archive_repo(
+            self.temp_file, kind='t{}'.format(compressor), archive_dir_name='repo')
+        out_dir = tempfile.mkdtemp()
+        out_file = tarfile.open(self.temp_file, 'r|{}'.format(compressor))
+        out_file.extractall(out_dir)
+        out_file.close()
+        dest = lambda inp: os.path.join(out_dir, 'repo/' + inp)
+
+        assert oct(os.stat(dest('not_executable')).st_mode) == '0100644'
+
     def test_archive_zip(self):
-        self.tip.archive_repo(self.temp_file, kind='zip', prefix='repo')
+        self.tip.archive_repo(self.temp_file, kind='zip', archive_dir_name='repo')
         out = zipfile.ZipFile(self.temp_file)
 
         for x in range(5):
@@ -91,10 +116,10 b' class TestArchives(BackendTestMixin):'
 
     def test_archive_zip_with_metadata(self):
         self.tip.archive_repo(self.temp_file, kind='zip',
-                              prefix='repo', write_metadata=True)
+                              archive_dir_name='repo', write_metadata=True)
 
         out = zipfile.ZipFile(self.temp_file)
-        metafile = out.read('.archival.txt')
+        metafile = out.read('repo/.archival.txt')
 
         raw_id = self.tip.raw_id
         assert 'commit_id:%s' % raw_id in metafile