largefiles: enabled download of largefiles for git and mercurial from web interface.

- fixes  #2817

default.nix

@@ -226,8 +226,8 b' let'
   rhodecode-testdata-src = sources.rhodecode-testdata or (
     pkgs.fetchhg {
       url = "https://code.rhodecode.com/upstream/rc_testdata";
-      rev = "v0.9.0";
-      sha256 = "0k0ccb7cncd6mmzwckfbr6l7fsymcympwcm948qc3i0f0m6bbg1y";
+      rev = "v0.10.0";
+      sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0";
   });
 
   # Apply all overrides and fix the final package set

rhodecode/controllers/files.py

@@ -223,6 +223,8 b' class FilesController(BaseRepoController'
             c.file_author = True
             c.file_tree = ''
             if c.file.is_file():
+                c.lf_node = c.file.get_largefile_node()
+
                 c.file_source_page = 'true'
                 c.file_last_commit = c.file.last_commit
                 if c.file.size < self.cut_off_limit_file:
@@ -343,6 +345,14 b' class FilesController(BaseRepoController'
         commit = self.__get_commit_or_redirect(revision, repo_name)
         file_node = self.__get_filenode_or_redirect(repo_name, commit, f_path)
 
+        if request.GET.get('lf'):
+            # only if lf get flag is passed, we download this file
+            # as LFS/Largefile
+            lf_node = file_node.get_largefile_node()
+            if lf_node:
+                # overwrite our pointer with the REAL large-file
+                file_node = lf_node
+
         response.content_disposition = 'attachment; filename=%s' % \
             safe_str(f_path.split(Repository.NAME_SEP)[-1])
 

rhodecode/lib/vcs/backends/base.py

@@ -893,9 +893,10 b' class BaseCommit(object):'
 
     def get_largefile_node(self, path):
         """
-        Returns the path to largefile from Mercurial storage.
+        Returns the path to largefile from Mercurial/Git-lfs storage.
+        or None if it's not a largefile node
         """
-        raise NotImplementedError
+        return None
 
     def archive_repo(self, file_path, kind='tgz', subrepos=None,
                      prefix=None, write_metadata=False, mtime=None):

rhodecode/lib/vcs/backends/git/commit.py

@@ -39,7 +39,7 b' from rhodecode.lib.vcs.exceptions import'
 from rhodecode.lib.vcs.nodes import (
     FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
     ChangedFileNodesGenerator, AddedFileNodesGenerator,
-    RemovedFileNodesGenerator)
+    RemovedFileNodesGenerator, LargeFileNode)
 
 
 class GitCommit(base.BaseCommit):
@@ -423,6 +423,17 b' class GitCommit(base.BaseCommit):'
             self.nodes[path] = node
         return self.nodes[path]
 
+    def get_largefile_node(self, path):
+        id_, _ = self._get_id_for_path(path)
+        pointer_spec = self._remote.is_large_file(id_)
+
+        if pointer_spec:
+            # content of that file regular FileNode is the hash of largefile
+            file_id = pointer_spec.get('oid_hash')
+            if self._remote.in_largefiles_store(file_id):
+                lf_path = self._remote.store_path(file_id)
+                return LargeFileNode(lf_path, commit=self, org_path=path)
+
     @LazyProperty
     def affected_files(self):
         """

rhodecode/lib/vcs/backends/hg/commit.py

@@ -312,18 +312,18 b' class MercurialCommit(base.BaseCommit):'
         return self.nodes[path]
 
     def get_largefile_node(self, path):
-        path = os.path.join(LARGEFILE_PREFIX, path)
 
         if self._remote.is_large_file(path):
             # content of that file regular FileNode is the hash of largefile
             file_id = self.get_file_content(path).strip()
-            if self._remote.in_store(file_id):
-                path = self._remote.store_path(file_id)
-                return LargeFileNode(path, commit=self)
+
+            if self._remote.in_largefiles_store(file_id):
+                lf_path = self._remote.store_path(file_id)
+                return LargeFileNode(lf_path, commit=self, org_path=path)
             elif self._remote.in_user_cache(file_id):
-                path = self._remote.store_path(file_id)
+                lf_path = self._remote.store_path(file_id)
                 self._remote.link(file_id, path)
-                return LargeFileNode(path, commit=self)
+                return LargeFileNode(lf_path, commit=self, org_path=path)
 
     @LazyProperty
     def _submodules(self):

rhodecode/lib/vcs/nodes.py

@@ -22,7 +22,7 b''
 Module holding everything related to vcs nodes, with vcs2 architecture.
 """
 
-
+import os
 import stat
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
@@ -547,9 +547,8 b' class FileNode(Node):'
         create special instance of LargeFileNode which can get content from
         LF store.
         """
-        if self.commit and self.path.startswith(LARGEFILE_PREFIX):
-            largefile_path = self.path.split(LARGEFILE_PREFIX)[-1].lstrip('/')
-            return self.commit.get_largefile_node(largefile_path)
+        if self.commit:
+            return self.commit.get_largefile_node(self.path)
 
     def lines(self, count_empty=False):
         all_lines, empty_lines = 0, 0
@@ -765,15 +764,37 b' class SubModuleNode(Node):'
 
 class LargeFileNode(FileNode):
 
+    def __init__(self, path, url=None, commit=None, alias=None, org_path=None):
+        self.path = path
+        self.org_path = org_path
+        self.kind = NodeKind.LARGEFILE
+        self.alias = alias
+
     def _validate_path(self, path):
         """
         we override check since the LargeFileNode path is system absolute
         """
+        pass
 
+    def __repr__(self):
+        return '<%s %r>' % (self.__class__.__name__, self.path)
+
+    @LazyProperty
+    def size(self):
+        return os.stat(self.path).st_size
+
+    @LazyProperty
     def raw_bytes(self):
         if self.commit:
             with open(self.path, 'rb') as f:
                 content = f.read()
         else:
             content = self._content
-        return content
  No newline at end of file
+        return content
+
+    @LazyProperty
+    def name(self):
+        """
+        Overwrites name to be the org lf path
+        """
+        return self.org_path

rhodecode/templates/files/files_source.mako

@@ -4,6 +4,9 b''
     <div class="codeblock-header">
       <div class="stats">
             <span> <strong>${c.file}</strong></span>
+            % if c.lf_node:
+            <span title="${_('This file is a pointer to large binary file')}"> | ${_('LargeFile')} ${h.format_byte_size_binary(c.lf_node.size)} </span>
+            % endif
             <span> | ${c.file.lines()[0]} ${ungettext('line', 'lines', c.file.lines()[0])}</span>
             <span> | ${h.format_byte_size_binary(c.file.size)}</span>
             <span> | ${c.file.mimetype} </span>
@@ -22,9 +25,16 b''
           ${h.link_to(_('Annotation'), h.url('files_annotate_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
         %endif
          | ${h.link_to(_('Raw'), h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
-         | <a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path)}">
-          ${_('Download')}
-        </a>
+         |
+          % if c.lf_node:
+              <a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path, lf=1)}">
+              ${_('Download largefile')}
+              </a>
+          % else:
+              <a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path)}">
+              ${_('Download')}
+              </a>
+          % endif
 
         %if h.HasRepoPermissionAny('repository.write','repository.admin')(c.repo_name):
            |
@@ -46,11 +56,11 b''
     </div>
     <div id="file_history_container"></div>
     <div class="code-body">
-       %if c.file.is_binary:
+     %if c.file.is_binary:
            <div>
            ${_('Binary file (%s)') % c.file.mimetype}
            </div>
-       %else:
+     %else:
         % if c.file.size < c.cut_off_limit:
             %if c.renderer and not c.annotate:
                 ${h.render(c.file.content, renderer=c.renderer, relative_url=h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
@@ -67,7 +77,6 b''
                   %endfor
                 %endif
                 </table>
-              </div>
             %endif
         %else:
             ${_('File is too big to display')} ${h.link_to(_('Show as raw'),

rhodecode/tests/plugin.py

@@ -387,20 +387,22 b' class TestRepoContainer(object):'
         self._fixture = Fixture()
         self._repos = {}
 
-    def __call__(self, dump_name, backend_alias):
+    def __call__(self, dump_name, backend_alias, config=None):
         key = (dump_name, backend_alias)
         if key not in self._repos:
-            repo = self._create_repo(dump_name, backend_alias)
+            repo = self._create_repo(dump_name, backend_alias, config)
             self._repos[key] = repo.repo_id
         return Repository.get(self._repos[key])
 
-    def _create_repo(self, dump_name, backend_alias):
+    def _create_repo(self, dump_name, backend_alias, config):
         repo_name = '%s-%s' % (backend_alias, dump_name)
         backend_class = get_backend(backend_alias)
         dump_extractor = self.dump_extractors[backend_alias]
         repo_path = dump_extractor(dump_name, repo_name)
-        vcs_repo = backend_class(repo_path)
+
+        vcs_repo = backend_class(repo_path, config=config)
         repo2db_mapper({repo_name: vcs_repo})
+
         repo = RepoModel().get_by_repo_name(repo_name)
         self._cleanup_repos.append(repo_name)
         return repo
@@ -515,6 +517,9 b' class Backend(object):'
     def __getitem__(self, key):
         return self._test_repo_container(key, self.alias)
 
+    def create_test_repo(self, key, config=None):
+        return self._test_repo_container(key, self.alias, config)
+
     @property
     def repo(self):
         """

rhodecode/tests/vcs/test_git.py

@@ -22,15 +22,16 b' import datetime'
 import mock
 import os
 import sys
+import shutil
 
 import pytest
 
+from rhodecode.lib.utils import make_db_config
 from rhodecode.lib.vcs.backends.base import Reference
 from rhodecode.lib.vcs.backends.git import (
     GitRepository, GitCommit, discover_git_version)
 from rhodecode.lib.vcs.exceptions import (
-    RepositoryError, VCSError, NodeDoesNotExistError
-)
+    RepositoryError, VCSError, NodeDoesNotExistError)
 from rhodecode.lib.vcs.nodes import (
     NodeKind, FileNode, DirNode, NodeState, SubModuleNode)
 from rhodecode.tests import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, get_new_dir
@@ -1003,6 +1004,32 b' class TestGitCommit(object):'
         assert author == commit.author_name
 
 
+class TestLargeFileRepo(object):
+
+    def test_large_file(self, backend_git):
+        conf = make_db_config()
+        repo = backend_git.create_test_repo('largefiles', conf)
+
+        tip = repo.scm_instance().get_commit()
+
+        # extract stored LF node into the origin cache
+        lfs_store = os.path.join(repo.repo_path, repo.repo_name, 'lfs_store')
+
+        oid = '7b331c02e313c7599d5a90212e17e6d3cb729bd2e1c9b873c302a63c95a2f9bf'
+        oid_path = os.path.join(lfs_store, oid)
+        oid_destination = os.path.join(
+            conf.get('vcs_git_lfs', 'store_location'), oid)
+        shutil.copy(oid_path, oid_destination)
+
+        node = tip.get_node('1MB.zip')
+
+        lf_node = node.get_largefile_node()
+
+        assert lf_node.is_largefile() is True
+        assert lf_node.size == 1024000
+        assert lf_node.name == '1MB.zip'
+
+
 class TestGitSpecificWithRepo(BackendTestMixin):
 
     @classmethod

rhodecode/tests/vcs/test_hg.py

@@ -23,14 +23,13 b' import os'
 import mock
 import pytest
 
-import rhodecode.lib.vcs.conf.settings
+from rhodecode.lib.utils import make_db_config
 from rhodecode.lib.vcs import backends
 from rhodecode.lib.vcs.backends.base import (
     Reference, MergeResponse, MergeFailureReason)
 from rhodecode.lib.vcs.backends.hg import MercurialRepository, MercurialCommit
 from rhodecode.lib.vcs.exceptions import (
-    CommitError, RepositoryError, VCSError, NodeDoesNotExistError,
-    CommitDoesNotExistError)
+    RepositoryError, VCSError, NodeDoesNotExistError, CommitDoesNotExistError)
 from rhodecode.lib.vcs.nodes import FileNode, NodeKind, NodeState
 from rhodecode.tests import TEST_HG_REPO, TEST_HG_REPO_CLONE
 
@@ -1095,12 +1094,6 b' class TestMercurialCommit(object):'
         with pytest.raises(VCSError):
             self.repo.get_commit().get_node(path)
 
-    def test_large_file(self):
-        # TODO: valid large file
-        tip = self.repo.get_commit()
-        with pytest.raises(CommitError):
-            tip.get_largefile_node("invalid")
-
     def test_author_email(self):
         assert '[e-mail]' == \
             self.repo.get_commit('b986218ba1c9').author_email
@@ -1117,6 +1110,21 b' class TestMercurialCommit(object):'
             self.repo.get_commit('84478366594b').author_name
 
 
+class TestLargeFileRepo(object):
+
+    def test_large_file(self, backend_hg):
+        repo = backend_hg.create_test_repo('largefiles', make_db_config())
+
+        tip = repo.scm_instance().get_commit()
+        node = tip.get_node('.hglf/thisfileislarge')
+
+        lf_node = node.get_largefile_node()
+
+        assert lf_node.is_largefile() is True
+        assert lf_node.size == 1024000
+        assert lf_node.name == '.hglf/thisfileislarge'
+
+
 class TestGetBranchName(object):
     def test_returns_ref_name_when_type_is_branch(self):
         ref = self._create_ref('branch', 'fake-name')