diff --git a/default.nix b/default.nix --- a/default.nix +++ b/default.nix @@ -226,8 +226,8 @@ let rhodecode-testdata-src = sources.rhodecode-testdata or ( pkgs.fetchhg { url = "https://code.rhodecode.com/upstream/rc_testdata"; - rev = "v0.9.0"; - sha256 = "0k0ccb7cncd6mmzwckfbr6l7fsymcympwcm948qc3i0f0m6bbg1y"; + rev = "v0.10.0"; + sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0"; }); # Apply all overrides and fix the final package set diff --git a/rhodecode/controllers/files.py b/rhodecode/controllers/files.py --- a/rhodecode/controllers/files.py +++ b/rhodecode/controllers/files.py @@ -223,6 +223,8 @@ class FilesController(BaseRepoController c.file_author = True c.file_tree = '' if c.file.is_file(): + c.lf_node = c.file.get_largefile_node() + c.file_source_page = 'true' c.file_last_commit = c.file.last_commit if c.file.size < self.cut_off_limit_file: @@ -343,6 +345,14 @@ class FilesController(BaseRepoController commit = self.__get_commit_or_redirect(revision, repo_name) file_node = self.__get_filenode_or_redirect(repo_name, commit, f_path) + if request.GET.get('lf'): + # only if lf get flag is passed, we download this file + # as LFS/Largefile + lf_node = file_node.get_largefile_node() + if lf_node: + # overwrite our pointer with the REAL large-file + file_node = lf_node + response.content_disposition = 'attachment; filename=%s' % \ safe_str(f_path.split(Repository.NAME_SEP)[-1]) diff --git a/rhodecode/lib/vcs/backends/base.py b/rhodecode/lib/vcs/backends/base.py --- a/rhodecode/lib/vcs/backends/base.py +++ b/rhodecode/lib/vcs/backends/base.py @@ -893,9 +893,10 @@ class BaseCommit(object): def get_largefile_node(self, path): """ - Returns the path to largefile from Mercurial storage. + Returns the path to largefile from Mercurial/Git-lfs storage. + or None if it's not a largefile node """ - raise NotImplementedError + return None def archive_repo(self, file_path, kind='tgz', subrepos=None, prefix=None, write_metadata=False, mtime=None): diff --git a/rhodecode/lib/vcs/backends/git/commit.py b/rhodecode/lib/vcs/backends/git/commit.py --- a/rhodecode/lib/vcs/backends/git/commit.py +++ b/rhodecode/lib/vcs/backends/git/commit.py @@ -39,7 +39,7 @@ from rhodecode.lib.vcs.exceptions import from rhodecode.lib.vcs.nodes import ( FileNode, DirNode, NodeKind, RootNode, SubModuleNode, ChangedFileNodesGenerator, AddedFileNodesGenerator, - RemovedFileNodesGenerator) + RemovedFileNodesGenerator, LargeFileNode) class GitCommit(base.BaseCommit): @@ -423,6 +423,17 @@ class GitCommit(base.BaseCommit): self.nodes[path] = node return self.nodes[path] + def get_largefile_node(self, path): + id_, _ = self._get_id_for_path(path) + pointer_spec = self._remote.is_large_file(id_) + + if pointer_spec: + # content of that file regular FileNode is the hash of largefile + file_id = pointer_spec.get('oid_hash') + if self._remote.in_largefiles_store(file_id): + lf_path = self._remote.store_path(file_id) + return LargeFileNode(lf_path, commit=self, org_path=path) + @LazyProperty def affected_files(self): """ diff --git a/rhodecode/lib/vcs/backends/hg/commit.py b/rhodecode/lib/vcs/backends/hg/commit.py --- a/rhodecode/lib/vcs/backends/hg/commit.py +++ b/rhodecode/lib/vcs/backends/hg/commit.py @@ -312,18 +312,18 @@ class MercurialCommit(base.BaseCommit): return self.nodes[path] def get_largefile_node(self, path): - path = os.path.join(LARGEFILE_PREFIX, path) if self._remote.is_large_file(path): # content of that file regular FileNode is the hash of largefile file_id = self.get_file_content(path).strip() - if self._remote.in_store(file_id): - path = self._remote.store_path(file_id) - return LargeFileNode(path, commit=self) + + if self._remote.in_largefiles_store(file_id): + lf_path = self._remote.store_path(file_id) + return LargeFileNode(lf_path, commit=self, org_path=path) elif self._remote.in_user_cache(file_id): - path = self._remote.store_path(file_id) + lf_path = self._remote.store_path(file_id) self._remote.link(file_id, path) - return LargeFileNode(path, commit=self) + return LargeFileNode(lf_path, commit=self, org_path=path) @LazyProperty def _submodules(self): diff --git a/rhodecode/lib/vcs/nodes.py b/rhodecode/lib/vcs/nodes.py --- a/rhodecode/lib/vcs/nodes.py +++ b/rhodecode/lib/vcs/nodes.py @@ -22,7 +22,7 @@ Module holding everything related to vcs nodes, with vcs2 architecture. """ - +import os import stat from zope.cachedescriptors.property import Lazy as LazyProperty @@ -547,9 +547,8 @@ class FileNode(Node): create special instance of LargeFileNode which can get content from LF store. """ - if self.commit and self.path.startswith(LARGEFILE_PREFIX): - largefile_path = self.path.split(LARGEFILE_PREFIX)[-1].lstrip('/') - return self.commit.get_largefile_node(largefile_path) + if self.commit: + return self.commit.get_largefile_node(self.path) def lines(self, count_empty=False): all_lines, empty_lines = 0, 0 @@ -765,15 +764,37 @@ class SubModuleNode(Node): class LargeFileNode(FileNode): + def __init__(self, path, url=None, commit=None, alias=None, org_path=None): + self.path = path + self.org_path = org_path + self.kind = NodeKind.LARGEFILE + self.alias = alias + def _validate_path(self, path): """ we override check since the LargeFileNode path is system absolute """ + pass + def __repr__(self): + return '<%s %r>' % (self.__class__.__name__, self.path) + + @LazyProperty + def size(self): + return os.stat(self.path).st_size + + @LazyProperty def raw_bytes(self): if self.commit: with open(self.path, 'rb') as f: content = f.read() else: content = self._content - return content \ No newline at end of file + return content + + @LazyProperty + def name(self): + """ + Overwrites name to be the org lf path + """ + return self.org_path diff --git a/rhodecode/templates/files/files_source.mako b/rhodecode/templates/files/files_source.mako --- a/rhodecode/templates/files/files_source.mako +++ b/rhodecode/templates/files/files_source.mako @@ -4,6 +4,9 @@
${c.file} + % if c.lf_node: + | ${_('LargeFile')} ${h.format_byte_size_binary(c.lf_node.size)} + % endif | ${c.file.lines()[0]} ${ungettext('line', 'lines', c.file.lines()[0])} | ${h.format_byte_size_binary(c.file.size)} | ${c.file.mimetype} @@ -22,9 +25,16 @@ ${h.link_to(_('Annotation'), h.url('files_annotate_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))} %endif | ${h.link_to(_('Raw'), h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))} - | - ${_('Download')} - + | + % if c.lf_node: + + ${_('Download largefile')} + + % else: + + ${_('Download')} + + % endif %if h.HasRepoPermissionAny('repository.write','repository.admin')(c.repo_name): | @@ -46,11 +56,11 @@
- %if c.file.is_binary: + %if c.file.is_binary:
${_('Binary file (%s)') % c.file.mimetype}
- %else: + %else: % if c.file.size < c.cut_off_limit: %if c.renderer and not c.annotate: ${h.render(c.file.content, renderer=c.renderer, relative_url=h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))} @@ -67,7 +77,6 @@ %endfor %endif -
%endif %else: ${_('File is too big to display')} ${h.link_to(_('Show as raw'), diff --git a/rhodecode/tests/plugin.py b/rhodecode/tests/plugin.py --- a/rhodecode/tests/plugin.py +++ b/rhodecode/tests/plugin.py @@ -387,20 +387,22 @@ class TestRepoContainer(object): self._fixture = Fixture() self._repos = {} - def __call__(self, dump_name, backend_alias): + def __call__(self, dump_name, backend_alias, config=None): key = (dump_name, backend_alias) if key not in self._repos: - repo = self._create_repo(dump_name, backend_alias) + repo = self._create_repo(dump_name, backend_alias, config) self._repos[key] = repo.repo_id return Repository.get(self._repos[key]) - def _create_repo(self, dump_name, backend_alias): + def _create_repo(self, dump_name, backend_alias, config): repo_name = '%s-%s' % (backend_alias, dump_name) backend_class = get_backend(backend_alias) dump_extractor = self.dump_extractors[backend_alias] repo_path = dump_extractor(dump_name, repo_name) - vcs_repo = backend_class(repo_path) + + vcs_repo = backend_class(repo_path, config=config) repo2db_mapper({repo_name: vcs_repo}) + repo = RepoModel().get_by_repo_name(repo_name) self._cleanup_repos.append(repo_name) return repo @@ -515,6 +517,9 @@ class Backend(object): def __getitem__(self, key): return self._test_repo_container(key, self.alias) + def create_test_repo(self, key, config=None): + return self._test_repo_container(key, self.alias, config) + @property def repo(self): """ diff --git a/rhodecode/tests/vcs/test_git.py b/rhodecode/tests/vcs/test_git.py --- a/rhodecode/tests/vcs/test_git.py +++ b/rhodecode/tests/vcs/test_git.py @@ -22,15 +22,16 @@ import datetime import mock import os import sys +import shutil import pytest +from rhodecode.lib.utils import make_db_config from rhodecode.lib.vcs.backends.base import Reference from rhodecode.lib.vcs.backends.git import ( GitRepository, GitCommit, discover_git_version) from rhodecode.lib.vcs.exceptions import ( - RepositoryError, VCSError, NodeDoesNotExistError -) + RepositoryError, VCSError, NodeDoesNotExistError) from rhodecode.lib.vcs.nodes import ( NodeKind, FileNode, DirNode, NodeState, SubModuleNode) from rhodecode.tests import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, get_new_dir @@ -1003,6 +1004,32 @@ class TestGitCommit(object): assert author == commit.author_name +class TestLargeFileRepo(object): + + def test_large_file(self, backend_git): + conf = make_db_config() + repo = backend_git.create_test_repo('largefiles', conf) + + tip = repo.scm_instance().get_commit() + + # extract stored LF node into the origin cache + lfs_store = os.path.join(repo.repo_path, repo.repo_name, 'lfs_store') + + oid = '7b331c02e313c7599d5a90212e17e6d3cb729bd2e1c9b873c302a63c95a2f9bf' + oid_path = os.path.join(lfs_store, oid) + oid_destination = os.path.join( + conf.get('vcs_git_lfs', 'store_location'), oid) + shutil.copy(oid_path, oid_destination) + + node = tip.get_node('1MB.zip') + + lf_node = node.get_largefile_node() + + assert lf_node.is_largefile() is True + assert lf_node.size == 1024000 + assert lf_node.name == '1MB.zip' + + class TestGitSpecificWithRepo(BackendTestMixin): @classmethod diff --git a/rhodecode/tests/vcs/test_hg.py b/rhodecode/tests/vcs/test_hg.py --- a/rhodecode/tests/vcs/test_hg.py +++ b/rhodecode/tests/vcs/test_hg.py @@ -23,14 +23,13 @@ import os import mock import pytest -import rhodecode.lib.vcs.conf.settings +from rhodecode.lib.utils import make_db_config from rhodecode.lib.vcs import backends from rhodecode.lib.vcs.backends.base import ( Reference, MergeResponse, MergeFailureReason) from rhodecode.lib.vcs.backends.hg import MercurialRepository, MercurialCommit from rhodecode.lib.vcs.exceptions import ( - CommitError, RepositoryError, VCSError, NodeDoesNotExistError, - CommitDoesNotExistError) + RepositoryError, VCSError, NodeDoesNotExistError, CommitDoesNotExistError) from rhodecode.lib.vcs.nodes import FileNode, NodeKind, NodeState from rhodecode.tests import TEST_HG_REPO, TEST_HG_REPO_CLONE @@ -1095,12 +1094,6 @@ class TestMercurialCommit(object): with pytest.raises(VCSError): self.repo.get_commit().get_node(path) - def test_large_file(self): - # TODO: valid large file - tip = self.repo.get_commit() - with pytest.raises(CommitError): - tip.get_largefile_node("invalid") - def test_author_email(self): assert 'marcin@python-blog.com' == \ self.repo.get_commit('b986218ba1c9').author_email @@ -1117,6 +1110,21 @@ class TestMercurialCommit(object): self.repo.get_commit('84478366594b').author_name +class TestLargeFileRepo(object): + + def test_large_file(self, backend_hg): + repo = backend_hg.create_test_repo('largefiles', make_db_config()) + + tip = repo.scm_instance().get_commit() + node = tip.get_node('.hglf/thisfileislarge') + + lf_node = node.get_largefile_node() + + assert lf_node.is_largefile() is True + assert lf_node.size == 1024000 + assert lf_node.name == '.hglf/thisfileislarge' + + class TestGetBranchName(object): def test_returns_ref_name_when_type_is_branch(self): ref = self._create_ref('branch', 'fake-name')