##// END OF EJS Templates
largefiles: enabled download of largefiles for git and mercurial from web interface....
marcink -
r1577:3fd4ff52 default
parent child Browse files
Show More
@@ -226,8 +226,8 b' let'
226 226 rhodecode-testdata-src = sources.rhodecode-testdata or (
227 227 pkgs.fetchhg {
228 228 url = "https://code.rhodecode.com/upstream/rc_testdata";
229 rev = "v0.9.0";
230 sha256 = "0k0ccb7cncd6mmzwckfbr6l7fsymcympwcm948qc3i0f0m6bbg1y";
229 rev = "v0.10.0";
230 sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0";
231 231 });
232 232
233 233 # Apply all overrides and fix the final package set
@@ -223,6 +223,8 b' class FilesController(BaseRepoController'
223 223 c.file_author = True
224 224 c.file_tree = ''
225 225 if c.file.is_file():
226 c.lf_node = c.file.get_largefile_node()
227
226 228 c.file_source_page = 'true'
227 229 c.file_last_commit = c.file.last_commit
228 230 if c.file.size < self.cut_off_limit_file:
@@ -343,6 +345,14 b' class FilesController(BaseRepoController'
343 345 commit = self.__get_commit_or_redirect(revision, repo_name)
344 346 file_node = self.__get_filenode_or_redirect(repo_name, commit, f_path)
345 347
348 if request.GET.get('lf'):
349 # only if lf get flag is passed, we download this file
350 # as LFS/Largefile
351 lf_node = file_node.get_largefile_node()
352 if lf_node:
353 # overwrite our pointer with the REAL large-file
354 file_node = lf_node
355
346 356 response.content_disposition = 'attachment; filename=%s' % \
347 357 safe_str(f_path.split(Repository.NAME_SEP)[-1])
348 358
@@ -893,9 +893,10 b' class BaseCommit(object):'
893 893
894 894 def get_largefile_node(self, path):
895 895 """
896 Returns the path to largefile from Mercurial storage.
896 Returns the path to largefile from Mercurial/Git-lfs storage.
897 or None if it's not a largefile node
897 898 """
898 raise NotImplementedError
899 return None
899 900
900 901 def archive_repo(self, file_path, kind='tgz', subrepos=None,
901 902 prefix=None, write_metadata=False, mtime=None):
@@ -39,7 +39,7 b' from rhodecode.lib.vcs.exceptions import'
39 39 from rhodecode.lib.vcs.nodes import (
40 40 FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
41 41 ChangedFileNodesGenerator, AddedFileNodesGenerator,
42 RemovedFileNodesGenerator)
42 RemovedFileNodesGenerator, LargeFileNode)
43 43
44 44
45 45 class GitCommit(base.BaseCommit):
@@ -423,6 +423,17 b' class GitCommit(base.BaseCommit):'
423 423 self.nodes[path] = node
424 424 return self.nodes[path]
425 425
426 def get_largefile_node(self, path):
427 id_, _ = self._get_id_for_path(path)
428 pointer_spec = self._remote.is_large_file(id_)
429
430 if pointer_spec:
431 # content of that file regular FileNode is the hash of largefile
432 file_id = pointer_spec.get('oid_hash')
433 if self._remote.in_largefiles_store(file_id):
434 lf_path = self._remote.store_path(file_id)
435 return LargeFileNode(lf_path, commit=self, org_path=path)
436
426 437 @LazyProperty
427 438 def affected_files(self):
428 439 """
@@ -312,18 +312,18 b' class MercurialCommit(base.BaseCommit):'
312 312 return self.nodes[path]
313 313
314 314 def get_largefile_node(self, path):
315 path = os.path.join(LARGEFILE_PREFIX, path)
316 315
317 316 if self._remote.is_large_file(path):
318 317 # content of that file regular FileNode is the hash of largefile
319 318 file_id = self.get_file_content(path).strip()
320 if self._remote.in_store(file_id):
321 path = self._remote.store_path(file_id)
322 return LargeFileNode(path, commit=self)
319
320 if self._remote.in_largefiles_store(file_id):
321 lf_path = self._remote.store_path(file_id)
322 return LargeFileNode(lf_path, commit=self, org_path=path)
323 323 elif self._remote.in_user_cache(file_id):
324 path = self._remote.store_path(file_id)
324 lf_path = self._remote.store_path(file_id)
325 325 self._remote.link(file_id, path)
326 return LargeFileNode(path, commit=self)
326 return LargeFileNode(lf_path, commit=self, org_path=path)
327 327
328 328 @LazyProperty
329 329 def _submodules(self):
@@ -22,7 +22,7 b''
22 22 Module holding everything related to vcs nodes, with vcs2 architecture.
23 23 """
24 24
25
25 import os
26 26 import stat
27 27
28 28 from zope.cachedescriptors.property import Lazy as LazyProperty
@@ -547,9 +547,8 b' class FileNode(Node):'
547 547 create special instance of LargeFileNode which can get content from
548 548 LF store.
549 549 """
550 if self.commit and self.path.startswith(LARGEFILE_PREFIX):
551 largefile_path = self.path.split(LARGEFILE_PREFIX)[-1].lstrip('/')
552 return self.commit.get_largefile_node(largefile_path)
550 if self.commit:
551 return self.commit.get_largefile_node(self.path)
553 552
554 553 def lines(self, count_empty=False):
555 554 all_lines, empty_lines = 0, 0
@@ -765,15 +764,37 b' class SubModuleNode(Node):'
765 764
766 765 class LargeFileNode(FileNode):
767 766
767 def __init__(self, path, url=None, commit=None, alias=None, org_path=None):
768 self.path = path
769 self.org_path = org_path
770 self.kind = NodeKind.LARGEFILE
771 self.alias = alias
772
768 773 def _validate_path(self, path):
769 774 """
770 775 we override check since the LargeFileNode path is system absolute
771 776 """
777 pass
772 778
779 def __repr__(self):
780 return '<%s %r>' % (self.__class__.__name__, self.path)
781
782 @LazyProperty
783 def size(self):
784 return os.stat(self.path).st_size
785
786 @LazyProperty
773 787 def raw_bytes(self):
774 788 if self.commit:
775 789 with open(self.path, 'rb') as f:
776 790 content = f.read()
777 791 else:
778 792 content = self._content
779 return content No newline at end of file
793 return content
794
795 @LazyProperty
796 def name(self):
797 """
798 Overwrites name to be the org lf path
799 """
800 return self.org_path
@@ -4,6 +4,9 b''
4 4 <div class="codeblock-header">
5 5 <div class="stats">
6 6 <span> <strong>${c.file}</strong></span>
7 % if c.lf_node:
8 <span title="${_('This file is a pointer to large binary file')}"> | ${_('LargeFile')} ${h.format_byte_size_binary(c.lf_node.size)} </span>
9 % endif
7 10 <span> | ${c.file.lines()[0]} ${ungettext('line', 'lines', c.file.lines()[0])}</span>
8 11 <span> | ${h.format_byte_size_binary(c.file.size)}</span>
9 12 <span> | ${c.file.mimetype} </span>
@@ -22,9 +25,16 b''
22 25 ${h.link_to(_('Annotation'), h.url('files_annotate_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
23 26 %endif
24 27 | ${h.link_to(_('Raw'), h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
25 | <a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path)}">
28 |
29 % if c.lf_node:
30 <a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path, lf=1)}">
31 ${_('Download largefile')}
32 </a>
33 % else:
34 <a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path)}">
26 35 ${_('Download')}
27 36 </a>
37 % endif
28 38
29 39 %if h.HasRepoPermissionAny('repository.write','repository.admin')(c.repo_name):
30 40 |
@@ -67,7 +77,6 b''
67 77 %endfor
68 78 %endif
69 79 </table>
70 </div>
71 80 %endif
72 81 %else:
73 82 ${_('File is too big to display')} ${h.link_to(_('Show as raw'),
@@ -387,20 +387,22 b' class TestRepoContainer(object):'
387 387 self._fixture = Fixture()
388 388 self._repos = {}
389 389
390 def __call__(self, dump_name, backend_alias):
390 def __call__(self, dump_name, backend_alias, config=None):
391 391 key = (dump_name, backend_alias)
392 392 if key not in self._repos:
393 repo = self._create_repo(dump_name, backend_alias)
393 repo = self._create_repo(dump_name, backend_alias, config)
394 394 self._repos[key] = repo.repo_id
395 395 return Repository.get(self._repos[key])
396 396
397 def _create_repo(self, dump_name, backend_alias):
397 def _create_repo(self, dump_name, backend_alias, config):
398 398 repo_name = '%s-%s' % (backend_alias, dump_name)
399 399 backend_class = get_backend(backend_alias)
400 400 dump_extractor = self.dump_extractors[backend_alias]
401 401 repo_path = dump_extractor(dump_name, repo_name)
402 vcs_repo = backend_class(repo_path)
402
403 vcs_repo = backend_class(repo_path, config=config)
403 404 repo2db_mapper({repo_name: vcs_repo})
405
404 406 repo = RepoModel().get_by_repo_name(repo_name)
405 407 self._cleanup_repos.append(repo_name)
406 408 return repo
@@ -515,6 +517,9 b' class Backend(object):'
515 517 def __getitem__(self, key):
516 518 return self._test_repo_container(key, self.alias)
517 519
520 def create_test_repo(self, key, config=None):
521 return self._test_repo_container(key, self.alias, config)
522
518 523 @property
519 524 def repo(self):
520 525 """
@@ -22,15 +22,16 b' import datetime'
22 22 import mock
23 23 import os
24 24 import sys
25 import shutil
25 26
26 27 import pytest
27 28
29 from rhodecode.lib.utils import make_db_config
28 30 from rhodecode.lib.vcs.backends.base import Reference
29 31 from rhodecode.lib.vcs.backends.git import (
30 32 GitRepository, GitCommit, discover_git_version)
31 33 from rhodecode.lib.vcs.exceptions import (
32 RepositoryError, VCSError, NodeDoesNotExistError
33 )
34 RepositoryError, VCSError, NodeDoesNotExistError)
34 35 from rhodecode.lib.vcs.nodes import (
35 36 NodeKind, FileNode, DirNode, NodeState, SubModuleNode)
36 37 from rhodecode.tests import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, get_new_dir
@@ -1003,6 +1004,32 b' class TestGitCommit(object):'
1003 1004 assert author == commit.author_name
1004 1005
1005 1006
1007 class TestLargeFileRepo(object):
1008
1009 def test_large_file(self, backend_git):
1010 conf = make_db_config()
1011 repo = backend_git.create_test_repo('largefiles', conf)
1012
1013 tip = repo.scm_instance().get_commit()
1014
1015 # extract stored LF node into the origin cache
1016 lfs_store = os.path.join(repo.repo_path, repo.repo_name, 'lfs_store')
1017
1018 oid = '7b331c02e313c7599d5a90212e17e6d3cb729bd2e1c9b873c302a63c95a2f9bf'
1019 oid_path = os.path.join(lfs_store, oid)
1020 oid_destination = os.path.join(
1021 conf.get('vcs_git_lfs', 'store_location'), oid)
1022 shutil.copy(oid_path, oid_destination)
1023
1024 node = tip.get_node('1MB.zip')
1025
1026 lf_node = node.get_largefile_node()
1027
1028 assert lf_node.is_largefile() is True
1029 assert lf_node.size == 1024000
1030 assert lf_node.name == '1MB.zip'
1031
1032
1006 1033 class TestGitSpecificWithRepo(BackendTestMixin):
1007 1034
1008 1035 @classmethod
@@ -23,14 +23,13 b' import os'
23 23 import mock
24 24 import pytest
25 25
26 import rhodecode.lib.vcs.conf.settings
26 from rhodecode.lib.utils import make_db_config
27 27 from rhodecode.lib.vcs import backends
28 28 from rhodecode.lib.vcs.backends.base import (
29 29 Reference, MergeResponse, MergeFailureReason)
30 30 from rhodecode.lib.vcs.backends.hg import MercurialRepository, MercurialCommit
31 31 from rhodecode.lib.vcs.exceptions import (
32 CommitError, RepositoryError, VCSError, NodeDoesNotExistError,
33 CommitDoesNotExistError)
32 RepositoryError, VCSError, NodeDoesNotExistError, CommitDoesNotExistError)
34 33 from rhodecode.lib.vcs.nodes import FileNode, NodeKind, NodeState
35 34 from rhodecode.tests import TEST_HG_REPO, TEST_HG_REPO_CLONE
36 35
@@ -1095,12 +1094,6 b' class TestMercurialCommit(object):'
1095 1094 with pytest.raises(VCSError):
1096 1095 self.repo.get_commit().get_node(path)
1097 1096
1098 def test_large_file(self):
1099 # TODO: valid large file
1100 tip = self.repo.get_commit()
1101 with pytest.raises(CommitError):
1102 tip.get_largefile_node("invalid")
1103
1104 1097 def test_author_email(self):
1105 1098 assert 'marcin@python-blog.com' == \
1106 1099 self.repo.get_commit('b986218ba1c9').author_email
@@ -1117,6 +1110,21 b' class TestMercurialCommit(object):'
1117 1110 self.repo.get_commit('84478366594b').author_name
1118 1111
1119 1112
1113 class TestLargeFileRepo(object):
1114
1115 def test_large_file(self, backend_hg):
1116 repo = backend_hg.create_test_repo('largefiles', make_db_config())
1117
1118 tip = repo.scm_instance().get_commit()
1119 node = tip.get_node('.hglf/thisfileislarge')
1120
1121 lf_node = node.get_largefile_node()
1122
1123 assert lf_node.is_largefile() is True
1124 assert lf_node.size == 1024000
1125 assert lf_node.name == '.hglf/thisfileislarge'
1126
1127
1120 1128 class TestGetBranchName(object):
1121 1129 def test_returns_ref_name_when_type_is_branch(self):
1122 1130 ref = self._create_ref('branch', 'fake-name')
General Comments 0
You need to be logged in to leave comments. Login now