##// END OF EJS Templates
largefiles: enabled download of largefiles for git and mercurial from web interface....
marcink -
r1577:3fd4ff52 default
parent child Browse files
Show More
@@ -226,8 +226,8 b' let'
226 rhodecode-testdata-src = sources.rhodecode-testdata or (
226 rhodecode-testdata-src = sources.rhodecode-testdata or (
227 pkgs.fetchhg {
227 pkgs.fetchhg {
228 url = "https://code.rhodecode.com/upstream/rc_testdata";
228 url = "https://code.rhodecode.com/upstream/rc_testdata";
229 rev = "v0.9.0";
229 rev = "v0.10.0";
230 sha256 = "0k0ccb7cncd6mmzwckfbr6l7fsymcympwcm948qc3i0f0m6bbg1y";
230 sha256 = "0zn9swwvx4vgw4qn8q3ri26vvzgrxn15x6xnjrysi1bwmz01qjl0";
231 });
231 });
232
232
233 # Apply all overrides and fix the final package set
233 # Apply all overrides and fix the final package set
@@ -223,6 +223,8 b' class FilesController(BaseRepoController'
223 c.file_author = True
223 c.file_author = True
224 c.file_tree = ''
224 c.file_tree = ''
225 if c.file.is_file():
225 if c.file.is_file():
226 c.lf_node = c.file.get_largefile_node()
227
226 c.file_source_page = 'true'
228 c.file_source_page = 'true'
227 c.file_last_commit = c.file.last_commit
229 c.file_last_commit = c.file.last_commit
228 if c.file.size < self.cut_off_limit_file:
230 if c.file.size < self.cut_off_limit_file:
@@ -343,6 +345,14 b' class FilesController(BaseRepoController'
343 commit = self.__get_commit_or_redirect(revision, repo_name)
345 commit = self.__get_commit_or_redirect(revision, repo_name)
344 file_node = self.__get_filenode_or_redirect(repo_name, commit, f_path)
346 file_node = self.__get_filenode_or_redirect(repo_name, commit, f_path)
345
347
348 if request.GET.get('lf'):
349 # only if lf get flag is passed, we download this file
350 # as LFS/Largefile
351 lf_node = file_node.get_largefile_node()
352 if lf_node:
353 # overwrite our pointer with the REAL large-file
354 file_node = lf_node
355
346 response.content_disposition = 'attachment; filename=%s' % \
356 response.content_disposition = 'attachment; filename=%s' % \
347 safe_str(f_path.split(Repository.NAME_SEP)[-1])
357 safe_str(f_path.split(Repository.NAME_SEP)[-1])
348
358
@@ -893,9 +893,10 b' class BaseCommit(object):'
893
893
894 def get_largefile_node(self, path):
894 def get_largefile_node(self, path):
895 """
895 """
896 Returns the path to largefile from Mercurial storage.
896 Returns the path to largefile from Mercurial/Git-lfs storage.
897 or None if it's not a largefile node
897 """
898 """
898 raise NotImplementedError
899 return None
899
900
900 def archive_repo(self, file_path, kind='tgz', subrepos=None,
901 def archive_repo(self, file_path, kind='tgz', subrepos=None,
901 prefix=None, write_metadata=False, mtime=None):
902 prefix=None, write_metadata=False, mtime=None):
@@ -39,7 +39,7 b' from rhodecode.lib.vcs.exceptions import'
39 from rhodecode.lib.vcs.nodes import (
39 from rhodecode.lib.vcs.nodes import (
40 FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
40 FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
41 ChangedFileNodesGenerator, AddedFileNodesGenerator,
41 ChangedFileNodesGenerator, AddedFileNodesGenerator,
42 RemovedFileNodesGenerator)
42 RemovedFileNodesGenerator, LargeFileNode)
43
43
44
44
45 class GitCommit(base.BaseCommit):
45 class GitCommit(base.BaseCommit):
@@ -423,6 +423,17 b' class GitCommit(base.BaseCommit):'
423 self.nodes[path] = node
423 self.nodes[path] = node
424 return self.nodes[path]
424 return self.nodes[path]
425
425
426 def get_largefile_node(self, path):
427 id_, _ = self._get_id_for_path(path)
428 pointer_spec = self._remote.is_large_file(id_)
429
430 if pointer_spec:
431 # content of that file regular FileNode is the hash of largefile
432 file_id = pointer_spec.get('oid_hash')
433 if self._remote.in_largefiles_store(file_id):
434 lf_path = self._remote.store_path(file_id)
435 return LargeFileNode(lf_path, commit=self, org_path=path)
436
426 @LazyProperty
437 @LazyProperty
427 def affected_files(self):
438 def affected_files(self):
428 """
439 """
@@ -312,18 +312,18 b' class MercurialCommit(base.BaseCommit):'
312 return self.nodes[path]
312 return self.nodes[path]
313
313
314 def get_largefile_node(self, path):
314 def get_largefile_node(self, path):
315 path = os.path.join(LARGEFILE_PREFIX, path)
316
315
317 if self._remote.is_large_file(path):
316 if self._remote.is_large_file(path):
318 # content of that file regular FileNode is the hash of largefile
317 # content of that file regular FileNode is the hash of largefile
319 file_id = self.get_file_content(path).strip()
318 file_id = self.get_file_content(path).strip()
320 if self._remote.in_store(file_id):
319
321 path = self._remote.store_path(file_id)
320 if self._remote.in_largefiles_store(file_id):
322 return LargeFileNode(path, commit=self)
321 lf_path = self._remote.store_path(file_id)
322 return LargeFileNode(lf_path, commit=self, org_path=path)
323 elif self._remote.in_user_cache(file_id):
323 elif self._remote.in_user_cache(file_id):
324 path = self._remote.store_path(file_id)
324 lf_path = self._remote.store_path(file_id)
325 self._remote.link(file_id, path)
325 self._remote.link(file_id, path)
326 return LargeFileNode(path, commit=self)
326 return LargeFileNode(lf_path, commit=self, org_path=path)
327
327
328 @LazyProperty
328 @LazyProperty
329 def _submodules(self):
329 def _submodules(self):
@@ -22,7 +22,7 b''
22 Module holding everything related to vcs nodes, with vcs2 architecture.
22 Module holding everything related to vcs nodes, with vcs2 architecture.
23 """
23 """
24
24
25
25 import os
26 import stat
26 import stat
27
27
28 from zope.cachedescriptors.property import Lazy as LazyProperty
28 from zope.cachedescriptors.property import Lazy as LazyProperty
@@ -547,9 +547,8 b' class FileNode(Node):'
547 create special instance of LargeFileNode which can get content from
547 create special instance of LargeFileNode which can get content from
548 LF store.
548 LF store.
549 """
549 """
550 if self.commit and self.path.startswith(LARGEFILE_PREFIX):
550 if self.commit:
551 largefile_path = self.path.split(LARGEFILE_PREFIX)[-1].lstrip('/')
551 return self.commit.get_largefile_node(self.path)
552 return self.commit.get_largefile_node(largefile_path)
553
552
554 def lines(self, count_empty=False):
553 def lines(self, count_empty=False):
555 all_lines, empty_lines = 0, 0
554 all_lines, empty_lines = 0, 0
@@ -765,15 +764,37 b' class SubModuleNode(Node):'
765
764
766 class LargeFileNode(FileNode):
765 class LargeFileNode(FileNode):
767
766
767 def __init__(self, path, url=None, commit=None, alias=None, org_path=None):
768 self.path = path
769 self.org_path = org_path
770 self.kind = NodeKind.LARGEFILE
771 self.alias = alias
772
768 def _validate_path(self, path):
773 def _validate_path(self, path):
769 """
774 """
770 we override check since the LargeFileNode path is system absolute
775 we override check since the LargeFileNode path is system absolute
771 """
776 """
777 pass
772
778
779 def __repr__(self):
780 return '<%s %r>' % (self.__class__.__name__, self.path)
781
782 @LazyProperty
783 def size(self):
784 return os.stat(self.path).st_size
785
786 @LazyProperty
773 def raw_bytes(self):
787 def raw_bytes(self):
774 if self.commit:
788 if self.commit:
775 with open(self.path, 'rb') as f:
789 with open(self.path, 'rb') as f:
776 content = f.read()
790 content = f.read()
777 else:
791 else:
778 content = self._content
792 content = self._content
779 return content No newline at end of file
793 return content
794
795 @LazyProperty
796 def name(self):
797 """
798 Overwrites name to be the org lf path
799 """
800 return self.org_path
@@ -4,6 +4,9 b''
4 <div class="codeblock-header">
4 <div class="codeblock-header">
5 <div class="stats">
5 <div class="stats">
6 <span> <strong>${c.file}</strong></span>
6 <span> <strong>${c.file}</strong></span>
7 % if c.lf_node:
8 <span title="${_('This file is a pointer to large binary file')}"> | ${_('LargeFile')} ${h.format_byte_size_binary(c.lf_node.size)} </span>
9 % endif
7 <span> | ${c.file.lines()[0]} ${ungettext('line', 'lines', c.file.lines()[0])}</span>
10 <span> | ${c.file.lines()[0]} ${ungettext('line', 'lines', c.file.lines()[0])}</span>
8 <span> | ${h.format_byte_size_binary(c.file.size)}</span>
11 <span> | ${h.format_byte_size_binary(c.file.size)}</span>
9 <span> | ${c.file.mimetype} </span>
12 <span> | ${c.file.mimetype} </span>
@@ -22,9 +25,16 b''
22 ${h.link_to(_('Annotation'), h.url('files_annotate_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
25 ${h.link_to(_('Annotation'), h.url('files_annotate_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
23 %endif
26 %endif
24 | ${h.link_to(_('Raw'), h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
27 | ${h.link_to(_('Raw'), h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
25 | <a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path)}">
28 |
26 ${_('Download')}
29 % if c.lf_node:
27 </a>
30 <a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path, lf=1)}">
31 ${_('Download largefile')}
32 </a>
33 % else:
34 <a href="${h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path)}">
35 ${_('Download')}
36 </a>
37 % endif
28
38
29 %if h.HasRepoPermissionAny('repository.write','repository.admin')(c.repo_name):
39 %if h.HasRepoPermissionAny('repository.write','repository.admin')(c.repo_name):
30 |
40 |
@@ -46,11 +56,11 b''
46 </div>
56 </div>
47 <div id="file_history_container"></div>
57 <div id="file_history_container"></div>
48 <div class="code-body">
58 <div class="code-body">
49 %if c.file.is_binary:
59 %if c.file.is_binary:
50 <div>
60 <div>
51 ${_('Binary file (%s)') % c.file.mimetype}
61 ${_('Binary file (%s)') % c.file.mimetype}
52 </div>
62 </div>
53 %else:
63 %else:
54 % if c.file.size < c.cut_off_limit:
64 % if c.file.size < c.cut_off_limit:
55 %if c.renderer and not c.annotate:
65 %if c.renderer and not c.annotate:
56 ${h.render(c.file.content, renderer=c.renderer, relative_url=h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
66 ${h.render(c.file.content, renderer=c.renderer, relative_url=h.url('files_raw_home',repo_name=c.repo_name,revision=c.commit.raw_id,f_path=c.f_path))}
@@ -67,7 +77,6 b''
67 %endfor
77 %endfor
68 %endif
78 %endif
69 </table>
79 </table>
70 </div>
71 %endif
80 %endif
72 %else:
81 %else:
73 ${_('File is too big to display')} ${h.link_to(_('Show as raw'),
82 ${_('File is too big to display')} ${h.link_to(_('Show as raw'),
@@ -387,20 +387,22 b' class TestRepoContainer(object):'
387 self._fixture = Fixture()
387 self._fixture = Fixture()
388 self._repos = {}
388 self._repos = {}
389
389
390 def __call__(self, dump_name, backend_alias):
390 def __call__(self, dump_name, backend_alias, config=None):
391 key = (dump_name, backend_alias)
391 key = (dump_name, backend_alias)
392 if key not in self._repos:
392 if key not in self._repos:
393 repo = self._create_repo(dump_name, backend_alias)
393 repo = self._create_repo(dump_name, backend_alias, config)
394 self._repos[key] = repo.repo_id
394 self._repos[key] = repo.repo_id
395 return Repository.get(self._repos[key])
395 return Repository.get(self._repos[key])
396
396
397 def _create_repo(self, dump_name, backend_alias):
397 def _create_repo(self, dump_name, backend_alias, config):
398 repo_name = '%s-%s' % (backend_alias, dump_name)
398 repo_name = '%s-%s' % (backend_alias, dump_name)
399 backend_class = get_backend(backend_alias)
399 backend_class = get_backend(backend_alias)
400 dump_extractor = self.dump_extractors[backend_alias]
400 dump_extractor = self.dump_extractors[backend_alias]
401 repo_path = dump_extractor(dump_name, repo_name)
401 repo_path = dump_extractor(dump_name, repo_name)
402 vcs_repo = backend_class(repo_path)
402
403 vcs_repo = backend_class(repo_path, config=config)
403 repo2db_mapper({repo_name: vcs_repo})
404 repo2db_mapper({repo_name: vcs_repo})
405
404 repo = RepoModel().get_by_repo_name(repo_name)
406 repo = RepoModel().get_by_repo_name(repo_name)
405 self._cleanup_repos.append(repo_name)
407 self._cleanup_repos.append(repo_name)
406 return repo
408 return repo
@@ -515,6 +517,9 b' class Backend(object):'
515 def __getitem__(self, key):
517 def __getitem__(self, key):
516 return self._test_repo_container(key, self.alias)
518 return self._test_repo_container(key, self.alias)
517
519
520 def create_test_repo(self, key, config=None):
521 return self._test_repo_container(key, self.alias, config)
522
518 @property
523 @property
519 def repo(self):
524 def repo(self):
520 """
525 """
@@ -22,15 +22,16 b' import datetime'
22 import mock
22 import mock
23 import os
23 import os
24 import sys
24 import sys
25 import shutil
25
26
26 import pytest
27 import pytest
27
28
29 from rhodecode.lib.utils import make_db_config
28 from rhodecode.lib.vcs.backends.base import Reference
30 from rhodecode.lib.vcs.backends.base import Reference
29 from rhodecode.lib.vcs.backends.git import (
31 from rhodecode.lib.vcs.backends.git import (
30 GitRepository, GitCommit, discover_git_version)
32 GitRepository, GitCommit, discover_git_version)
31 from rhodecode.lib.vcs.exceptions import (
33 from rhodecode.lib.vcs.exceptions import (
32 RepositoryError, VCSError, NodeDoesNotExistError
34 RepositoryError, VCSError, NodeDoesNotExistError)
33 )
34 from rhodecode.lib.vcs.nodes import (
35 from rhodecode.lib.vcs.nodes import (
35 NodeKind, FileNode, DirNode, NodeState, SubModuleNode)
36 NodeKind, FileNode, DirNode, NodeState, SubModuleNode)
36 from rhodecode.tests import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, get_new_dir
37 from rhodecode.tests import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, get_new_dir
@@ -1003,6 +1004,32 b' class TestGitCommit(object):'
1003 assert author == commit.author_name
1004 assert author == commit.author_name
1004
1005
1005
1006
1007 class TestLargeFileRepo(object):
1008
1009 def test_large_file(self, backend_git):
1010 conf = make_db_config()
1011 repo = backend_git.create_test_repo('largefiles', conf)
1012
1013 tip = repo.scm_instance().get_commit()
1014
1015 # extract stored LF node into the origin cache
1016 lfs_store = os.path.join(repo.repo_path, repo.repo_name, 'lfs_store')
1017
1018 oid = '7b331c02e313c7599d5a90212e17e6d3cb729bd2e1c9b873c302a63c95a2f9bf'
1019 oid_path = os.path.join(lfs_store, oid)
1020 oid_destination = os.path.join(
1021 conf.get('vcs_git_lfs', 'store_location'), oid)
1022 shutil.copy(oid_path, oid_destination)
1023
1024 node = tip.get_node('1MB.zip')
1025
1026 lf_node = node.get_largefile_node()
1027
1028 assert lf_node.is_largefile() is True
1029 assert lf_node.size == 1024000
1030 assert lf_node.name == '1MB.zip'
1031
1032
1006 class TestGitSpecificWithRepo(BackendTestMixin):
1033 class TestGitSpecificWithRepo(BackendTestMixin):
1007
1034
1008 @classmethod
1035 @classmethod
@@ -23,14 +23,13 b' import os'
23 import mock
23 import mock
24 import pytest
24 import pytest
25
25
26 import rhodecode.lib.vcs.conf.settings
26 from rhodecode.lib.utils import make_db_config
27 from rhodecode.lib.vcs import backends
27 from rhodecode.lib.vcs import backends
28 from rhodecode.lib.vcs.backends.base import (
28 from rhodecode.lib.vcs.backends.base import (
29 Reference, MergeResponse, MergeFailureReason)
29 Reference, MergeResponse, MergeFailureReason)
30 from rhodecode.lib.vcs.backends.hg import MercurialRepository, MercurialCommit
30 from rhodecode.lib.vcs.backends.hg import MercurialRepository, MercurialCommit
31 from rhodecode.lib.vcs.exceptions import (
31 from rhodecode.lib.vcs.exceptions import (
32 CommitError, RepositoryError, VCSError, NodeDoesNotExistError,
32 RepositoryError, VCSError, NodeDoesNotExistError, CommitDoesNotExistError)
33 CommitDoesNotExistError)
34 from rhodecode.lib.vcs.nodes import FileNode, NodeKind, NodeState
33 from rhodecode.lib.vcs.nodes import FileNode, NodeKind, NodeState
35 from rhodecode.tests import TEST_HG_REPO, TEST_HG_REPO_CLONE
34 from rhodecode.tests import TEST_HG_REPO, TEST_HG_REPO_CLONE
36
35
@@ -1095,12 +1094,6 b' class TestMercurialCommit(object):'
1095 with pytest.raises(VCSError):
1094 with pytest.raises(VCSError):
1096 self.repo.get_commit().get_node(path)
1095 self.repo.get_commit().get_node(path)
1097
1096
1098 def test_large_file(self):
1099 # TODO: valid large file
1100 tip = self.repo.get_commit()
1101 with pytest.raises(CommitError):
1102 tip.get_largefile_node("invalid")
1103
1104 def test_author_email(self):
1097 def test_author_email(self):
1105 assert 'marcin@python-blog.com' == \
1098 assert 'marcin@python-blog.com' == \
1106 self.repo.get_commit('b986218ba1c9').author_email
1099 self.repo.get_commit('b986218ba1c9').author_email
@@ -1117,6 +1110,21 b' class TestMercurialCommit(object):'
1117 self.repo.get_commit('84478366594b').author_name
1110 self.repo.get_commit('84478366594b').author_name
1118
1111
1119
1112
1113 class TestLargeFileRepo(object):
1114
1115 def test_large_file(self, backend_hg):
1116 repo = backend_hg.create_test_repo('largefiles', make_db_config())
1117
1118 tip = repo.scm_instance().get_commit()
1119 node = tip.get_node('.hglf/thisfileislarge')
1120
1121 lf_node = node.get_largefile_node()
1122
1123 assert lf_node.is_largefile() is True
1124 assert lf_node.size == 1024000
1125 assert lf_node.name == '.hglf/thisfileislarge'
1126
1127
1120 class TestGetBranchName(object):
1128 class TestGetBranchName(object):
1121 def test_returns_ref_name_when_type_is_branch(self):
1129 def test_returns_ref_name_when_type_is_branch(self):
1122 ref = self._create_ref('branch', 'fake-name')
1130 ref = self._create_ref('branch', 'fake-name')
General Comments 0
You need to be logged in to leave comments. Login now