files: ported repository files controllers to pyramid views.
marcink -
r1927:e6df2b71 default
Not Reviewed
Show More
Add another comment
TODOs: 0 unresolved 0 Resolved
COMMENTS: 0 General 0 Inline
This diff has been collapsed as it changes many lines, (1278 lines changed) Show them Hide them
@@ -0,0 +1,1278
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2011-2017 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 import itertools
22 import logging
23 import os
24 import shutil
25 import tempfile
26 import collections
27
28 from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest, HTTPFound
29 from pyramid.view import view_config
30 from pyramid.renderers import render
31 from pyramid.response import Response
32
33 from rhodecode.apps._base import RepoAppView
34
35 from rhodecode.controllers.utils import parse_path_ref
36 from rhodecode.lib import diffs, helpers as h, caches
37 from rhodecode.lib import audit_logger
38 from rhodecode.lib.exceptions import NonRelativePathError
39 from rhodecode.lib.codeblocks import (
40 filenode_as_lines_tokens, filenode_as_annotated_lines_tokens)
41 from rhodecode.lib.utils2 import (
42 convert_line_endings, detect_mode, safe_str, str2bool)
43 from rhodecode.lib.auth import (
44 LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
45 from rhodecode.lib.vcs import path as vcspath
46 from rhodecode.lib.vcs.backends.base import EmptyCommit
47 from rhodecode.lib.vcs.conf import settings
48 from rhodecode.lib.vcs.nodes import FileNode
49 from rhodecode.lib.vcs.exceptions import (
50 RepositoryError, CommitDoesNotExistError, EmptyRepositoryError,
51 ImproperArchiveTypeError, VCSError, NodeAlreadyExistsError,
52 NodeDoesNotExistError, CommitError, NodeError)
53
54 from rhodecode.model.scm import ScmModel
55 from rhodecode.model.db import Repository
56
57 log = logging.getLogger(__name__)
58
59
60 class RepoFilesView(RepoAppView):
61
62 @staticmethod
63 def adjust_file_path_for_svn(f_path, repo):
64 """
65 Computes the relative path of `f_path`.
66
67 This is mainly based on prefix matching of the recognized tags and
68 branches in the underlying repository.
69 """
70 tags_and_branches = itertools.chain(
71 repo.branches.iterkeys(),
72 repo.tags.iterkeys())
73 tags_and_branches = sorted(tags_and_branches, key=len, reverse=True)
74
75 for name in tags_and_branches:
76 if f_path.startswith('{}/'.format(name)):
77 f_path = vcspath.relpath(f_path, name)
78 break
79 return f_path
80
81 def load_default_context(self):
82 c = self._get_local_tmpl_context(include_app_defaults=True)
83
84 # TODO(marcink): remove repo_info and use c.rhodecode_db_repo instead
85 c.repo_info = self.db_repo
86 c.rhodecode_repo = self.rhodecode_vcs_repo
87
88 self._register_global_c(c)
89 return c
90
91 def _ensure_not_locked(self):
92 _ = self.request.translate
93
94 repo = self.db_repo
95 if repo.enable_locking and repo.locked[0]:
96 h.flash(_('This repository has been locked by %s on %s')
97 % (h.person_by_id(repo.locked[0]),
98 h.format_date(h.time_to_datetime(repo.locked[1]))),
99 'warning')
100 files_url = h.route_path(
101 'repo_files:default_path',
102 repo_name=self.db_repo_name, commit_id='tip')
103 raise HTTPFound(files_url)
104
105 def _get_commit_and_path(self):
106 default_commit_id = self.db_repo.landing_rev[1]
107 default_f_path = '/'
108
109 commit_id = self.request.matchdict.get(
110 'commit_id', default_commit_id)
111 f_path = self.request.matchdict.get('f_path', default_f_path)
112 return commit_id, f_path
113
114 def _get_default_encoding(self, c):
115 enc_list = getattr(c, 'default_encodings', [])
116 return enc_list[0] if enc_list else 'UTF-8'
117
118 def _get_commit_or_redirect(self, commit_id, redirect_after=True):
119 """
120 This is a safe way to get commit. If an error occurs it redirects to
121 tip with proper message
122
123 :param commit_id: id of commit to fetch
124 :param redirect_after: toggle redirection
125 """
126 _ = self.request.translate
127
128 try:
129 return self.rhodecode_vcs_repo.get_commit(commit_id)
130 except EmptyRepositoryError:
131 if not redirect_after:
132 return None
133
134 _url = h.route_path(
135 'repo_files_add_file',
136 repo_name=self.db_repo_name, commit_id=0, f_path='',
137 _anchor='edit')
138
139 if h.HasRepoPermissionAny(
140 'repository.write', 'repository.admin')(self.db_repo_name):
141 add_new = h.link_to(
142 _('Click here to add a new file.'), _url, class_="alert-link")
143 else:
144 add_new = ""
145
146 h.flash(h.literal(
147 _('There are no files yet. %s') % add_new), category='warning')
148 raise HTTPFound(
149 h.route_path('repo_summary', repo_name=self.db_repo_name))
150
151 except (CommitDoesNotExistError, LookupError):
152 msg = _('No such commit exists for this repository')
153 h.flash(msg, category='error')
154 raise HTTPNotFound()
155 except RepositoryError as e:
156 h.flash(safe_str(h.escape(e)), category='error')
157 raise HTTPNotFound()
158
159 def _get_filenode_or_redirect(self, commit_obj, path):
160 """
161 Returns file_node, if error occurs or given path is directory,
162 it'll redirect to top level path
163 """
164 _ = self.request.translate
165
166 try:
167 file_node = commit_obj.get_node(path)
168 if file_node.is_dir():
169 raise RepositoryError('The given path is a directory')
170 except CommitDoesNotExistError:
171 log.exception('No such commit exists for this repository')
172 h.flash(_('No such commit exists for this repository'), category='error')
173 raise HTTPNotFound()
174 except RepositoryError as e:
175 log.warning('Repository error while fetching '
176 'filenode `%s`. Err:%s', path, e)
177 h.flash(safe_str(h.escape(e)), category='error')
178 raise HTTPNotFound()
179
180 return file_node
181
182 def _is_valid_head(self, commit_id, repo):
183 # check if commit is a branch identifier- basically we cannot
184 # create multiple heads via file editing
185 valid_heads = repo.branches.keys() + repo.branches.values()
186
187 if h.is_svn(repo) and not repo.is_empty():
188 # Note: Subversion only has one head, we add it here in case there
189 # is no branch matched.
190 valid_heads.append(repo.get_commit(commit_idx=-1).raw_id)
191
192 # check if commit is a branch name or branch hash
193 return commit_id in valid_heads
194
195 def _get_tree_cache_manager(self, namespace_type):
196 _namespace = caches.get_repo_namespace_key(
197 namespace_type, self.db_repo_name)
198 return caches.get_cache_manager('repo_cache_long', _namespace)
199
200 def _get_tree_at_commit(
201 self, c, commit_id, f_path, full_load=False, force=False):
202 def _cached_tree():
203 log.debug('Generating cached file tree for %s, %s, %s',
204 self.db_repo_name, commit_id, f_path)
205
206 c.full_load = full_load
207 return render(
208 'rhodecode:templates/files/files_browser_tree.mako',
209 self._get_template_context(c), self.request)
210
211 cache_manager = self._get_tree_cache_manager(caches.FILE_TREE)
212
213 cache_key = caches.compute_key_from_params(
214 self.db_repo_name, commit_id, f_path)
215
216 if force:
217 # we want to force recompute of caches
218 cache_manager.remove_value(cache_key)
219
220 return cache_manager.get(cache_key, createfunc=_cached_tree)
221
222 def _get_archive_spec(self, fname):
223 log.debug('Detecting archive spec for: `%s`', fname)
224
225 fileformat = None
226 ext = None
227 content_type = None
228 for a_type, ext_data in settings.ARCHIVE_SPECS.items():
229 content_type, extension = ext_data
230
231 if fname.endswith(extension):
232 fileformat = a_type
233 log.debug('archive is of type: %s', fileformat)
234 ext = extension
235 break
236
237 if not fileformat:
238 raise ValueError()
239
240 # left over part of whole fname is the commit
241 commit_id = fname[:-len(ext)]
242
243 return commit_id, ext, fileformat, content_type
244
245 @LoginRequired()
246 @HasRepoPermissionAnyDecorator(
247 'repository.read', 'repository.write', 'repository.admin')
248 @view_config(
249 route_name='repo_archivefile', request_method='GET',
250 renderer=None)
251 def repo_archivefile(self):
252 # archive cache config
253 from rhodecode import CONFIG
254 _ = self.request.translate
255 self.load_default_context()
256
257 fname = self.request.matchdict['fname']
258 subrepos = self.request.GET.get('subrepos') == 'true'
259
260 if not self.db_repo.enable_downloads:
261 return Response(_('Downloads disabled'))
262
263 try:
264 commit_id, ext, fileformat, content_type = \
265 self._get_archive_spec(fname)
266 except ValueError:
267 return Response(_('Unknown archive type for: `{}`').format(fname))
268
269 try:
270 commit = self.rhodecode_vcs_repo.get_commit(commit_id)
271 except CommitDoesNotExistError:
272 return Response(_('Unknown commit_id %s') % commit_id)
273 except EmptyRepositoryError:
274 return Response(_('Empty repository'))
275
276 archive_name = '%s-%s%s%s' % (
277 safe_str(self.db_repo_name.replace('/', '_')),
278 '-sub' if subrepos else '',
279 safe_str(commit.short_id), ext)
280
281 use_cached_archive = False
282 archive_cache_enabled = CONFIG.get(
283 'archive_cache_dir') and not self.request.GET.get('no_cache')
284
285 if archive_cache_enabled:
286 # check if we it's ok to write
287 if not os.path.isdir(CONFIG['archive_cache_dir']):
288 os.makedirs(CONFIG['archive_cache_dir'])
289 cached_archive_path = os.path.join(
290 CONFIG['archive_cache_dir'], archive_name)
291 if os.path.isfile(cached_archive_path):
292 log.debug('Found cached archive in %s', cached_archive_path)
293 fd, archive = None, cached_archive_path
294 use_cached_archive = True
295 else:
296 log.debug('Archive %s is not yet cached', archive_name)
297
298 if not use_cached_archive:
299 # generate new archive
300 fd, archive = tempfile.mkstemp()
301 log.debug('Creating new temp archive in %s', archive)
302 try:
303 commit.archive_repo(archive, kind=fileformat, subrepos=subrepos)
304 except ImproperArchiveTypeError:
305 return _('Unknown archive type')
306 if archive_cache_enabled:
307 # if we generated the archive and we have cache enabled
308 # let's use this for future
309 log.debug('Storing new archive in %s', cached_archive_path)
310 shutil.move(archive, cached_archive_path)
311 archive = cached_archive_path
312
313 # store download action
314 audit_logger.store_web(
315 'repo.archive.download', action_data={
316 'user_agent': self.request.user_agent,
317 'archive_name': archive_name,
318 'archive_spec': fname,
319 'archive_cached': use_cached_archive},
320 user=self._rhodecode_user,
321 repo=self.db_repo,
322 commit=True
323 )
324
325 def get_chunked_archive(archive):
326 with open(archive, 'rb') as stream:
327 while True:
328 data = stream.read(16 * 1024)
329 if not data:
330 if fd: # fd means we used temporary file
331 os.close(fd)
332 if not archive_cache_enabled:
333 log.debug('Destroying temp archive %s', archive)
334 os.remove(archive)
335 break
336 yield data
337
338 response = Response(app_iter=get_chunked_archive(archive))
339 response.content_disposition = str(
340 'attachment; filename=%s' % archive_name)
341 response.content_type = str(content_type)
342
343 return response
344
345 def _get_file_node(self, commit_id, f_path):
346 if commit_id not in ['', None, 'None', '0' * 12, '0' * 40]:
347 commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
348 try:
349 node = commit.get_node(f_path)
350 if node.is_dir():
351 raise NodeError('%s path is a %s not a file'
352 % (node, type(node)))
353 except NodeDoesNotExistError:
354 commit = EmptyCommit(
355 commit_id=commit_id,
356 idx=commit.idx,
357 repo=commit.repository,
358 alias=commit.repository.alias,
359 message=commit.message,
360 author=commit.author,
361 date=commit.date)
362 node = FileNode(f_path, '', commit=commit)
363 else:
364 commit = EmptyCommit(
365 repo=self.rhodecode_vcs_repo,
366 alias=self.rhodecode_vcs_repo.alias)
367 node = FileNode(f_path, '', commit=commit)
368 return node
369
370 @LoginRequired()
371 @HasRepoPermissionAnyDecorator(
372 'repository.read', 'repository.write', 'repository.admin')
373 @view_config(
374 route_name='repo_files_diff', request_method='GET',
375 renderer=None)
376 def repo_files_diff(self):
377 c = self.load_default_context()
378 diff1 = self.request.GET.get('diff1', '')
379 diff2 = self.request.GET.get('diff2', '')
380 f_path = self.request.matchdict['f_path']
381
382 path1, diff1 = parse_path_ref(diff1, default_path=f_path)
383
384 ignore_whitespace = str2bool(self.request.GET.get('ignorews'))
385 line_context = self.request.GET.get('context', 3)
386
387 if not any((diff1, diff2)):
388 h.flash(
389 'Need query parameter "diff1" or "diff2" to generate a diff.',
390 category='error')
391 raise HTTPBadRequest()
392
393 c.action = self.request.GET.get('diff')
394 if c.action not in ['download', 'raw']:
395 compare_url = h.url(
396 'compare_url', repo_name=self.db_repo_name,
397 source_ref_type='rev',
398 source_ref=diff1,
399 target_repo=self.db_repo_name,
400 target_ref_type='rev',
401 target_ref=diff2,
402 f_path=f_path)
403 # redirect to new view if we render diff
404 raise HTTPFound(compare_url)
405
406 try:
407 node1 = self._get_file_node(diff1, path1)
408 node2 = self._get_file_node(diff2, f_path)
409 except (RepositoryError, NodeError):
410 log.exception("Exception while trying to get node from repository")
411 raise HTTPFound(
412 h.route_path('repo_files', repo_name=self.db_repo_name,
413 commit_id='tip', f_path=f_path))
414
415 if all(isinstance(node.commit, EmptyCommit)
416 for node in (node1, node2)):
417 raise HTTPNotFound()
418
419 c.commit_1 = node1.commit
420 c.commit_2 = node2.commit
421
422 if c.action == 'download':
423 _diff = diffs.get_gitdiff(node1, node2,
424 ignore_whitespace=ignore_whitespace,
425 context=line_context)
426 diff = diffs.DiffProcessor(_diff, format='gitdiff')
427
428 response = Response(diff.as_raw())
429 response.content_type = 'text/plain'
430 response.content_disposition = (
431 'attachment; filename=%s_%s_vs_%s.diff' % (f_path, diff1, diff2)
432 )
433 charset = self._get_default_encoding(c)
434 if charset:
435 response.charset = charset
436 return response
437
438 elif c.action == 'raw':
439 _diff = diffs.get_gitdiff(node1, node2,
440 ignore_whitespace=ignore_whitespace,
441 context=line_context)
442 diff = diffs.DiffProcessor(_diff, format='gitdiff')
443
444 response = Response(diff.as_raw())
445 response.content_type = 'text/plain'
446 charset = self._get_default_encoding(c)
447 if charset:
448 response.charset = charset
449 return response
450
451 # in case we ever end up here
452 raise HTTPNotFound()
453
454 @LoginRequired()
455 @HasRepoPermissionAnyDecorator(
456 'repository.read', 'repository.write', 'repository.admin')
457 @view_config(
458 route_name='repo_files_diff_2way_redirect', request_method='GET',
459 renderer=None)
460 def repo_files_diff_2way_redirect(self):
461 """
462 Kept only to make OLD links work
463 """
464 diff1 = self.request.GET.get('diff1', '')
465 diff2 = self.request.GET.get('diff2', '')
466 f_path = self.request.matchdict['f_path']
467
468 if not any((diff1, diff2)):
469 h.flash(
470 'Need query parameter "diff1" or "diff2" to generate a diff.',
471 category='error')
472 raise HTTPBadRequest()
473
474 compare_url = h.url(
475 'compare_url', repo_name=self.db_repo_name,
476 source_ref_type='rev',
477 source_ref=diff1,
478 target_repo=self.db_repo_name,
479 target_ref_type='rev',
480 target_ref=diff2,
481 f_path=f_path,
482 diffmode='sideside')
483 raise HTTPFound(compare_url)
484
485 @LoginRequired()
486 @HasRepoPermissionAnyDecorator(
487 'repository.read', 'repository.write', 'repository.admin')
488 @view_config(
489 route_name='repo_files', request_method='GET',
490 renderer=None)
491 @view_config(
492 route_name='repo_files:default_path', request_method='GET',
493 renderer=None)
494 @view_config(
495 route_name='repo_files:default_commit', request_method='GET',
496 renderer=None)
497 @view_config(
498 route_name='repo_files:rendered', request_method='GET',
499 renderer=None)
500 @view_config(
501 route_name='repo_files:annotated', request_method='GET',
502 renderer=None)
503 def repo_files(self):
504 c = self.load_default_context()
505
506 view_name = getattr(self.request.matched_route, 'name', None)
507
508 c.annotate = view_name == 'repo_files:annotated'
509 # default is false, but .rst/.md files later are auto rendered, we can
510 # overwrite auto rendering by setting this GET flag
511 c.renderer = view_name == 'repo_files:rendered' or \
512 not self.request.GET.get('no-render', False)
513
514 # redirect to given commit_id from form if given
515 get_commit_id = self.request.GET.get('at_rev', None)
516 if get_commit_id:
517 self._get_commit_or_redirect(get_commit_id)
518
519 commit_id, f_path = self._get_commit_and_path()
520 c.commit = self._get_commit_or_redirect(commit_id)
521 c.branch = self.request.GET.get('branch', None)
522 c.f_path = f_path
523
524 # prev link
525 try:
526 prev_commit = c.commit.prev(c.branch)
527 c.prev_commit = prev_commit
528 c.url_prev = h.route_path(
529 'repo_files', repo_name=self.db_repo_name,
530 commit_id=prev_commit.raw_id, f_path=f_path)
531 if c.branch:
532 c.url_prev += '?branch=%s' % c.branch
533 except (CommitDoesNotExistError, VCSError):
534 c.url_prev = '#'
535 c.prev_commit = EmptyCommit()
536
537 # next link
538 try:
539 next_commit = c.commit.next(c.branch)
540 c.next_commit = next_commit
541 c.url_next = h.route_path(
542 'repo_files', repo_name=self.db_repo_name,
543 commit_id=next_commit.raw_id, f_path=f_path)
544 if c.branch:
545 c.url_next += '?branch=%s' % c.branch
546 except (CommitDoesNotExistError, VCSError):
547 c.url_next = '#'
548 c.next_commit = EmptyCommit()
549
550 # files or dirs
551 try:
552 c.file = c.commit.get_node(f_path)
553 c.file_author = True
554 c.file_tree = ''
555
556 # load file content
557 if c.file.is_file():
558 c.lf_node = c.file.get_largefile_node()
559
560 c.file_source_page = 'true'
561 c.file_last_commit = c.file.last_commit
562 if c.file.size < c.visual.cut_off_limit_diff:
563 if c.annotate: # annotation has precedence over renderer
564 c.annotated_lines = filenode_as_annotated_lines_tokens(
565 c.file
566 )
567 else:
568 c.renderer = (
569 c.renderer and h.renderer_from_filename(c.file.path)
570 )
571 if not c.renderer:
572 c.lines = filenode_as_lines_tokens(c.file)
573
574 c.on_branch_head = self._is_valid_head(
575 commit_id, self.rhodecode_vcs_repo)
576
577 branch = c.commit.branch if (
578 c.commit.branch and '/' not in c.commit.branch) else None
579 c.branch_or_raw_id = branch or c.commit.raw_id
580 c.branch_name = c.commit.branch or h.short_id(c.commit.raw_id)
581
582 author = c.file_last_commit.author
583 c.authors = [[
584 h.email(author),
585 h.person(author, 'username_or_name_or_email'),
586 1
587 ]]
588
589 else: # load tree content at path
590 c.file_source_page = 'false'
591 c.authors = []
592 # this loads a simple tree without metadata to speed things up
593 # later via ajax we call repo_nodetree_full and fetch whole
594 c.file_tree = self._get_tree_at_commit(
595 c, c.commit.raw_id, f_path)
596
597 except RepositoryError as e:
598 h.flash(safe_str(h.escape(e)), category='error')
599 raise HTTPNotFound()
600
601 if self.request.environ.get('HTTP_X_PJAX'):
602 html = render('rhodecode:templates/files/files_pjax.mako',
603 self._get_template_context(c), self.request)
604 else:
605 html = render('rhodecode:templates/files/files.mako',
606 self._get_template_context(c), self.request)
607 return Response(html)
608
609 @HasRepoPermissionAnyDecorator(
610 'repository.read', 'repository.write', 'repository.admin')
611 @view_config(
612 route_name='repo_files:annotated_previous', request_method='GET',
613 renderer=None)
614 def repo_files_annotated_previous(self):
615 self.load_default_context()
616
617 commit_id, f_path = self._get_commit_and_path()
618 commit = self._get_commit_or_redirect(commit_id)
619 prev_commit_id = commit.raw_id
620 line_anchor = self.request.GET.get('line_anchor')
621 is_file = False
622 try:
623 _file = commit.get_node(f_path)
624 is_file = _file.is_file()
625 except (NodeDoesNotExistError, CommitDoesNotExistError, VCSError):
626 pass
627
628 if is_file:
629 history = commit.get_file_history(f_path)
630 prev_commit_id = history[1].raw_id \
631 if len(history) > 1 else prev_commit_id
632 prev_url = h.route_path(
633 'repo_files:annotated', repo_name=self.db_repo_name,
634 commit_id=prev_commit_id, f_path=f_path,
635 _anchor='L{}'.format(line_anchor))
636
637 raise HTTPFound(prev_url)
638
639 @LoginRequired()
640 @HasRepoPermissionAnyDecorator(
641 'repository.read', 'repository.write', 'repository.admin')
642 @view_config(
643 route_name='repo_nodetree_full', request_method='GET',
644 renderer=None, xhr=True)
645 @view_config(
646 route_name='repo_nodetree_full:default_path', request_method='GET',
647 renderer=None, xhr=True)
648 def repo_nodetree_full(self):
649 """
650 Returns rendered html of file tree that contains commit date,
651 author, commit_id for the specified combination of
652 repo, commit_id and file path
653 """
654 c = self.load_default_context()
655
656 commit_id, f_path = self._get_commit_and_path()
657 commit = self._get_commit_or_redirect(commit_id)
658 try:
659 dir_node = commit.get_node(f_path)
660 except RepositoryError as e:
661 return Response('error: {}'.format(safe_str(e)))
662
663 if dir_node.is_file():
664 return Response('')
665
666 c.file = dir_node
667 c.commit = commit
668
669 # using force=True here, make a little trick. We flush the cache and
670 # compute it using the same key as without previous full_load, so now
671 # the fully loaded tree is now returned instead of partial,
672 # and we store this in caches
673 html = self._get_tree_at_commit(
674 c, commit.raw_id, dir_node.path, full_load=True, force=True)
675
676 return Response(html)
677
678 def _get_attachement_disposition(self, f_path):
679 return 'attachment; filename=%s' % \
680 safe_str(f_path.split(Repository.NAME_SEP)[-1])
681
682 @LoginRequired()
683 @HasRepoPermissionAnyDecorator(
684 'repository.read', 'repository.write', 'repository.admin')
685 @view_config(
686 route_name='repo_file_raw', request_method='GET',
687 renderer=None)