fixed issues with gitsubmodule diffs

rhodecode/lib/diffs.py

@@ -1,517 +1,524 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.diffs
     ~~~~~~~~~~~~~~~~~~~
 
     Set of diffing helpers, previously part of vcs
 
 
     :created_on: Dec 4, 2011
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :original copyright: 2007-2008 by Armin Ronacher
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import re
 import difflib
 import markupsafe
 from itertools import tee, imap
 
 from pylons.i18n.translation import _
 
 from rhodecode.lib.vcs.exceptions import VCSError
-from rhodecode.lib.vcs.nodes import FileNode
-
+from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode
+from rhodecode.lib.helpers import escape
 from rhodecode.lib.utils import EmptyChangeset
 
 
 def wrap_to_table(str_):
     return '''<table class="code-difftable">
                 <tr class="line no-comment">
                 <td class="lineno new"></td>
                 <td class="code no-comment"><pre>%s</pre></td>
                 </tr>
               </table>''' % str_
 
 
 def wrapped_diff(filenode_old, filenode_new, cut_off_limit=None,
                 ignore_whitespace=True, line_context=3,
                 enable_comments=False):
     """
     returns a wrapped diff into a table, checks for cut_off_limit and presents
     proper message
     """
 
     if filenode_old is None:
         filenode_old = FileNode(filenode_new.path, '', EmptyChangeset())
 
     if filenode_old.is_binary or filenode_new.is_binary:
         diff = wrap_to_table(_('binary file'))
         stats = (0, 0)
         size = 0
 
     elif cut_off_limit != -1 and (cut_off_limit is None or
     (filenode_old.size < cut_off_limit and filenode_new.size < cut_off_limit)):
 
         f_gitdiff = get_gitdiff(filenode_old, filenode_new,
                                 ignore_whitespace=ignore_whitespace,
                                 context=line_context)
         diff_processor = DiffProcessor(f_gitdiff, format='gitdiff')
 
         diff = diff_processor.as_html(enable_comments=enable_comments)
         stats = diff_processor.stat()
         size = len(diff or '')
     else:
         diff = wrap_to_table(_('Changeset was to big and was cut off, use '
                                'diff menu to display this diff'))
         stats = (0, 0)
         size = 0
-
     if not diff:
+        submodules = filter(lambda o: isinstance(o, SubModuleNode),
+                            [filenode_new, filenode_old])
+        if submodules:
+            diff = wrap_to_table(escape('Submodule %r' % submodules[0]))
+        else:
         diff = wrap_to_table(_('No changes detected'))
 
     cs1 = filenode_old.changeset.raw_id
     cs2 = filenode_new.changeset.raw_id
 
     return size, cs1, cs2, diff, stats
 
 
 def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):
     """
     Returns git style diff between given ``filenode_old`` and ``filenode_new``.
 
     :param ignore_whitespace: ignore whitespaces in diff
     """
     # make sure we pass in default context
     context = context or 3
+    submodules = filter(lambda o: isinstance(o, SubModuleNode),
+                        [filenode_new, filenode_old])
+    if submodules:
+        return ''
 
     for filenode in (filenode_old, filenode_new):
         if not isinstance(filenode, FileNode):
             raise VCSError("Given object should be FileNode object, not %s"
                 % filenode.__class__)
 
     repo = filenode_new.changeset.repository
     old_raw_id = getattr(filenode_old.changeset, 'raw_id', repo.EMPTY_CHANGESET)
     new_raw_id = getattr(filenode_new.changeset, 'raw_id', repo.EMPTY_CHANGESET)
 
     vcs_gitdiff = repo.get_diff(old_raw_id, new_raw_id, filenode_new.path,
                                  ignore_whitespace, context)
-
     return vcs_gitdiff
 
 
 class DiffProcessor(object):
     """
     Give it a unified diff and it returns a list of the files that were
     mentioned in the diff together with a dict of meta information that
     can be used to render it in a HTML template.
     """
     _chunk_re = re.compile(r'@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
 
     def __init__(self, diff, differ='diff', format='udiff'):
         """
         :param diff:   a text in diff format or generator
         :param format: format of diff passed, `udiff` or `gitdiff`
         """
         if isinstance(diff, basestring):
             diff = [diff]
 
         self.__udiff = diff
         self.__format = format
         self.adds = 0
         self.removes = 0
 
         if isinstance(self.__udiff, basestring):
             self.lines = iter(self.__udiff.splitlines(1))
 
         elif self.__format == 'gitdiff':
             udiff_copy = self.copy_iterator()
             self.lines = imap(self.escaper, self._parse_gitdiff(udiff_copy))
         else:
             udiff_copy = self.copy_iterator()
             self.lines = imap(self.escaper, udiff_copy)
 
         # Select a differ.
         if differ == 'difflib':
             self.differ = self._highlight_line_difflib
         else:
             self.differ = self._highlight_line_udiff
 
     def escaper(self, string):
         return markupsafe.escape(string)
 
     def copy_iterator(self):
         """
         make a fresh copy of generator, we should not iterate thru
         an original as it's needed for repeating operations on
         this instance of DiffProcessor
         """
         self.__udiff, iterator_copy = tee(self.__udiff)
         return iterator_copy
 
     def _extract_rev(self, line1, line2):
         """
         Extract the filename and revision hint from a line.
         """
 
         try:
             if line1.startswith('--- ') and line2.startswith('+++ '):
                 l1 = line1[4:].split(None, 1)
                 old_filename = (l1[0].replace('a/', '', 1)
                                 if len(l1) >= 1 else None)
                 old_rev = l1[1] if len(l1) == 2 else 'old'
 
                 l2 = line2[4:].split(None, 1)
                 new_filename = (l2[0].replace('b/', '', 1)
                                 if len(l1) >= 1 else None)
                 new_rev = l2[1] if len(l2) == 2 else 'new'
 
                 filename = (old_filename
                             if old_filename != '/dev/null' else new_filename)
 
                 return filename, new_rev, old_rev
         except (ValueError, IndexError):
             pass
 
         return None, None, None
 
     def _parse_gitdiff(self, diffiterator):
         def line_decoder(l):
             if l.startswith('+') and not l.startswith('+++'):
                 self.adds += 1
             elif l.startswith('-') and not l.startswith('---'):
                 self.removes += 1
             return l.decode('utf8', 'replace')
 
         output = list(diffiterator)
         size = len(output)
 
         if size == 2:
             l = []
             l.extend([output[0]])
             l.extend(output[1].splitlines(1))
             return map(line_decoder, l)
         elif size == 1:
             return  map(line_decoder, output[0].splitlines(1))
         elif size == 0:
             return []
 
         raise Exception('wrong size of diff %s' % size)
 
     def _highlight_line_difflib(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
 
         if line['action'] == 'del':
             old, new = line, next_
         else:
             old, new = next_, line
 
         oldwords = re.split(r'(\W)', old['line'])
         newwords = re.split(r'(\W)', new['line'])
 
         sequence = difflib.SequenceMatcher(None, oldwords, newwords)
 
         oldfragments, newfragments = [], []
         for tag, i1, i2, j1, j2 in sequence.get_opcodes():
             oldfrag = ''.join(oldwords[i1:i2])
             newfrag = ''.join(newwords[j1:j2])
             if tag != 'equal':
                 if oldfrag:
                     oldfrag = '<del>%s</del>' % oldfrag
                 if newfrag:
                     newfrag = '<ins>%s</ins>' % newfrag
             oldfragments.append(oldfrag)
             newfragments.append(newfrag)
 
         old['line'] = "".join(oldfragments)
         new['line'] = "".join(newfragments)
 
     def _highlight_line_udiff(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
         start = 0
         limit = min(len(line['line']), len(next_['line']))
         while start < limit and line['line'][start] == next_['line'][start]:
             start += 1
         end = -1
         limit -= start
         while -end <= limit and line['line'][end] == next_['line'][end]:
             end -= 1
         end += 1
         if start or end:
             def do(l):
                 last = end + len(l['line'])
                 if l['action'] == 'add':
                     tag = 'ins'
                 else:
                     tag = 'del'
                 l['line'] = '%s<%s>%s</%s>%s' % (
                     l['line'][:start],
                     tag,
                     l['line'][start:last],
                     tag,
                     l['line'][last:]
                 )
             do(line)
             do(next_)
 
     def _parse_udiff(self):
         """
         Parse the diff an return data for the template.
         """
         lineiter = self.lines
         files = []
         try:
             line = lineiter.next()
             # skip first context
             skipfirst = True
             while 1:
                 # continue until we found the old file
                 if not line.startswith('--- '):
                     line = lineiter.next()
                     continue
 
                 chunks = []
                 filename, old_rev, new_rev = \
                     self._extract_rev(line, lineiter.next())
                 files.append({
                     'filename':         filename,
                     'old_revision':     old_rev,
                     'new_revision':     new_rev,
                     'chunks':           chunks
                 })
 
                 line = lineiter.next()
                 while line:
                     match = self._chunk_re.match(line)
                     if not match:
                         break
 
                     lines = []
                     chunks.append(lines)
 
                     old_line, old_end, new_line, new_end = \
                         [int(x or 1) for x in match.groups()[:-1]]
                     old_line -= 1
                     new_line -= 1
                     context = len(match.groups()) == 5
                     old_end += old_line
                     new_end += new_line
 
                     if context:
                         if not skipfirst:
                             lines.append({
                                 'old_lineno': '...',
                                 'new_lineno': '...',
                                 'action':     'context',
                                 'line':       line,
                             })
                         else:
                             skipfirst = False
 
                     line = lineiter.next()
                     while old_line < old_end or new_line < new_end:
                         if line:
                             command, line = line[0], line[1:]
                         else:
                             command = ' '
                         affects_old = affects_new = False
 
                         # ignore those if we don't expect them
                         if command in '#@':
                             continue
                         elif command == '+':
                             affects_new = True
                             action = 'add'
                         elif command == '-':
                             affects_old = True
                             action = 'del'
                         else:
                             affects_old = affects_new = True
                             action = 'unmod'
 
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         line
                         })
                         line = lineiter.next()
 
         except StopIteration:
             pass
 
         # highlight inline changes
         for _ in files:
             for chunk in chunks:
                 lineiter = iter(chunk)
                 #first = True
                 try:
                     while 1:
                         line = lineiter.next()
                         if line['action'] != 'unmod':
                             nextline = lineiter.next()
                             if nextline['action'] == 'unmod' or \
                                nextline['action'] == line['action']:
                                 continue
                             self.differ(line, nextline)
                 except StopIteration:
                     pass
 
         return files
 
     def prepare(self):
         """
         Prepare the passed udiff for HTML rendering. It'l return a list
         of dicts
         """
         return self._parse_udiff()
 
     def _safe_id(self, idstring):
         """Make a string safe for including in an id attribute.
 
         The HTML spec says that id attributes 'must begin with
         a letter ([A-Za-z]) and may be followed by any number
         of letters, digits ([0-9]), hyphens ("-"), underscores
         ("_"), colons (":"), and periods (".")'. These regexps
         are slightly over-zealous, in that they remove colons
         and periods unnecessarily.
 
         Whitespace is transformed into underscores, and then
         anything which is not a hyphen or a character that
         matches \w (alphanumerics and underscore) is removed.
 
         """
         # Transform all whitespace to underscore
         idstring = re.sub(r'\s', "_", '%s' % idstring)
         # Remove everything that is not a hyphen or a member of \w
         idstring = re.sub(r'(?!-)\W', "", idstring).lower()
         return idstring
 
     def raw_diff(self):
         """
         Returns raw string as udiff
         """
         udiff_copy = self.copy_iterator()
         if self.__format == 'gitdiff':
             udiff_copy = self._parse_gitdiff(udiff_copy)
         return u''.join(udiff_copy)
 
     def as_html(self, table_class='code-difftable', line_class='line',
                 new_lineno_class='lineno old', old_lineno_class='lineno new',
                 code_class='code', enable_comments=False):
         """
         Return udiff as html table with customized css classes
         """
         def _link_to_if(condition, label, url):
             """
             Generates a link if condition is meet or just the label if not.
             """
 
             if condition:
                 return '''<a href="%(url)s">%(label)s</a>''' % {
                     'url': url,
                     'label': label
                 }
             else:
                 return label
         diff_lines = self.prepare()
         _html_empty = True
         _html = []
         _html.append('''<table class="%(table_class)s">\n''' % {
             'table_class': table_class
         })
         for diff in diff_lines:
             for line in diff['chunks']:
                 _html_empty = False
                 for change in line:
                     _html.append('''<tr class="%(lc)s %(action)s">\n''' % {
                         'lc': line_class,
                         'action': change['action']
                     })
                     anchor_old_id = ''
                     anchor_new_id = ''
                     anchor_old = "%(filename)s_o%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['old_lineno']
                     }
                     anchor_new = "%(filename)s_n%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['new_lineno']
                     }
                     cond_old = (change['old_lineno'] != '...' and
                                 change['old_lineno'])
                     cond_new = (change['new_lineno'] != '...' and
                                 change['new_lineno'])
                     if cond_old:
                         anchor_old_id = 'id="%s"' % anchor_old
                     if cond_new:
                         anchor_new_id = 'id="%s"' % anchor_new
                     ###########################################################
                     # OLD LINE NUMBER
                     ###########################################################
                     _html.append('''\t<td %(a_id)s class="%(olc)s">''' % {
                         'a_id': anchor_old_id,
                         'olc': old_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(True, change['old_lineno'],
                                             '#%s' % anchor_old)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # NEW LINE NUMBER
                     ###########################################################
 
                     _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {
                         'a_id': anchor_new_id,
                         'nlc': new_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(True, change['new_lineno'],
                                             '#%s' % anchor_new)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # CODE
                     ###########################################################
                     comments = '' if enable_comments else 'no-comment'
                     _html.append('''\t<td class="%(cc)s %(inc)s">''' % {
                         'cc': code_class,
                         'inc': comments
                     })
                     _html.append('''\n\t\t<pre>%(code)s</pre>\n''' % {
                         'code': change['line']
                     })
                     _html.append('''\t</td>''')
                     _html.append('''\n</tr>\n''')
         _html.append('''</table>''')
         if _html_empty:
             return None
         return ''.join(_html)
 
     def stat(self):
         """
         Returns tuple of added, and removed lines for this instance
         """
         return self.adds, self.removes

rhodecode/lib/vcs/backends/git/changeset.py

@@ -1,454 +1,460 b''
 import re
 from itertools import chain
 from dulwich import objects
 from subprocess import Popen, PIPE
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.exceptions import RepositoryError
 from rhodecode.lib.vcs.exceptions import ChangesetError
 from rhodecode.lib.vcs.exceptions import NodeDoesNotExistError
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.exceptions import ChangesetDoesNotExistError
 from rhodecode.lib.vcs.exceptions import ImproperArchiveTypeError
 from rhodecode.lib.vcs.backends.base import BaseChangeset
 from rhodecode.lib.vcs.nodes import FileNode, DirNode, NodeKind, RootNode, \
     RemovedFileNode, SubModuleNode
 from rhodecode.lib.vcs.utils import safe_unicode
 from rhodecode.lib.vcs.utils import date_fromtimestamp
 from rhodecode.lib.vcs.utils.lazy import LazyProperty
 
 
 class GitChangeset(BaseChangeset):
     """
     Represents state of the repository at single revision.
     """
 
     def __init__(self, repository, revision):
         self._stat_modes = {}
         self.repository = repository
         self.raw_id = revision
         self.revision = repository.revisions.index(revision)
 
         self.short_id = self.raw_id[:12]
         self.id = self.raw_id
         try:
             commit = self.repository._repo.get_object(self.raw_id)
         except KeyError:
             raise RepositoryError("Cannot get object with id %s" % self.raw_id)
         self._commit = commit
         self._tree_id = commit.tree
 
         try:
             self.message = safe_unicode(commit.message[:-1])
             # Always strip last eol
         except UnicodeDecodeError:
             self.message = commit.message[:-1].decode(commit.encoding
                 or 'utf-8')
         #self.branch = None
         self.tags = []
         #tree = self.repository.get_object(self._tree_id)
         self.nodes = {}
         self._paths = {}
 
     @LazyProperty
     def author(self):
         return safe_unicode(self._commit.committer)
 
     @LazyProperty
     def date(self):
         return date_fromtimestamp(self._commit.commit_time,
                                   self._commit.commit_timezone)
 
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current changeset
         """
         return self.changed, self.added, self.removed
 
     @LazyProperty
     def branch(self):
 
         heads = self.repository._heads(reverse=False)
 
         ref = heads.get(self.raw_id)
         if ref:
             return safe_unicode(ref)
 
     def _fix_path(self, path):
         """
         Paths are stored without trailing slash so we need to get rid off it if
         needed.
         """
         if path.endswith('/'):
             path = path.rstrip('/')
         return path
 
     def _get_id_for_path(self, path):
 
         # FIXME: Please, spare a couple of minutes and make those codes cleaner;
         if not path in self._paths:
             path = path.strip('/')
             # set root tree
             tree = self.repository._repo[self._commit.tree]
             if path == '':
                 self._paths[''] = tree.id
                 return tree.id
             splitted = path.split('/')
             dirs, name = splitted[:-1], splitted[-1]
             curdir = ''
 
             # initially extract things from root dir
             for item, stat, id in tree.iteritems():
                 if curdir:
                     name = '/'.join((curdir, item))
                 else:
                     name = item
                 self._paths[name] = id
                 self._stat_modes[name] = stat
 
             for dir in dirs:
                 if curdir:
                     curdir = '/'.join((curdir, dir))
                 else:
                     curdir = dir
                 dir_id = None
                 for item, stat, id in tree.iteritems():
                     if dir == item:
                         dir_id = id
                 if dir_id:
                     # Update tree
                     tree = self.repository._repo[dir_id]
                     if not isinstance(tree, objects.Tree):
                         raise ChangesetError('%s is not a directory' % curdir)
                 else:
                     raise ChangesetError('%s have not been found' % curdir)
 
                 # cache all items from the given traversed tree
                 for item, stat, id in tree.iteritems():
                     if curdir:
                         name = '/'.join((curdir, item))
                     else:
                         name = item
                     self._paths[name] = id
                     self._stat_modes[name] = stat
             if not path in self._paths:
                 raise NodeDoesNotExistError("There is no file nor directory "
                     "at the given path %r at revision %r"
                     % (path, self.short_id))
         return self._paths[path]
 
     def _get_kind(self, path):
         id = self._get_id_for_path(path)
         obj = self.repository._repo[id]
         if isinstance(obj, objects.Blob):
             return NodeKind.FILE
         elif isinstance(obj, objects.Tree):
             return NodeKind.DIR
 
     def _get_file_nodes(self):
         return chain(*(t[2] for t in self.walk()))
 
     @LazyProperty
     def parents(self):
         """
         Returns list of parents changesets.
         """
         return [self.repository.get_changeset(parent)
             for parent in self._commit.parents]
 
     def next(self, branch=None):
 
         if branch and self.branch != branch:
             raise VCSError('Branch option used on changeset not belonging '
                            'to that branch')
 
         def _next(changeset, branch):
             try:
                 next_ = changeset.revision + 1
                 next_rev = changeset.repository.revisions[next_]
             except IndexError:
                 raise ChangesetDoesNotExistError
             cs = changeset.repository.get_changeset(next_rev)
 
             if branch and branch != cs.branch:
                 return _next(cs, branch)
 
             return cs
 
         return _next(self, branch)
 
     def prev(self, branch=None):
         if branch and self.branch != branch:
             raise VCSError('Branch option used on changeset not belonging '
                            'to that branch')
 
         def _prev(changeset, branch):
             try:
                 prev_ = changeset.revision - 1
                 if prev_ < 0:
                     raise IndexError
                 prev_rev = changeset.repository.revisions[prev_]
             except IndexError:
                 raise ChangesetDoesNotExistError
 
             cs = changeset.repository.get_changeset(prev_rev)
 
             if branch and branch != cs.branch:
                 return _prev(cs, branch)
 
             return cs
 
         return _prev(self, branch)
 
     def get_file_mode(self, path):
         """
         Returns stat mode of the file at the given ``path``.
         """
         # ensure path is traversed
         self._get_id_for_path(path)
         return self._stat_modes[path]
 
     def get_file_content(self, path):
         """
         Returns content of the file at given ``path``.
         """
         id = self._get_id_for_path(path)
         blob = self.repository._repo[id]
         return blob.as_pretty_string()
 
     def get_file_size(self, path):
         """
         Returns size of the file at given ``path``.
         """
         id = self._get_id_for_path(path)
         blob = self.repository._repo[id]
         return blob.raw_length()
 
     def get_file_changeset(self, path):
         """
         Returns last commit of the file at the given ``path``.
         """
         node = self.get_node(path)
         return node.history[0]
 
     def get_file_history(self, path):
         """
         Returns history of file as reversed list of ``Changeset`` objects for
         which file at given ``path`` has been modified.
 
         TODO: This function now uses os underlying 'git' and 'grep' commands
         which is generally not good. Should be replaced with algorithm
         iterating commits.
         """
         cmd = 'log --pretty="format: %%H" --name-status -p %s -- "%s"' % (
                   self.id, path
                )
         so, se = self.repository.run_git_command(cmd)
         ids = re.findall(r'\w{40}', so)
         return [self.repository.get_changeset(id) for id in ids]
 
     def get_file_annotate(self, path):
         """
         Returns a list of three element tuples with lineno,changeset and line
 
         TODO: This function now uses os underlying 'git' command which is
         generally not good. Should be replaced with algorithm iterating
         commits.
         """
         cmd = 'blame -l --root -r %s -- "%s"' % (self.id, path)
         # -l     ==> outputs long shas (and we need all 40 characters)
         # --root ==> doesn't put '^' character for bounderies
         # -r sha ==> blames for the given revision
         so, se = self.repository.run_git_command(cmd)
         annotate = []
         for i, blame_line in enumerate(so.split('\n')[:-1]):
             ln_no = i + 1
             id, line = re.split(r' \(.+?\) ', blame_line, 1)
             annotate.append((ln_no, self.repository.get_changeset(id), line))
         return annotate
 
     def fill_archive(self, stream=None, kind='tgz', prefix=None,
                      subrepos=False):
         """
         Fills up given stream.
 
         :param stream: file like object.
         :param kind: one of following: ``zip``, ``tgz`` or ``tbz2``.
             Default: ``tgz``.
         :param prefix: name of root directory in archive.
             Default is repository name and changeset's raw_id joined with dash
             (``repo-tip.<KIND>``).
         :param subrepos: include subrepos in this archive.
 
         :raise ImproperArchiveTypeError: If given kind is wrong.
         :raise VcsError: If given stream is None
 
         """
         allowed_kinds = settings.ARCHIVE_SPECS.keys()
         if kind not in allowed_kinds:
             raise ImproperArchiveTypeError('Archive kind not supported use one'
                 'of %s', allowed_kinds)
 
         if prefix is None:
             prefix = '%s-%s' % (self.repository.name, self.short_id)
         elif prefix.startswith('/'):
             raise VCSError("Prefix cannot start with leading slash")
         elif prefix.strip() == '':
             raise VCSError("Prefix cannot be empty")
 
         if kind == 'zip':
             frmt = 'zip'
         else:
             frmt = 'tar'
         cmd = 'git archive --format=%s --prefix=%s/ %s' % (frmt, prefix,
             self.raw_id)
         if kind == 'tgz':
             cmd += ' | gzip -9'
         elif kind == 'tbz2':
             cmd += ' | bzip2 -9'
 
         if stream is None:
             raise VCSError('You need to pass in a valid stream for filling'
                            ' with archival data')
         popen = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True,
             cwd=self.repository.path)
 
         buffer_size = 1024 * 8
         chunk = popen.stdout.read(buffer_size)
         while chunk:
             stream.write(chunk)
             chunk = popen.stdout.read(buffer_size)
         # Make sure all descriptors would be read
         popen.communicate()
 
     def get_nodes(self, path):
         if self._get_kind(path) != NodeKind.DIR:
             raise ChangesetError("Directory does not exist for revision %r at "
                 " %r" % (self.revision, path))
         path = self._fix_path(path)
         id = self._get_id_for_path(path)
         tree = self.repository._repo[id]
         dirnodes = []
         filenodes = []
         als = self.repository.alias
         for name, stat, id in tree.iteritems():
             if objects.S_ISGITLINK(stat):
                 dirnodes.append(SubModuleNode(name, url=None, changeset=id,
                                               alias=als))
                 continue
 
             obj = self.repository._repo.get_object(id)
             if path != '':
                 obj_path = '/'.join((path, name))
             else:
                 obj_path = name
             if obj_path not in self._stat_modes:
                 self._stat_modes[obj_path] = stat
             if isinstance(obj, objects.Tree):
                 dirnodes.append(DirNode(obj_path, changeset=self))
             elif isinstance(obj, objects.Blob):
                 filenodes.append(FileNode(obj_path, changeset=self, mode=stat))
             else:
                 raise ChangesetError("Requested object should be Tree "
                                      "or Blob, is %r" % type(obj))
         nodes = dirnodes + filenodes
         for node in nodes:
             if not node.path in self.nodes:
                 self.nodes[node.path] = node
         nodes.sort()
         return nodes
 
     def get_node(self, path):
         if isinstance(path, unicode):
             path = path.encode('utf-8')
         path = self._fix_path(path)
         if not path in self.nodes:
             try:
-                id = self._get_id_for_path(path)
+                id_ = self._get_id_for_path(path)
             except ChangesetError:
                 raise NodeDoesNotExistError("Cannot find one of parents' "
                     "directories for a given path: %s" % path)
-            obj = self.repository._repo.get_object(id)
+
+            als = self.repository.alias
+            _GL = lambda m: m and objects.S_ISGITLINK(m)
+            if _GL(self._stat_modes.get(path)):
+                node = SubModuleNode(path, url=None, changeset=id_, alias=als)
+            else:
+                obj = self.repository._repo.get_object(id_)
+
             if isinstance(obj, objects.Tree):
                 if path == '':
                     node = RootNode(changeset=self)
                 else:
                     node = DirNode(path, changeset=self)
                 node._tree = obj
             elif isinstance(obj, objects.Blob):
                 node = FileNode(path, changeset=self)
                 node._blob = obj
             else:
                 raise NodeDoesNotExistError("There is no file nor directory "
                     "at the given path %r at revision %r"
                     % (path, self.short_id))
             # cache node
             self.nodes[path] = node
         return self.nodes[path]
 
     @LazyProperty
     def affected_files(self):
         """
         Get's a fast accessible file changes for given changeset
         """
 
         return self.added + self.changed
 
     @LazyProperty
     def _diff_name_status(self):
         output = []
         for parent in self.parents:
             cmd = 'diff --name-status %s %s --encoding=utf8' % (parent.raw_id, self.raw_id)
             so, se = self.repository.run_git_command(cmd)
             output.append(so.strip())
         return '\n'.join(output)
 
     def _get_paths_for_status(self, status):
         """
         Returns sorted list of paths for given ``status``.
 
         :param status: one of: *added*, *modified* or *deleted*
         """
         paths = set()
         char = status[0].upper()
         for line in self._diff_name_status.splitlines():
             if not line:
                 continue
 
             if line.startswith(char):
                 splitted = line.split(char, 1)
                 if not len(splitted) == 2:
                     raise VCSError("Couldn't parse diff result:\n%s\n\n and "
                         "particularly that line: %s" % (self._diff_name_status,
                         line))
                 _path = splitted[1].strip()
                 paths.add(_path)
-
         return sorted(paths)
 
     @LazyProperty
     def added(self):
         """
         Returns list of added ``FileNode`` objects.
         """
         if not self.parents:
             return list(self._get_file_nodes())
         return [self.get_node(path) for path in self._get_paths_for_status('added')]
 
     @LazyProperty
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
         if not self.parents:
             return []
         return [self.get_node(path) for path in self._get_paths_for_status('modified')]
 
     @LazyProperty
     def removed(self):
         """
         Returns list of removed ``FileNode`` objects.
         """
         if not self.parents:
             return []
         return [RemovedFileNode(path) for path in self._get_paths_for_status('deleted')]

rhodecode/lib/vcs/nodes.py

@@ -1,600 +1,611 b''
 # -*- coding: utf-8 -*-
 """
     vcs.nodes
     ~~~~~~~~~
 
     Module holding everything related to vcs nodes.
 
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 import os
 import stat
 import posixpath
 import mimetypes
 
 from pygments import lexers
 
 from rhodecode.lib.vcs.utils.lazy import LazyProperty
 from rhodecode.lib.vcs.utils import safe_unicode, safe_str
 from rhodecode.lib.vcs.exceptions import NodeError
 from rhodecode.lib.vcs.exceptions import RemovedFileNodeError
+from rhodecode.lib.utils import EmptyChangeset
 
 
 class NodeKind:
     SUBMODULE = -1
     DIR = 1
     FILE = 2
 
 
 class NodeState:
     ADDED = u'added'
     CHANGED = u'changed'
     NOT_CHANGED = u'not changed'
     REMOVED = u'removed'
 
 
 class NodeGeneratorBase(object):
     """
     Base class for removed added and changed filenodes, it's a lazy generator
     class that will create filenodes only on iteration or call
 
     The len method doesn't need to create filenodes at all
     """
 
     def __init__(self, current_paths, cs):
         self.cs = cs
         self.current_paths = current_paths
 
     def __call__(self):
         return [n for n in self]
 
     def __getslice__(self, i, j):
         for p in self.current_paths[i:j]:
             yield self.cs.get_node(p)
 
     def __len__(self):
         return len(self.current_paths)
 
     def __iter__(self):
         for p in self.current_paths:
             yield self.cs.get_node(p)
 
 
 class AddedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding Added files for current changeset
     """
     pass
 
 
 class ChangedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding Changed files for current changeset
     """
     pass
 
 
 class RemovedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding removed files for current changeset
     """
     def __iter__(self):
         for p in self.current_paths:
             yield RemovedFileNode(path=p)
 
     def __getslice__(self, i, j):
         for p in self.current_paths[i:j]:
             yield RemovedFileNode(path=p)
 
 
 class Node(object):
     """
     Simplest class representing file or directory on repository.  SCM backends
     should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node``
     directly.
 
     Node's ``path`` cannot start with slash as we operate on *relative* paths
     only. Moreover, every single node is identified by the ``path`` attribute,
     so it cannot end with slash, too. Otherwise, path could lead to mistakes.
     """
 
     def __init__(self, path, kind):
         if path.startswith('/'):
             raise NodeError("Cannot initialize Node objects with slash at "
                 "the beginning as only relative paths are supported")
         self.path = path.rstrip('/')
         if path == '' and kind != NodeKind.DIR:
             raise NodeError("Only DirNode and its subclasses may be "
                             "initialized with empty path")
         self.kind = kind
         #self.dirs, self.files = [], []
         if self.is_root() and not self.is_dir():
             raise NodeError("Root node cannot be FILE kind")
 
     @LazyProperty
     def parent(self):
         parent_path = self.get_parent_path()
         if parent_path:
             if self.changeset:
                 return self.changeset.get_node(parent_path)
             return DirNode(parent_path)
         return None
 
     @LazyProperty
     def unicode_path(self):
         return safe_unicode(self.path)
 
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
         return safe_unicode(self.path.rstrip('/').split('/')[-1])
 
     def _get_kind(self):
         return self._kind
 
     def _set_kind(self, kind):
         if hasattr(self, '_kind'):
             raise NodeError("Cannot change node's kind")
         else:
             self._kind = kind
             # Post setter check (path's trailing slash)
             if self.path.endswith('/'):
                 raise NodeError("Node's path cannot end with slash")
 
     kind = property(_get_kind, _set_kind)
 
     def __cmp__(self, other):
         """
         Comparator using name of the node, needed for quick list sorting.
         """
         kind_cmp = cmp(self.kind, other.kind)
         if kind_cmp:
             return kind_cmp
         return cmp(self.name, other.name)
 
     def __eq__(self, other):
         for attr in ['name', 'path', 'kind']:
             if getattr(self, attr) != getattr(other, attr):
                 return False
         if self.is_file():
             if self.content != other.content:
                 return False
         else:
             # For DirNode's check without entering each dir
             self_nodes_paths = list(sorted(n.path for n in self.nodes))
             other_nodes_paths = list(sorted(n.path for n in self.nodes))
             if self_nodes_paths != other_nodes_paths:
                 return False
         return True
 
     def __nq__(self, other):
         return not self.__eq__(other)
 
     def __repr__(self):
         return '<%s %r>' % (self.__class__.__name__, self.path)
 
     def __str__(self):
         return self.__repr__()
 
     def __unicode__(self):
         return self.name
 
     def get_parent_path(self):
         """
         Returns node's parent path or empty string if node is root.
         """
         if self.is_root():
             return ''
         return posixpath.dirname(self.path.rstrip('/')) + '/'
 
     def is_file(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.FILE
 
     def is_dir(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.DIR
 
     def is_root(self):
         """
         Returns ``True`` if node is a root node and ``False`` otherwise.
         """
         return self.kind == NodeKind.DIR and self.path == ''
 
     def is_submodule(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.SUBMODULE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.SUBMODULE
 
     @LazyProperty
     def added(self):
         return self.state is NodeState.ADDED
 
     @LazyProperty
     def changed(self):
         return self.state is NodeState.CHANGED
 
     @LazyProperty
     def not_changed(self):
         return self.state is NodeState.NOT_CHANGED
 
     @LazyProperty
     def removed(self):
         return self.state is NodeState.REMOVED
 
 
 class FileNode(Node):
     """
     Class representing file nodes.
 
     :attribute: path: path to the node, relative to repostiory's root
     :attribute: content: if given arbitrary sets content of the file
     :attribute: changeset: if given, first time content is accessed, callback
     :attribute: mode: octal stat mode for a node. Default is 0100644.
     """
 
     def __init__(self, path, content=None, changeset=None, mode=None):
         """
         Only one of ``content`` and ``changeset`` may be given. Passing both
         would raise ``NodeError`` exception.
 
         :param path: relative path to the node
         :param content: content may be passed to constructor
         :param changeset: if given, will use it to lazily fetch content
         :param mode: octal representation of ST_MODE (i.e. 0100644)
         """
 
         if content and changeset:
             raise NodeError("Cannot use both content and changeset")
         super(FileNode, self).__init__(path, kind=NodeKind.FILE)
         self.changeset = changeset
         self._content = content
         self._mode = mode or 0100644
 
     @LazyProperty
     def mode(self):
         """
         Returns lazily mode of the FileNode. If ``changeset`` is not set, would
         use value given at initialization or 0100644 (default).
         """
         if self.changeset:
             mode = self.changeset.get_file_mode(self.path)
         else:
             mode = self._mode
         return mode
 
     @property
     def content(self):
         """
         Returns lazily content of the FileNode. If possible, would try to
         decode content from UTF-8.
         """
         if self.changeset:
             content = self.changeset.get_file_content(self.path)
         else:
             content = self._content
 
         if bool(content and '\0' in content):
             return content
         return safe_unicode(content)
 
     @LazyProperty
     def size(self):
         if self.changeset:
             return self.changeset.get_file_size(self.path)
         raise NodeError("Cannot retrieve size of the file without related "
             "changeset attribute")
 
     @LazyProperty
     def message(self):
         if self.changeset:
             return self.last_changeset.message
         raise NodeError("Cannot retrieve message of the file without related "
             "changeset attribute")
 
     @LazyProperty
     def last_changeset(self):
         if self.changeset:
             return self.changeset.get_file_changeset(self.path)
         raise NodeError("Cannot retrieve last changeset of the file without "
             "related changeset attribute")
 
     def get_mimetype(self):
         """
         Mimetype is calculated based on the file's content. If ``_mimetype``
         attribute is available, it will be returned (backends which store
         mimetypes or can easily recognize them, should set this private
         attribute to indicate that type should *NOT* be calculated).
         """
         if hasattr(self, '_mimetype'):
             if (isinstance(self._mimetype, (tuple, list,)) and
                 len(self._mimetype) == 2):
                 return self._mimetype
             else:
                 raise NodeError('given _mimetype attribute must be an 2 '
                                'element list or tuple')
 
         mtype, encoding = mimetypes.guess_type(self.name)
 
         if mtype is None:
             if self.is_binary:
                 mtype = 'application/octet-stream'
                 encoding = None
             else:
                 mtype = 'text/plain'
                 encoding = None
         return mtype, encoding
 
     @LazyProperty
     def mimetype(self):
         """
         Wrapper around full mimetype info. It returns only type of fetched
         mimetype without the encoding part. use get_mimetype function to fetch
         full set of (type,encoding)
         """
         return self.get_mimetype()[0]
 
     @LazyProperty
     def mimetype_main(self):
         return self.mimetype.split('/')[0]
 
     @LazyProperty
     def lexer(self):
         """
         Returns pygment's lexer class. Would try to guess lexer taking file's
         content, name and mimetype.
         """
         try:
             lexer = lexers.guess_lexer_for_filename(self.name, self.content)
         except lexers.ClassNotFound:
             lexer = lexers.TextLexer()
         # returns first alias
         return lexer
 
     @LazyProperty
     def lexer_alias(self):
         """
         Returns first alias of the lexer guessed for this file.
         """
         return self.lexer.aliases[0]
 
     @LazyProperty
     def history(self):
         """
         Returns a list of changeset for this file in which the file was changed
         """
         if self.changeset is None:
             raise NodeError('Unable to get changeset for this FileNode')
         return self.changeset.get_file_history(self.path)
 
     @LazyProperty
     def annotate(self):
         """
         Returns a list of three element tuples with lineno,changeset and line
         """
         if self.changeset is None:
             raise NodeError('Unable to get changeset for this FileNode')
         return self.changeset.get_file_annotate(self.path)
 
     @LazyProperty
     def state(self):
         if not self.changeset:
             raise NodeError("Cannot check state of the node if it's not "
                 "linked with changeset")
         elif self.path in (node.path for node in self.changeset.added):
             return NodeState.ADDED
         elif self.path in (node.path for node in self.changeset.changed):
             return NodeState.CHANGED
         else:
             return NodeState.NOT_CHANGED
 
     @property
     def is_binary(self):
         """
         Returns True if file has binary content.
         """
         _bin = '\0' in self.content
         return _bin
 
     @LazyProperty
     def extension(self):
         """Returns filenode extension"""
         return self.name.split('.')[-1]
 
     def is_executable(self):
         """
         Returns ``True`` if file has executable flag turned on.
         """
         return bool(self.mode & stat.S_IXUSR)
 
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  self.changeset.short_id)
 
 
 class RemovedFileNode(FileNode):
     """
     Dummy FileNode class - trying to access any public attribute except path,
     name, kind or state (or methods/attributes checking those two) would raise
     RemovedFileNodeError.
     """
     ALLOWED_ATTRIBUTES = ['name', 'path', 'state', 'is_root', 'is_file',
         'is_dir', 'kind', 'added', 'changed', 'not_changed', 'removed']
 
     def __init__(self, path):
         """
         :param path: relative path to the node
         """
         super(RemovedFileNode, self).__init__(path=path)
 
     def __getattribute__(self, attr):
         if attr.startswith('_') or attr in RemovedFileNode.ALLOWED_ATTRIBUTES:
             return super(RemovedFileNode, self).__getattribute__(attr)
         raise RemovedFileNodeError("Cannot access attribute %s on "
             "RemovedFileNode" % attr)
 
     @LazyProperty
     def state(self):
         return NodeState.REMOVED
 
 
 class DirNode(Node):
     """
     DirNode stores list of files and directories within this node.
     Nodes may be used standalone but within repository context they
     lazily fetch data within same repositorty's changeset.
     """
 
     def __init__(self, path, nodes=(), changeset=None):
         """
         Only one of ``nodes`` and ``changeset`` may be given. Passing both
         would raise ``NodeError`` exception.
 
         :param path: relative path to the node
         :param nodes: content may be passed to constructor
         :param changeset: if given, will use it to lazily fetch content
         :param size: always 0 for ``DirNode``
         """
         if nodes and changeset:
             raise NodeError("Cannot use both nodes and changeset")
         super(DirNode, self).__init__(path, NodeKind.DIR)
         self.changeset = changeset
         self._nodes = nodes
 
     @LazyProperty
     def content(self):
         raise NodeError("%s represents a dir and has no ``content`` attribute"
             % self)
 
     @LazyProperty
     def nodes(self):
         if self.changeset:
             nodes = self.changeset.get_nodes(self.path)
         else:
             nodes = self._nodes
         self._nodes_dict = dict((node.path, node) for node in nodes)
         return sorted(nodes)
 
     @LazyProperty
     def files(self):
         return sorted((node for node in self.nodes if node.is_file()))
 
     @LazyProperty
     def dirs(self):
         return sorted((node for node in self.nodes if node.is_dir()))
 
     def __iter__(self):
         for node in self.nodes:
             yield node
 
     def get_node(self, path):
         """
         Returns node from within this particular ``DirNode``, so it is now
         allowed to fetch, i.e. node located at 'docs/api/index.rst' from node
         'docs'. In order to access deeper nodes one must fetch nodes between
         them first - this would work::
 
            docs = root.get_node('docs')
            docs.get_node('api').get_node('index.rst')
 
         :param: path - relative to the current node
 
         .. note::
            To access lazily (as in example above) node have to be initialized
            with related changeset object - without it node is out of
            context and may know nothing about anything else than nearest
            (located at same level) nodes.
         """
         try:
             path = path.rstrip('/')
             if path == '':
                 raise NodeError("Cannot retrieve node without path")
             self.nodes  # access nodes first in order to set _nodes_dict
             paths = path.split('/')
             if len(paths) == 1:
                 if not self.is_root():
                     path = '/'.join((self.path, paths[0]))
                 else:
                     path = paths[0]
                 return self._nodes_dict[path]
             elif len(paths) > 1:
                 if self.changeset is None:
                     raise NodeError("Cannot access deeper "
                                     "nodes without changeset")
                 else:
                     path1, path2 = paths[0], '/'.join(paths[1:])
                     return self.get_node(path1).get_node(path2)
             else:
                 raise KeyError
         except KeyError:
             raise NodeError("Node does not exist at %s" % path)
 
     @LazyProperty
     def state(self):
         raise NodeError("Cannot access state of DirNode")
 
     @LazyProperty
     def size(self):
         size = 0
         for root, dirs, files in self.changeset.walk(self.path):
             for f in files:
                 size += f.size
 
         return size
 
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  self.changeset.short_id)
 
 
 class RootNode(DirNode):
     """
     DirNode being the root node of the repository.
     """
 
     def __init__(self, nodes=(), changeset=None):
         super(RootNode, self).__init__(path='', nodes=nodes,
             changeset=changeset)
 
     def __repr__(self):
         return '<%s>' % self.__class__.__name__
 
 
 class SubModuleNode(Node):
     """
     represents a SubModule of Git or SubRepo of Mercurial
     """
+    is_binary = False
+    size = 0
+
     def __init__(self, name, url=None, changeset=None, alias=None):
         self.path = name
         self.kind = NodeKind.SUBMODULE
         self.alias = alias
-        # changeset MUST be STR !! since it can point to non-valid SCM
-        self.changeset = str(changeset)
+        # we have to use emptyChangeset here since this can point to svn/git/hg
+        # submodules we cannot get from repository
+        self.changeset = EmptyChangeset(str(changeset), alias=alias)
         self.url = url or self._extract_submodule_url()
 
+    def __repr__(self):
+        return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
+                                 self.changeset.short_id)
+
     def _extract_submodule_url(self):
         if self.alias == 'git':
+            #TODO: find a way to parse gits submodule file and extract the
+            # linking URL
             return self.path
         if self.alias == 'hg':
             return self.path
 
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
         org = safe_unicode(self.path.rstrip('/').split('/')[-1])
-        return u'%s @ %s' % (org, self.changeset[:12])
+        return u'%s @ %s' % (org, self.changeset.short_id)