changelog pagination with branch filtering now uses

common logic that non branch filtered version.
- introduced CollectionGenerator object as a helper for
  achieving abstraction layer on top of filtered results

rhodecode/controllers/changelog.py

@@ -1,125 +1,123
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.changelog
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     changelog controller for rhodecode
 
     :created_on: Apr 21, 2010
     :author: marcink
     :copyright: (C) 2010-2012 Marcin Kuzminski <[e-mail]>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
 import traceback
 
 from pylons import request, url, session, tmpl_context as c
 from pylons.controllers.util import redirect
 from pylons.i18n.translation import _
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.helpers import RepoPage
 from rhodecode.lib.compat import json
 from rhodecode.lib.graphmod import _colored, _dagwalker
 from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetDoesNotExistError
 from rhodecode.lib.utils2 import safe_int
 
 log = logging.getLogger(__name__)
 
 
 class ChangelogController(BaseRepoController):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def __before__(self):
         super(ChangelogController, self).__before__()
         c.affected_files_cut_off = 60
 
     def index(self):
         limit = 100
         default = 20
         if request.params.get('size'):
             try:
                 int_size = int(request.params.get('size'))
             except ValueError:
                 int_size = default
             c.size = max(min(int_size, limit), 1)
             session['changelog_size'] = c.size
             session.save()
         else:
             c.size = int(session.get('changelog_size', default))
         # min size must be 1
         c.size = max(c.size, 1)
         p = safe_int(request.params.get('page', 1), 1)
         branch_name = request.params.get('branch', None)
         try:
-            if branch_name:
-                collection = [z for z in
-                              c.rhodecode_repo.get_changesets(start=0,
-                                                    branch_name=branch_name)]
+            collection = c.rhodecode_repo.get_changesets(start=0,
+                                                    branch_name=branch_name)
                 c.total_cs = len(collection)
-            else:
-                collection = c.rhodecode_repo
-                c.total_cs = len(c.rhodecode_repo)
 
             c.pagination = RepoPage(collection, page=p, item_count=c.total_cs,
                                     items_per_page=c.size, branch=branch_name)
             collection = list(c.pagination)
-            page_revisions = [x.raw_id for x in collection]
+            page_revisions = [x.raw_id for x in c.pagination]
             c.comments = c.rhodecode_db_repo.get_comments(page_revisions)
             c.statuses = c.rhodecode_db_repo.statuses(page_revisions)
         except (RepositoryError, ChangesetDoesNotExistError, Exception), e:
             log.error(traceback.format_exc())
             h.flash(str(e), category='error')
             return redirect(url('changelog_home', repo_name=c.repo_name))
 
-        self._graph(c.rhodecode_repo, collection, c.total_cs, c.size, p)
-
         c.branch_name = branch_name
         c.branch_filters = [('', _('All Branches'))] + \
             [(k, k) for k in c.rhodecode_repo.branches.keys()]
 
+        self._graph(c.rhodecode_repo, [x.revision for x in c.pagination],
+                    c.total_cs, c.size, p)
+
         return render('changelog/changelog.html')
 
     def changelog_details(self, cs):
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             c.cs = c.rhodecode_repo.get_changeset(cs)
             return render('changelog/changelog_details.html')
 
-    def _graph(self, repo, collection, repo_size, size, p):
+    def _graph(self, repo, revs_int, repo_size, size, p):
         """
-        Generates a DAG graph for mercurial
+        Generates a DAG graph for repo
 
-        :param repo: repo instance
-        :param size: number of commits to show
-        :param p: page number
+        :param repo:
+        :param revs_int:
+        :param repo_size:
+        :param size:
+        :param p:
         """
-        if not collection:
+        if not revs_int:
             c.jsdata = json.dumps([])
             return
 
         data = []
-        revs = [x.revision for x in collection]
+        revs = revs_int
 
         dag = _dagwalker(repo, revs, repo.alias)
         dag = _colored(dag)
         for (id, type, ctx, vtx, edges) in dag:
             data.append(['', vtx, edges])
 
         c.jsdata = json.dumps(data)

rhodecode/lib/vcs/backends/base.py

@@ -1,1004 +1,1028
 # -*- coding: utf-8 -*-
 """
     vcs.backends.base
     ~~~~~~~~~~~~~~~~~
 
     Base for all available scm backends
 
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 
 import datetime
 from itertools import chain
 from rhodecode.lib.vcs.utils import author_name, author_email
 from rhodecode.lib.vcs.utils.lazy import LazyProperty
 from rhodecode.lib.vcs.utils.helpers import get_dict_for_attrs
 from rhodecode.lib.vcs.conf import settings
 
 from rhodecode.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, \
     NodeAlreadyAddedError, NodeAlreadyChangedError, NodeAlreadyExistsError, \
     NodeAlreadyRemovedError, NodeDoesNotExistError, NodeNotChangedError, \
     RepositoryError
 
 
 class BaseRepository(object):
     """
     Base Repository for final backends
 
     **Attributes**
 
         ``DEFAULT_BRANCH_NAME``
             name of default branch (i.e. "trunk" for svn, "master" for git etc.
 
         ``scm``
             alias of scm, i.e. *git* or *hg*
 
         ``repo``
             object from external api
 
         ``revisions``
             list of all available revisions' ids, in ascending order
 
         ``changesets``
             storage dict caching returned changesets
 
         ``path``
             absolute path to the repository
 
         ``branches``
             branches as list of changesets
 
         ``tags``
             tags as list of changesets
     """
     scm = None
     DEFAULT_BRANCH_NAME = None
     EMPTY_CHANGESET = '0' * 40
 
     def __init__(self, repo_path, create=False, **kwargs):
         """
         Initializes repository. Raises RepositoryError if repository could
         not be find at the given ``repo_path`` or directory at ``repo_path``
         exists and ``create`` is set to True.
 
         :param repo_path: local path of the repository
         :param create=False: if set to True, would try to craete repository.
         :param src_url=None: if set, should be proper url from which repository
           would be cloned; requires ``create`` parameter to be set to True -
           raises RepositoryError if src_url is set and create evaluates to
           False
         """
         raise NotImplementedError
 
     def __str__(self):
         return '<%s at %s>' % (self.__class__.__name__, self.path)
 
     def __repr__(self):
         return self.__str__()
 
     def __len__(self):
         return self.count()
 
     def __eq__(self, other):
         same_instance = isinstance(other, self.__class__)
         return same_instance and getattr(other, 'path', None) == self.path
 
     def __ne__(self, other):
         return not self.__eq__(other)
 
     @LazyProperty
     def alias(self):
         for k, v in settings.BACKENDS.items():
             if v.split('.')[-1] == str(self.__class__.__name__):
                 return k
 
     @LazyProperty
     def name(self):
         raise NotImplementedError
 
     @LazyProperty
     def owner(self):
         raise NotImplementedError
 
     @LazyProperty
     def description(self):
         raise NotImplementedError
 
     @LazyProperty
     def size(self):
         """
         Returns combined size in bytes for all repository files
         """
 
         size = 0
         try:
             tip = self.get_changeset()
             for topnode, dirs, files in tip.walk('/'):
                 for f in files:
                     size += tip.get_file_size(f.path)
                 for dir in dirs:
                     for f in files:
                         size += tip.get_file_size(f.path)
 
         except RepositoryError, e:
             pass
         return size
 
     def is_valid(self):
         """
         Validates repository.
         """
         raise NotImplementedError
 
     def get_last_change(self):
         self.get_changesets()
 
     #==========================================================================
     # CHANGESETS
     #==========================================================================
 
     def get_changeset(self, revision=None):
         """
         Returns instance of ``Changeset`` class. If ``revision`` is None, most
         recent changeset is returned.
 
         :raises ``EmptyRepositoryError``: if there are no revisions
         """
         raise NotImplementedError
 
     def __iter__(self):
         """
         Allows Repository objects to be iterated.
 
         *Requires* implementation of ``__getitem__`` method.
         """
         for revision in self.revisions:
             yield self.get_changeset(revision)
 
     def get_changesets(self, start=None, end=None, start_date=None,
                        end_date=None, branch_name=None, reverse=False):
         """
         Returns iterator of ``MercurialChangeset`` objects from start to end
         not inclusive This should behave just like a list, ie. end is not
         inclusive
 
         :param start: None or str
         :param end: None or str
         :param start_date:
         :param end_date:
         :param branch_name:
         :param reversed:
         """
         raise NotImplementedError
 
     def __getslice__(self, i, j):
         """
         Returns a iterator of sliced repository
         """
         for rev in self.revisions[i:j]:
             yield self.get_changeset(rev)
 
     def __getitem__(self, key):
         return self.get_changeset(key)
 
     def count(self):
         return len(self.revisions)
 
     def tag(self, name, user, revision=None, message=None, date=None, **opts):
         """
         Creates and returns a tag for the given ``revision``.
 
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param revision: changeset id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
 
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         raise NotImplementedError
 
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given ``name``.
 
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
 
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         raise NotImplementedError
 
     def get_diff(self, rev1, rev2, path=None, ignore_whitespace=False,
             context=3):
         """
         Returns (git like) *diff*, as plain text. Shows changes introduced by
         ``rev2`` since ``rev1``.
 
         :param rev1: Entry point from which diff is shown. Can be
           ``self.EMPTY_CHANGESET`` - in this case, patch showing all
           the changes since empty state of the repository until ``rev2``
         :param rev2: Until which revision changes should be shown.
         :param ignore_whitespace: If set to ``True``, would not show whitespace
           changes. Defaults to ``False``.
         :param context: How many lines before/after changed lines should be
           shown. Defaults to ``3``.
         """
         raise NotImplementedError
 
     # ========== #
     # COMMIT API #
     # ========== #
 
     @LazyProperty
     def in_memory_changeset(self):
         """
         Returns ``InMemoryChangeset`` object for this repository.
         """
         raise NotImplementedError
 
     def add(self, filenode, **kwargs):
         """
         Commit api function that will add given ``FileNode`` into this
         repository.
 
         :raises ``NodeAlreadyExistsError``: if there is a file with same path
           already in repository
         :raises ``NodeAlreadyAddedError``: if given node is already marked as
           *added*
         """
         raise NotImplementedError
 
     def remove(self, filenode, **kwargs):
         """
         Commit api function that will remove given ``FileNode`` into this
         repository.
 
         :raises ``EmptyRepositoryError``: if there are no changesets yet
         :raises ``NodeDoesNotExistError``: if there is no file with given path
         """
         raise NotImplementedError
 
     def commit(self, message, **kwargs):
         """
         Persists current changes made on this repository and returns newly
         created changeset.
 
         :raises ``NothingChangedError``: if no changes has been made
         """
         raise NotImplementedError
 
     def get_state(self):
         """
         Returns dictionary with ``added``, ``changed`` and ``removed`` lists
         containing ``FileNode`` objects.
         """
         raise NotImplementedError
 
     def get_config_value(self, section, name, config_file=None):
         """
         Returns configuration value for a given [``section``] and ``name``.
 
         :param section: Section we want to retrieve value from
         :param name: Name of configuration we want to retrieve
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         raise NotImplementedError
 
     def get_user_name(self, config_file=None):
         """
         Returns user's name from global configuration file.
 
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         raise NotImplementedError
 
     def get_user_email(self, config_file=None):
         """
         Returns user's email from global configuration file.
 
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         raise NotImplementedError
 
     # =========== #
     # WORKDIR API #
     # =========== #
 
     @LazyProperty
     def workdir(self):
         """
         Returns ``Workdir`` instance for this repository.
         """
         raise NotImplementedError
 
 
 class BaseChangeset(object):
     """
     Each backend should implement it's changeset representation.
 
     **Attributes**
 
         ``repository``
             repository object within which changeset exists
 
         ``id``
             may be ``raw_id`` or i.e. for mercurial's tip just ``tip``
 
         ``raw_id``
             raw changeset representation (i.e. full 40 length sha for git
             backend)
 
         ``short_id``
             shortened (if apply) version of ``raw_id``; it would be simple
             shortcut for ``raw_id[:12]`` for git/mercurial backends or same
             as ``raw_id`` for subversion
 
         ``revision``
             revision number as integer
 
         ``files``
             list of ``FileNode`` (``Node`` with NodeKind.FILE) objects
 
         ``dirs``
             list of ``DirNode`` (``Node`` with NodeKind.DIR) objects
 
         ``nodes``
             combined list of ``Node`` objects
 
         ``author``
             author of the changeset, as unicode
 
         ``message``
             message of the changeset, as unicode
 
         ``parents``
             list of parent changesets
 
         ``last``
             ``True`` if this is last changeset in repository, ``False``
             otherwise; trying to access this attribute while there is no
             changesets would raise ``EmptyRepositoryError``
     """
     def __str__(self):
         return '<%s at %s:%s>' % (self.__class__.__name__, self.revision,
             self.short_id)
 
     def __repr__(self):
         return self.__str__()
 
     def __unicode__(self):
         return u'%s:%s' % (self.revision, self.short_id)
 
     def __eq__(self, other):
         return self.raw_id == other.raw_id
 
     def __json__(self):
         return dict(
             short_id=self.short_id,
             raw_id=self.raw_id,
             revision=self.revision,
             message=self.message,
             date=self.date,
             author=self.author,
         )
 
     @LazyProperty
     def last(self):
         if self.repository is None:
             raise ChangesetError("Cannot check if it's most recent revision")
         return self.raw_id == self.repository.revisions[-1]
 
     @LazyProperty
     def parents(self):
         """
         Returns list of parents changesets.
         """
         raise NotImplementedError
 
     @LazyProperty
     def children(self):
         """
         Returns list of children changesets.
         """
         raise NotImplementedError
 
     @LazyProperty
     def id(self):
         """
         Returns string identifying this changeset.
         """
         raise NotImplementedError
 
     @LazyProperty
     def raw_id(self):
         """
         Returns raw string identifying this changeset.
         """
         raise NotImplementedError
 
     @LazyProperty
     def short_id(self):
         """
         Returns shortened version of ``raw_id`` attribute, as string,
         identifying this changeset, useful for web representation.
         """
         raise NotImplementedError
 
     @LazyProperty
     def revision(self):
         """
         Returns integer identifying this changeset.
 
         """
         raise NotImplementedError
 
     @LazyProperty
     def committer(self):
         """
         Returns Committer for given commit
         """
 
         raise NotImplementedError
 
     @LazyProperty
     def committer_name(self):
         """
         Returns Author name for given commit
         """
 
         return author_name(self.committer)
 
     @LazyProperty
     def committer_email(self):
         """
         Returns Author email address for given commit
         """
 
         return author_email(self.committer)
 
     @LazyProperty
     def author(self):
         """
         Returns Author for given commit
         """
 
         raise NotImplementedError
 
     @LazyProperty
     def author_name(self):
         """
         Returns Author name for given commit
         """
 
         return author_name(self.author)
 
     @LazyProperty
     def author_email(self):
         """
         Returns Author email address for given commit
         """
 
         return author_email(self.author)
 
     def get_file_mode(self, path):
         """
         Returns stat mode of the file at the given ``path``.
         """
         raise NotImplementedError
 
     def get_file_content(self, path):
         """
         Returns content of the file at the given ``path``.
         """
         raise NotImplementedError
 
     def get_file_size(self, path):
         """
         Returns size of the file at the given ``path``.
         """
         raise NotImplementedError
 
     def get_file_changeset(self, path):
         """
         Returns last commit of the file at the given ``path``.
         """
         raise NotImplementedError
 
     def get_file_history(self, path):
         """
         Returns history of file as reversed list of ``Changeset`` objects for
         which file at given ``path`` has been modified.
         """
         raise NotImplementedError
 
     def get_nodes(self, path):
         """
         Returns combined ``DirNode`` and ``FileNode`` objects list representing
         state of changeset at the given ``path``.
 
         :raises ``ChangesetError``: if node at the given ``path`` is not
           instance of ``DirNode``
         """
         raise NotImplementedError
 
     def get_node(self, path):
         """
         Returns ``Node`` object from the given ``path``.
 
         :raises ``NodeDoesNotExistError``: if there is no node at the given
           ``path``
         """
         raise NotImplementedError
 
     def fill_archive(self, stream=None, kind='tgz', prefix=None):
         """
         Fills up given stream.
 
         :param stream: file like object.
         :param kind: one of following: ``zip``, ``tar``, ``tgz``
             or ``tbz2``. Default: ``tgz``.
         :param prefix: name of root directory in archive.
             Default is repository name and changeset's raw_id joined with dash.
 
             repo-tip.<kind>
         """
 
         raise NotImplementedError
 
     def get_chunked_archive(self, **kwargs):
         """
         Returns iterable archive. Tiny wrapper around ``fill_archive`` method.
 
         :param chunk_size: extra parameter which controls size of returned
             chunks. Default:8k.
         """
 
         chunk_size = kwargs.pop('chunk_size', 8192)
         stream = kwargs.get('stream')
         self.fill_archive(**kwargs)
         while True:
             data = stream.read(chunk_size)
             if not data:
                 break
             yield data
 
     @LazyProperty
     def root(self):
         """
         Returns ``RootNode`` object for this changeset.
         """
         return self.get_node('')
 
     def next(self, branch=None):
         """
         Returns next changeset from current, if branch is gives it will return
         next changeset belonging to this branch
 
         :param branch: show changesets within the given named branch
         """
         raise NotImplementedError
 
     def prev(self, branch=None):
         """
         Returns previous changeset from current, if branch is gives it will
         return previous changeset belonging to this branch
 
         :param branch: show changesets within the given named branch
         """
         raise NotImplementedError
 
     @LazyProperty
     def added(self):
         """
         Returns list of added ``FileNode`` objects.
         """
         raise NotImplementedError
 
     @LazyProperty
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
         raise NotImplementedError
 
     @LazyProperty
     def removed(self):
         """
         Returns list of removed ``FileNode`` objects.
         """
         raise NotImplementedError
 
     @LazyProperty
     def size(self):
         """
         Returns total number of bytes from contents of all filenodes.
         """
         return sum((node.size for node in self.get_filenodes_generator()))
 
     def walk(self, topurl=''):
         """
         Similar to os.walk method. Insted of filesystem it walks through
         changeset starting at given ``topurl``.  Returns generator of tuples
         (topnode, dirnodes, filenodes).
         """
         topnode = self.get_node(topurl)
         yield (topnode, topnode.dirs, topnode.files)
         for dirnode in topnode.dirs:
             for tup in self.walk(dirnode.path):
                 yield tup
 
     def get_filenodes_generator(self):
         """
         Returns generator that yields *all* file nodes.
         """
         for topnode, dirs, files in self.walk():
             for node in files:
                 yield node
 
     def as_dict(self):
         """
         Returns dictionary with changeset's attributes and their values.
         """
         data = get_dict_for_attrs(self, ['id', 'raw_id', 'short_id',
             'revision', 'date', 'message'])
         data['author'] = {'name': self.author_name, 'email': self.author_email}
         data['added'] = [node.path for node in self.added]
         data['changed'] = [node.path for node in self.changed]
         data['removed'] = [node.path for node in self.removed]
         return data
 
 
 class BaseWorkdir(object):
     """
     Working directory representation of single repository.
 
     :attribute: repository: repository object of working directory
     """
 
     def __init__(self, repository):
         self.repository = repository
 
     def get_branch(self):
         """
         Returns name of current branch.
         """
         raise NotImplementedError
 
     def get_changeset(self):
         """
         Returns current changeset.
         """
         raise NotImplementedError
 
     def get_added(self):
         """
         Returns list of ``FileNode`` objects marked as *new* in working
         directory.
         """
         raise NotImplementedError
 
     def get_changed(self):
         """
         Returns list of ``FileNode`` objects *changed* in working directory.
         """
         raise NotImplementedError
 
     def get_removed(self):
         """
         Returns list of ``RemovedFileNode`` objects marked as *removed* in
         working directory.
         """
         raise NotImplementedError
 
     def get_untracked(self):
         """
         Returns list of ``FileNode`` objects which are present within working
         directory however are not tracked by repository.
         """
         raise NotImplementedError
 
     def get_status(self):
         """
         Returns dict with ``added``, ``changed``, ``removed`` and ``untracked``
         lists.
         """
         raise NotImplementedError
 
     def commit(self, message, **kwargs):
         """
         Commits local (from working directory) changes and returns newly
         created
         ``Changeset``. Updates repository's ``revisions`` list.
 
         :raises ``CommitError``: if any error occurs while committing
         """
         raise NotImplementedError
 
     def update(self, revision=None):
         """
         Fetches content of the given revision and populates it within working
         directory.
         """
         raise NotImplementedError
 
     def checkout_branch(self, branch=None):
         """
         Checks out ``branch`` or the backend's default branch.
 
         Raises ``BranchDoesNotExistError`` if the branch does not exist.
         """
         raise NotImplementedError
 
 
 class BaseInMemoryChangeset(object):
     """
     Represents differences between repository's state (most recent head) and
     changes made *in place*.
 
     **Attributes**
 
         ``repository``
             repository object for this in-memory-changeset
 
         ``added``
             list of ``FileNode`` objects marked as *added*
 
         ``changed``
             list of ``FileNode`` objects marked as *changed*
 
         ``removed``
             list of ``FileNode`` or ``RemovedFileNode`` objects marked to be
             *removed*
 
         ``parents``
             list of ``Changeset`` representing parents of in-memory changeset.
             Should always be 2-element sequence.
 
     """
 
     def __init__(self, repository):
         self.repository = repository
         self.added = []
         self.changed = []
         self.removed = []
         self.parents = []
 
     def add(self, *filenodes):
         """
         Marks given ``FileNode`` objects as *to be committed*.
 
         :raises ``NodeAlreadyExistsError``: if node with same path exists at
           latest changeset
         :raises ``NodeAlreadyAddedError``: if node with same path is already
           marked as *added*
         """
         # Check if not already marked as *added* first
         for node in filenodes:
             if node.path in (n.path for n in self.added):
                 raise NodeAlreadyAddedError("Such FileNode %s is already "
                     "marked for addition" % node.path)
         for node in filenodes:
             self.added.append(node)
 
     def change(self, *filenodes):
         """
         Marks given ``FileNode`` objects to be *changed* in next commit.
 
         :raises ``EmptyRepositoryError``: if there are no changesets yet
         :raises ``NodeAlreadyExistsError``: if node with same path is already
           marked to be *changed*
         :raises ``NodeAlreadyRemovedError``: if node with same path is already
           marked to be *removed*
         :raises ``NodeDoesNotExistError``: if node doesn't exist in latest
           changeset
         :raises ``NodeNotChangedError``: if node hasn't really be changed
         """
         for node in filenodes:
             if node.path in (n.path for n in self.removed):
                 raise NodeAlreadyRemovedError("Node at %s is already marked "
                     "as removed" % node.path)
         try:
             self.repository.get_changeset()
         except EmptyRepositoryError:
             raise EmptyRepositoryError("Nothing to change - try to *add* new "
                 "nodes rather than changing them")
         for node in filenodes:
             if node.path in (n.path for n in self.changed):
                 raise NodeAlreadyChangedError("Node at '%s' is already "
                     "marked as changed" % node.path)
             self.changed.append(node)
 
     def remove(self, *filenodes):
         """
         Marks given ``FileNode`` (or ``RemovedFileNode``) objects to be
         *removed* in next commit.
 
         :raises ``NodeAlreadyRemovedError``: if node has been already marked to
           be *removed*
         :raises ``NodeAlreadyChangedError``: if node has been already marked to
           be *changed*
         """
         for node in filenodes:
             if node.path in (n.path for n in self.removed):
                 raise NodeAlreadyRemovedError("Node is already marked to "
                     "for removal at %s" % node.path)
             if node.path in (n.path for n in self.changed):
                 raise NodeAlreadyChangedError("Node is already marked to "
                     "be changed at %s" % node.path)
             # We only mark node as *removed* - real removal is done by
             # commit method
             self.removed.append(node)
 
     def reset(self):
         """
         Resets this instance to initial state (cleans ``added``, ``changed``
         and ``removed`` lists).
         """
         self.added = []
         self.changed = []
         self.removed = []
         self.parents = []
 
     def get_ipaths(self):
         """
         Returns generator of paths from nodes marked as added, changed or
         removed.
         """
         for node in chain(self.added, self.changed, self.removed):
             yield node.path
 
     def get_paths(self):
         """
         Returns list of paths from nodes marked as added, changed or removed.
         """
         return list(self.get_ipaths())
 
     def check_integrity(self, parents=None):
         """
         Checks in-memory changeset's integrity. Also, sets parents if not
         already set.
 
         :raises CommitError: if any error occurs (i.e.
           ``NodeDoesNotExistError``).
         """
         if not self.parents:
             parents = parents or []
             if len(parents) == 0:
                 try:
                     parents = [self.repository.get_changeset(), None]
                 except EmptyRepositoryError:
                     parents = [None, None]
             elif len(parents) == 1:
                 parents += [None]
             self.parents = parents
 
         # Local parents, only if not None
         parents = [p for p in self.parents if p]
 
         # Check nodes marked as added
         for p in parents:
             for node in self.added:
                 try:
                     p.get_node(node.path)
                 except NodeDoesNotExistError:
                     pass
                 else:
                     raise NodeAlreadyExistsError("Node at %s already exists "
                         "at %s" % (node.path, p))
 
         # Check nodes marked as changed
         missing = set(self.changed)
         not_changed = set(self.changed)
         if self.changed and not parents:
             raise NodeDoesNotExistError(str(self.changed[0].path))
         for p in parents:
             for node in self.changed:
                 try:
                     old = p.get_node(node.path)
                     missing.remove(node)
                     if old.content != node.content:
                         not_changed.remove(node)
                 except NodeDoesNotExistError:
                     pass
         if self.changed and missing:
             raise NodeDoesNotExistError("Node at %s is missing "
                 "(parents: %s)" % (node.path, parents))
 
         if self.changed and not_changed:
             raise NodeNotChangedError("Node at %s wasn't actually changed "
                 "since parents' changesets: %s" % (not_changed.pop().path,
                     parents)
             )
 
         # Check nodes marked as removed
         if self.removed and not parents:
             raise NodeDoesNotExistError("Cannot remove node at %s as there "
                 "were no parents specified" % self.removed[0].path)
         really_removed = set()
         for p in parents:
             for node in self.removed:
                 try:
                     p.get_node(node.path)
                     really_removed.add(node)
                 except ChangesetError:
                     pass
         not_removed = set(self.removed) - really_removed
         if not_removed:
             raise NodeDoesNotExistError("Cannot remove node at %s from "
                 "following parents: %s" % (not_removed[0], parents))
 
     def commit(self, message, author, parents=None, branch=None, date=None,
             **kwargs):
         """
         Performs in-memory commit (doesn't check workdir in any way) and
         returns newly created ``Changeset``. Updates repository's
         ``revisions``.
 
         .. note::
             While overriding this method each backend's should call
             ``self.check_integrity(parents)`` in the first place.
 
         :param message: message of the commit
         :param author: full username, i.e. "Joe Doe <[e-mail]>"
         :param parents: single parent or sequence of parents from which commit
           would be derieved
         :param date: ``datetime.datetime`` instance. Defaults to
           ``datetime.datetime.now()``.
         :param branch: branch name, as string. If none given, default backend's
           branch would be used.
 
         :raises ``CommitError``: if any error occurs while committing
         """
         raise NotImplementedError
 
 
 class EmptyChangeset(BaseChangeset):
     """
     An dummy empty changeset. It's possible to pass hash when creating
     an EmptyChangeset
     """
 
     def __init__(self, cs='0' * 40, repo=None, requested_revision=None,
                  alias=None, revision=-1, message='', author='', date=None):
         self._empty_cs = cs
         self.revision = revision
         self.message = message
         self.author = author
         self.date = date or datetime.datetime.fromtimestamp(0)
         self.repository = repo
         self.requested_revision = requested_revision
         self.alias = alias
 
     @LazyProperty
     def raw_id(self):
         """
         Returns raw string identifying this changeset, useful for web
         representation.
         """
 
         return self._empty_cs
 
     @LazyProperty
     def branch(self):
         from rhodecode.lib.vcs.backends import get_backend
         return get_backend(self.alias).DEFAULT_BRANCH_NAME
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     def get_file_changeset(self, path):
         return self
 
     def get_file_content(self, path):
         return u''
 
     def get_file_size(self, path):
         return 0
+
+
+class CollectionGenerator(object):
+
+    def __init__(self, repo, revs):
+        self.repo = repo
+        self.revs = revs
+
+    def __len__(self):
+        return len(self.revs)
+
+    def __iter__(self):
+        for rev in self.revs:
+            yield self.repo.get_changeset(rev)
+
+    def __getslice__(self, i, j):
+        """
+        Returns a iterator of sliced repository
+        """
+        sliced_revs = self.revs[i:j]
+        return CollectionGenerator(self.repo, sliced_revs)
+
+    def __repr__(self):
+        return 'CollectionGenerator<%s>' % (len(self))

rhodecode/lib/vcs/backends/git/repository.py

@@ -1,693 +1,692
 # -*- coding: utf-8 -*-
 """
     vcs.backends.git
     ~~~~~~~~~~~~~~~~
 
     Git backend implementation.
 
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 
 import os
 import re
 import time
 import posixpath
 import logging
 import traceback
 import urllib
 import urllib2
 from dulwich.repo import Repo, NotGitRepository
 from dulwich.objects import Tag
 from string import Template
 
 import rhodecode
-from rhodecode.lib.vcs.backends.base import BaseRepository
+from rhodecode.lib.vcs.backends.base import BaseRepository, CollectionGenerator
 from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError
 from rhodecode.lib.vcs.exceptions import ChangesetDoesNotExistError
 from rhodecode.lib.vcs.exceptions import EmptyRepositoryError
 from rhodecode.lib.vcs.exceptions import RepositoryError
 from rhodecode.lib.vcs.exceptions import TagAlreadyExistError
 from rhodecode.lib.vcs.exceptions import TagDoesNotExistError
 from rhodecode.lib.vcs.utils import safe_unicode, makedate, date_fromtimestamp
 from rhodecode.lib.vcs.utils.lazy import LazyProperty, ThreadLocalLazyProperty
 from rhodecode.lib.vcs.utils.ordered_dict import OrderedDict
 from rhodecode.lib.vcs.utils.paths import abspath
 from rhodecode.lib.vcs.utils.paths import get_user_home
 from .workdir import GitWorkdir
 from .changeset import GitChangeset
 from .inmemory import GitInMemoryChangeset
 from .config import ConfigFile
 from rhodecode.lib import subprocessio
 
 
 log = logging.getLogger(__name__)
 
 
 class GitRepository(BaseRepository):
     """
     Git repository backend.
     """
     DEFAULT_BRANCH_NAME = 'master'
     scm = 'git'
 
     def __init__(self, repo_path, create=False, src_url=None,
                  update_after_clone=False, bare=False):
 
         self.path = abspath(repo_path)
         repo = self._get_repo(create, src_url, update_after_clone, bare)
         self.bare = repo.bare
 
         self._config_files = [
             bare and abspath(self.path, 'config')
                      or abspath(self.path, '.git', 'config'),
             abspath(get_user_home(), '.gitconfig'),
         ]
 
     @property
     def _repo(self):
         return Repo(self.path)
 
     @property
     def head(self):
         try:
             return self._repo.head()
         except KeyError:
             return None
 
     @LazyProperty
     def revisions(self):
         """
         Returns list of revisions' ids, in ascending order.  Being lazy
         attribute allows external tools to inject shas from cache.
         """
         return self._get_all_revisions()
 
     @classmethod
     def _run_git_command(cls, cmd, **opts):
         """
         Runs given ``cmd`` as git command and returns tuple
         (stdout, stderr).
 
         :param cmd: git command to be executed
         :param opts: env options to pass into Subprocess command
         """
 
         if '_bare' in opts:
             _copts = []
             del opts['_bare']
         else:
             _copts = ['-c', 'core.quotepath=false', ]
         safe_call = False
         if '_safe' in opts:
             #no exc on failure
             del opts['_safe']
             safe_call = True
 
         _str_cmd = False
         if isinstance(cmd, basestring):
             cmd = [cmd]
             _str_cmd = True
 
         gitenv = os.environ
         # need to clean fix GIT_DIR !
         if 'GIT_DIR' in gitenv:
             del gitenv['GIT_DIR']
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
 
         _git_path = rhodecode.CONFIG.get('git_path', 'git')
         cmd = [_git_path] + _copts + cmd
         if _str_cmd:
             cmd = ' '.join(cmd)
         try:
             _opts = dict(
                 env=gitenv,
                 shell=False,
             )
             _opts.update(opts)
             p = subprocessio.SubprocessIOChunker(cmd, **_opts)
         except (EnvironmentError, OSError), err:
             tb_err = ("Couldn't run git command (%s).\n"
                       "Original error was:%s\n" % (cmd, err))
             log.error(tb_err)
             if safe_call:
                 return '', err
             else:
                 raise RepositoryError(tb_err)
 
         return ''.join(p.output), ''.join(p.error)
 
     def run_git_command(self, cmd):
         opts = {}
         if os.path.isdir(self.path):
             opts['cwd'] = self.path
         return self._run_git_command(cmd, **opts)
 
     @classmethod
     def _check_url(cls, url):
         """
         Functon will check given url and try to verify if it's a valid
         link. Sometimes it may happened that mercurial will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
 
         On failures it'll raise urllib2.HTTPError
         """
         from mercurial.util import url as Url
 
         # those authnadlers are patched for python 2.6.5 bug an
         # infinit looping when given invalid resources
         from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
 
         # check first if it's not an local url
         if os.path.isdir(url) or url.startswith('file:'):
             return True
 
         if('+' in url[:url.find('://')]):
             url = url[url.find('+') + 1:]
 
         handlers = []
         test_uri, authinfo = Url(url).authinfo()
         if not test_uri.endswith('info/refs'):
             test_uri = test_uri.rstrip('/') + '/info/refs'
         if authinfo:
             #create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         o = urllib2.build_opener(*handlers)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
 
         q = {"service": 'git-upload-pack'}
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             resp = o.open(req)
             return resp.code == 200
         except Exception, e:
             # means it cannot be cloned
             raise urllib2.URLError("[%s] %s" % (url, e))
 
     def _get_repo(self, create, src_url=None, update_after_clone=False,
                   bare=False):
         if create and os.path.exists(self.path):
             raise RepositoryError("Location already exist")
         if src_url and not create:
             raise RepositoryError("Create should be set to True if src_url is "
                                   "given (clone operation creates repository)")
         try:
             if create and src_url:
                 GitRepository._check_url(src_url)
                 self.clone(src_url, update_after_clone, bare)
                 return Repo(self.path)
             elif create:
                 os.mkdir(self.path)
                 if bare:
                     return Repo.init_bare(self.path)
                 else:
                     return Repo.init(self.path)
             else:
                 return self._repo
         except (NotGitRepository, OSError), err:
             raise RepositoryError(err)
 
     def _get_all_revisions(self):
         # we must check if this repo is not empty, since later command
         # fails if it is. And it's cheaper to ask than throw the subprocess
         # errors
         try:
             self._repo.head()
         except KeyError:
             return []
         rev_filter = _git_path = rhodecode.CONFIG.get('git_rev_filter',
                                                       '--all').strip()
         cmd = 'rev-list %s --reverse --date-order' % (rev_filter)
         try:
             so, se = self.run_git_command(cmd)
         except RepositoryError:
             # Can be raised for empty repositories
             return []
         return so.splitlines()
 
     def _get_all_revisions2(self):
         #alternate implementation using dulwich
         includes = [x[1][0] for x in self._parsed_refs.iteritems()
                     if x[1][1] != 'T']
         return [c.commit.id for c in self._repo.get_walker(include=includes)]
 
     def _get_revision(self, revision):
         """
         For git backend we always return integer here. This way we ensure
         that changset's revision attribute would become integer.
         """
         pattern = re.compile(r'^[[0-9a-fA-F]{12}|[0-9a-fA-F]{40}]$')
         is_bstr = lambda o: isinstance(o, (str, unicode))
         is_null = lambda o: len(o) == revision.count('0')
 
         if len(self.revisions) == 0:
             raise EmptyRepositoryError("There are no changesets yet")
 
         if revision in (None, '', 'tip', 'HEAD', 'head', -1):
             revision = self.revisions[-1]
 
         if ((is_bstr(revision) and revision.isdigit() and len(revision) < 12)
             or isinstance(revision, int) or is_null(revision)):
             try:
                 revision = self.revisions[int(revision)]
             except Exception:
                 raise ChangesetDoesNotExistError("Revision %s does not exist "
                     "for this repository" % (revision))
 
         elif is_bstr(revision):
             # get by branch/tag name
             _ref_revision = self._parsed_refs.get(revision)
             _tags_shas = self.tags.values()
             if _ref_revision:  # and _ref_revision[1] in ['H', 'RH', 'T']:
                 return _ref_revision[0]
 
             # maybe it's a tag ? we don't have them in self.revisions
             elif revision in _tags_shas:
                 return _tags_shas[_tags_shas.index(revision)]
 
             elif not pattern.match(revision) or revision not in self.revisions:
                 raise ChangesetDoesNotExistError("Revision %s does not exist "
                     "for this repository" % (revision))
 
         # Ensure we return full id
         if not pattern.match(str(revision)):
             raise ChangesetDoesNotExistError("Given revision %s not recognized"
                 % revision)
         return revision
 
     def _get_archives(self, archive_name='tip'):
 
         for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]:
                 yield {"type": i[0], "extension": i[1], "node": archive_name}
 
     def _get_url(self, url):
         """
         Returns normalized url. If schema is not given, would fall to
         filesystem (``file:///``) schema.
         """
         url = str(url)
         if url != 'default' and not '://' in url:
             url = ':///'.join(('file', url))
         return url
 
     def get_hook_location(self):
         """
         returns absolute path to location where hooks are stored
         """
         loc = os.path.join(self.path, 'hooks')
         if not self.bare:
             loc = os.path.join(self.path, '.git', 'hooks')
         return loc
 
     @LazyProperty
     def name(self):
         return os.path.basename(self.path)
 
     @LazyProperty
     def last_change(self):
         """
         Returns last change made on this repository as datetime object
         """
         return date_fromtimestamp(self._get_mtime(), makedate()[1])
 
     def _get_mtime(self):
         try:
             return time.mktime(self.get_changeset().date.timetuple())
         except RepositoryError:
             idx_loc = '' if self.bare else '.git'
             # fallback to filesystem
             in_path = os.path.join(self.path, idx_loc, "index")
             he_path = os.path.join(self.path, idx_loc, "HEAD")
             if os.path.exists(in_path):
                 return os.stat(in_path).st_mtime
             else:
                 return os.stat(he_path).st_mtime
 
     @LazyProperty
     def description(self):
         idx_loc = '' if self.bare else '.git'
         undefined_description = u'unknown'
         description_path = os.path.join(self.path, idx_loc, 'description')
         if os.path.isfile(description_path):
             return safe_unicode(open(description_path).read())
         else:
             return undefined_description
 
     @LazyProperty
     def contact(self):
         undefined_contact = u'Unknown'
         return undefined_contact
 
     @property
     def branches(self):
         if not self.revisions:
             return {}
         sortkey = lambda ctx: ctx[0]
         _branches = [(x[0], x[1][0])
                      for x in self._parsed_refs.iteritems() if x[1][1] == 'H']
         return OrderedDict(sorted(_branches, key=sortkey, reverse=False))
 
     @LazyProperty
     def tags(self):
         return self._get_tags()
 
     def _get_tags(self):
         if not self.revisions:
             return {}
 
         sortkey = lambda ctx: ctx[0]
         _tags = [(x[0], x[1][0])
                  for x in self._parsed_refs.iteritems() if x[1][1] == 'T']
         return OrderedDict(sorted(_tags, key=sortkey, reverse=True))
 
     def tag(self, name, user, revision=None, message=None, date=None,
             **kwargs):
         """
         Creates and returns a tag for the given ``revision``.
 
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param revision: changeset id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
 
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         if name in self.tags:
             raise TagAlreadyExistError("Tag %s already exists" % name)
         changeset = self.get_changeset(revision)
         message = message or "Added tag %s for commit %s" % (name,
             changeset.raw_id)
         self._repo.refs["refs/tags/%s" % name] = changeset._commit.id
 
         self._parsed_refs = self._get_parsed_refs()
         self.tags = self._get_tags()
         return changeset
 
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given ``name``.
 
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
 
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
         tagpath = posixpath.join(self._repo.refs.path, 'refs', 'tags', name)
         try:
             os.remove(tagpath)
             self._parsed_refs = self._get_parsed_refs()
             self.tags = self._get_tags()
         except OSError, e:
             raise RepositoryError(e.strerror)
 
     @LazyProperty
     def _parsed_refs(self):
         return self._get_parsed_refs()
 
     def _get_parsed_refs(self):
         # cache the property
         _repo = self._repo
         refs = _repo.get_refs()
         keys = [('refs/heads/', 'H'),
                 ('refs/remotes/origin/', 'RH'),
                 ('refs/tags/', 'T')]
         _refs = {}
         for ref, sha in refs.iteritems():
             for k, type_ in keys:
                 if ref.startswith(k):
                     _key = ref[len(k):]
                     if type_ == 'T':
                         obj = _repo.get_object(sha)
                         if isinstance(obj, Tag):
                             sha = _repo.get_object(sha).object[1]
                     _refs[_key] = [sha, type_]
                     break
         return _refs
 
     def _heads(self, reverse=False):
         refs = self._repo.get_refs()
         heads = {}
 
         for key, val in refs.items():
             for ref_key in ['refs/heads/', 'refs/remotes/origin/']:
                 if key.startswith(ref_key):
                     n = key[len(ref_key):]
                     if n not in ['HEAD']:
                         heads[n] = val
 
         return heads if reverse else dict((y, x) for x, y in heads.iteritems())
 
     def get_changeset(self, revision=None):
         """
         Returns ``GitChangeset`` object representing commit from git repository
         at the given revision or head (most recent commit) if None given.
         """
         if isinstance(revision, GitChangeset):
             return revision
         revision = self._get_revision(revision)
         changeset = GitChangeset(repository=self, revision=revision)
         return changeset
 
     def get_changesets(self, start=None, end=None, start_date=None,
            end_date=None, branch_name=None, reverse=False):
         """
         Returns iterator of ``GitChangeset`` objects from start to end (both
         are inclusive), in ascending date order (unless ``reverse`` is set).
 
         :param start: changeset ID, as str; first returned changeset
         :param end: changeset ID, as str; last returned changeset
         :param start_date: if specified, changesets with commit date less than
           ``start_date`` would be filtered out from returned set
         :param end_date: if specified, changesets with commit date greater than
           ``end_date`` would be filtered out from returned set
         :param branch_name: if specified, changesets not reachable from given
           branch would be filtered out from returned set
         :param reverse: if ``True``, returned generator would be reversed
           (meaning that returned changesets would have descending date order)
 
         :raise BranchDoesNotExistError: If given ``branch_name`` does not
             exist.
         :raise ChangesetDoesNotExistError: If changeset for given ``start`` or
           ``end`` could not be found.
 
         """
         if branch_name and branch_name not in self.branches:
             raise BranchDoesNotExistError("Branch '%s' not found" \
                                           % branch_name)
         # %H at format means (full) commit hash, initial hashes are retrieved
         # in ascending date order
         cmd_template = 'log --date-order --reverse --pretty=format:"%H"'
         cmd_params = {}
         if start_date:
             cmd_template += ' --since "$since"'
             cmd_params['since'] = start_date.strftime('%m/%d/%y %H:%M:%S')
         if end_date:
             cmd_template += ' --until "$until"'
             cmd_params['until'] = end_date.strftime('%m/%d/%y %H:%M:%S')
         if branch_name:
             cmd_template += ' $branch_name'
             cmd_params['branch_name'] = branch_name
         else:
             rev_filter = _git_path = rhodecode.CONFIG.get('git_rev_filter',
                                                           '--all').strip()
             cmd_template += ' %s' % (rev_filter)
 
         cmd = Template(cmd_template).safe_substitute(**cmd_params)
         revs = self.run_git_command(cmd)[0].splitlines()
         start_pos = 0
         end_pos = len(revs)
         if start:
             _start = self._get_revision(start)
             try:
                 start_pos = revs.index(_start)
             except ValueError:
                 pass
 
         if end is not None:
             _end = self._get_revision(end)
             try:
                 end_pos = revs.index(_end)
             except ValueError:
                 pass
 
         if None not in [start, end] and start_pos > end_pos:
             raise RepositoryError('start cannot be after end')
 
         if end_pos is not None:
             end_pos += 1
 
         revs = revs[start_pos:end_pos]
         if reverse:
             revs = reversed(revs)
-        for rev in revs:
-            yield self.get_changeset(rev)
+        return CollectionGenerator(self, revs)
 
     def get_diff(self, rev1, rev2, path=None, ignore_whitespace=False,
                  context=3):
         """
         Returns (git like) *diff*, as plain text. Shows changes introduced by
         ``rev2`` since ``rev1``.
 
         :param rev1: Entry point from which diff is shown. Can be
           ``self.EMPTY_CHANGESET`` - in this case, patch showing all
           the changes since empty state of the repository until ``rev2``
         :param rev2: Until which revision changes should be shown.
         :param ignore_whitespace: If set to ``True``, would not show whitespace
           changes. Defaults to ``False``.
         :param context: How many lines before/after changed lines should be
           shown. Defaults to ``3``.
         """
         flags = ['-U%s' % context, '--full-index', '--binary', '-p', '-M', '--abbrev=40']
         if ignore_whitespace:
             flags.append('-w')
 
         if hasattr(rev1, 'raw_id'):
             rev1 = getattr(rev1, 'raw_id')
 
         if hasattr(rev2, 'raw_id'):
             rev2 = getattr(rev2, 'raw_id')
 
         if rev1 == self.EMPTY_CHANGESET:
             rev2 = self.get_changeset(rev2).raw_id
             cmd = ' '.join(['show'] + flags + [rev2])
         else:
             rev1 = self.get_changeset(rev1).raw_id
             rev2 = self.get_changeset(rev2).raw_id
             cmd = ' '.join(['diff'] + flags + [rev1, rev2])
 
         if path:
             cmd += ' -- "%s"' % path
 
         stdout, stderr = self.run_git_command(cmd)
         # If we used 'show' command, strip first few lines (until actual diff
         # starts)
         if rev1 == self.EMPTY_CHANGESET:
             lines = stdout.splitlines()
             x = 0
             for line in lines:
                 if line.startswith('diff'):
                     break
                 x += 1
             # Append new line just like 'diff' command do
             stdout = '\n'.join(lines[x:]) + '\n'
         return stdout
 
     @LazyProperty
     def in_memory_changeset(self):
         """
         Returns ``GitInMemoryChangeset`` object for this repository.
         """
         return GitInMemoryChangeset(self)
 
     def clone(self, url, update_after_clone=True, bare=False):
         """
         Tries to clone changes from external location.
 
         :param update_after_clone: If set to ``False``, git won't checkout
           working directory
         :param bare: If set to ``True``, repository would be cloned into
           *bare* git repository (no working directory at all).
         """
         url = self._get_url(url)
         cmd = ['clone']
         if bare:
             cmd.append('--bare')
         elif not update_after_clone:
             cmd.append('--no-checkout')
         cmd += ['--', '"%s"' % url, '"%s"' % self.path]
         cmd = ' '.join(cmd)
         # If error occurs run_git_command raises RepositoryError already
         self.run_git_command(cmd)
 
     def pull(self, url):
         """
         Tries to pull changes from external location.
         """
         url = self._get_url(url)
         cmd = ['pull']
         cmd.append("--ff-only")
         cmd.append(url)
         cmd = ' '.join(cmd)
         # If error occurs run_git_command raises RepositoryError already
         self.run_git_command(cmd)
 
     def fetch(self, url):
         """
         Tries to pull changes from external location.
         """
         url = self._get_url(url)
         so, se = self.run_git_command('ls-remote -h %s' % url)
         refs = []
         for line in (x for x in so.splitlines()):
             sha, ref = line.split('\t')
             refs.append(ref)
         refs = ' '.join(('+%s:%s' % (r, r) for r in refs))
         cmd = '''fetch %s -- %s''' % (url, refs)
         self.run_git_command(cmd)
 
     @LazyProperty
     def workdir(self):
         """
         Returns ``Workdir`` instance for this repository.
         """
         return GitWorkdir(self)
 
     def get_config_value(self, section, name, config_file=None):
         """
         Returns configuration value for a given [``section``] and ``name``.
 
         :param section: Section we want to retrieve value from
         :param name: Name of configuration we want to retrieve
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         if config_file is None:
             config_file = []
         elif isinstance(config_file, basestring):
             config_file = [config_file]
 
         def gen_configs():
             for path in config_file + self._config_files:
                 try:
                     yield ConfigFile.from_path(path)
                 except (IOError, OSError, ValueError):
                     continue
 
         for config in gen_configs():
             try:
                 return config.get(section, name)
             except KeyError:
                 continue
         return None
 
     def get_user_name(self, config_file=None):
         """
         Returns user's name from global configuration file.
 
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         return self.get_config_value('user', 'name', config_file)
 
     def get_user_email(self, config_file=None):
         """
         Returns user's email from global configuration file.
 
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         return self.get_config_value('user', 'email', config_file)

rhodecode/lib/vcs/backends/hg/repository.py

@@ -1,555 +1,553
 import os
 import time
 import datetime
 import urllib
 import urllib2
 
-from rhodecode.lib.vcs.backends.base import BaseRepository
+from rhodecode.lib.vcs.backends.base import BaseRepository, CollectionGenerator
 from .workdir import MercurialWorkdir
 from .changeset import MercurialChangeset
 from .inmemory import MercurialInMemoryChangeset
 
 from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError, \
     ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, \
     VCSError, TagAlreadyExistError, TagDoesNotExistError
 from rhodecode.lib.vcs.utils import author_email, author_name, date_fromtimestamp, \
     makedate, safe_unicode
 from rhodecode.lib.vcs.utils.lazy import LazyProperty
 from rhodecode.lib.vcs.utils.ordered_dict import OrderedDict
 from rhodecode.lib.vcs.utils.paths import abspath
 
 from rhodecode.lib.vcs.utils.hgcompat import ui, nullid, match, patch, \
     diffopts, clone, get_contact, pull, localrepository, RepoLookupError, \
     Abort, RepoError, hex, scmutil
 
 
 class MercurialRepository(BaseRepository):
     """
     Mercurial repository backend
     """
     DEFAULT_BRANCH_NAME = 'default'
     scm = 'hg'
 
     def __init__(self, repo_path, create=False, baseui=None, src_url=None,
                  update_after_clone=False):
         """
         Raises RepositoryError if repository could not be find at the given
         ``repo_path``.
 
         :param repo_path: local path of the repository
         :param create=False: if set to True, would try to create repository if
            it does not exist rather than raising exception
         :param baseui=None: user data
         :param src_url=None: would try to clone repository from given location
         :param update_after_clone=False: sets update of working copy after
           making a clone
         """
 
         if not isinstance(repo_path, str):
             raise VCSError('Mercurial backend requires repository path to '
                            'be instance of <str> got %s instead' %
                            type(repo_path))
 
         self.path = abspath(repo_path)
         self.baseui = baseui or ui.ui()
         # We've set path and ui, now we can set _repo itself
         self._repo = self._get_repo(create, src_url, update_after_clone)
 
     @property
     def _empty(self):
         """
         Checks if repository is empty without any changesets
         """
         # TODO: Following raises errors when using InMemoryChangeset...
         # return len(self._repo.changelog) == 0
         return len(self.revisions) == 0
 
     @LazyProperty
     def revisions(self):
         """
         Returns list of revisions' ids, in ascending order.  Being lazy
         attribute allows external tools to inject shas from cache.
         """
         return self._get_all_revisions()
 
     @LazyProperty
     def name(self):
         return os.path.basename(self.path)
 
     @LazyProperty
     def branches(self):
         return self._get_branches()
 
     @LazyProperty
     def allbranches(self):
         """
         List all branches, including closed branches.
         """
         return self._get_branches(closed=True)
 
     def _get_branches(self, closed=False):
         """
         Get's branches for this repository
         Returns only not closed branches by default
 
         :param closed: return also closed branches for mercurial
         """
 
         if self._empty:
             return {}
 
         def _branchtags(localrepo):
             """
             Patched version of mercurial branchtags to not return the closed
             branches
 
             :param localrepo: locarepository instance
             """
 
             bt = {}
             bt_closed = {}
             for bn, heads in localrepo.branchmap().iteritems():
                 tip = heads[-1]
                 if 'close' in localrepo.changelog.read(tip)[5]:
                     bt_closed[bn] = tip
                 else:
                     bt[bn] = tip
 
             if closed:
                 bt.update(bt_closed)
             return bt
 
         sortkey = lambda ctx: ctx[0]  # sort by name
         _branches = [(safe_unicode(n), hex(h),) for n, h in
                      _branchtags(self._repo).items()]
 
         return OrderedDict(sorted(_branches, key=sortkey, reverse=False))
 
     @LazyProperty
     def tags(self):
         """
         Get's tags for this repository
         """
         return self._get_tags()
 
     def _get_tags(self):
         if self._empty:
             return {}
 
         sortkey = lambda ctx: ctx[0]  # sort by name
         _tags = [(safe_unicode(n), hex(h),) for n, h in
                  self._repo.tags().items()]
 
         return OrderedDict(sorted(_tags, key=sortkey, reverse=True))
 
     def tag(self, name, user, revision=None, message=None, date=None,
             **kwargs):
         """
         Creates and returns a tag for the given ``revision``.
 
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param revision: changeset id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
 
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         if name in self.tags:
             raise TagAlreadyExistError("Tag %s already exists" % name)
         changeset = self.get_changeset(revision)
         local = kwargs.setdefault('local', False)
 
         if message is None:
             message = "Added tag %s for changeset %s" % (name,
                 changeset.short_id)
 
         if date is None:
             date = datetime.datetime.now().ctime()
 
         try:
             self._repo.tag(name, changeset._ctx.node(), message, local, user,
                 date)
         except Abort, e:
             raise RepositoryError(e.message)
 
         # Reinitialize tags
         self.tags = self._get_tags()
         tag_id = self.tags[name]
 
         return self.get_changeset(revision=tag_id)
 
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given ``name``.
 
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
 
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
         if message is None:
             message = "Removed tag %s" % name
         if date is None:
             date = datetime.datetime.now().ctime()
         local = False
 
         try:
             self._repo.tag(name, nullid, message, local, user, date)
             self.tags = self._get_tags()
         except Abort, e:
             raise RepositoryError(e.message)
 
     @LazyProperty
     def bookmarks(self):
         """
         Get's bookmarks for this repository
         """
         return self._get_bookmarks()
 
     def _get_bookmarks(self):
         if self._empty:
             return {}
 
         sortkey = lambda ctx: ctx[0]  # sort by name
         _bookmarks = [(safe_unicode(n), hex(h),) for n, h in
                  self._repo._bookmarks.items()]
         return OrderedDict(sorted(_bookmarks, key=sortkey, reverse=True))
 
     def _get_all_revisions(self):
 
         return map(lambda x: hex(x[7]), self._repo.changelog.index)[:-1]
 
     def get_diff(self, rev1, rev2, path='', ignore_whitespace=False,
                   context=3):
         """
         Returns (git like) *diff*, as plain text. Shows changes introduced by
         ``rev2`` since ``rev1``.
 
         :param rev1: Entry point from which diff is shown. Can be
           ``self.EMPTY_CHANGESET`` - in this case, patch showing all
           the changes since empty state of the repository until ``rev2``
         :param rev2: Until which revision changes should be shown.
         :param ignore_whitespace: If set to ``True``, would not show whitespace
           changes. Defaults to ``False``.
         :param context: How many lines before/after changed lines should be
           shown. Defaults to ``3``.
         """
         if hasattr(rev1, 'raw_id'):
             rev1 = getattr(rev1, 'raw_id')
 
         if hasattr(rev2, 'raw_id'):
             rev2 = getattr(rev2, 'raw_id')
 
         # Check if given revisions are present at repository (may raise
         # ChangesetDoesNotExistError)
         if rev1 != self.EMPTY_CHANGESET:
             self.get_changeset(rev1)
         self.get_changeset(rev2)
         if path:
             file_filter = match(self.path, '', [path])
         else:
             file_filter = None
 
         return ''.join(patch.diff(self._repo, rev1, rev2, match=file_filter,
                           opts=diffopts(git=True,
                                         ignorews=ignore_whitespace,
                                         context=context)))
 
     @classmethod
     def _check_url(cls, url):
         """
         Function will check given url and try to verify if it's a valid
         link. Sometimes it may happened that mercurial will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
 
         On failures it'll raise urllib2.HTTPError, return code 200 if url
         is valid or True if it's a local path
         """
 
         from mercurial.util import url as Url
 
         # those authnadlers are patched for python 2.6.5 bug an
         # infinit looping when given invalid resources
         from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
 
         # check first if it's not an local url
         if os.path.isdir(url) or url.startswith('file:'):
             return True
 
         if('+' in url[:url.find('://')]):
             url = url[url.find('+') + 1:]
 
         handlers = []
         test_uri, authinfo = Url(url).authinfo()
 
         if authinfo:
             #create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         o = urllib2.build_opener(*handlers)
         o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
                         ('Accept', 'application/mercurial-0.1')]
 
         q = {"cmd": 'between'}
         q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             resp = o.open(req)
             return resp.code == 200
         except Exception, e:
             # means it cannot be cloned
             raise urllib2.URLError("[%s] %s" % (url, e))
 
     def _get_repo(self, create, src_url=None, update_after_clone=False):
         """
         Function will check for mercurial repository in given path and return
         a localrepo object. If there is no repository in that path it will
         raise an exception unless ``create`` parameter is set to True - in
         that case repository would be created and returned.
         If ``src_url`` is given, would try to clone repository from the
         location at given clone_point. Additionally it'll make update to
         working copy accordingly to ``update_after_clone`` flag
         """
 
         try:
             if src_url:
                 url = str(self._get_url(src_url))
                 opts = {}
                 if not update_after_clone:
                     opts.update({'noupdate': True})
                 try:
                     MercurialRepository._check_url(url)
                     clone(self.baseui, url, self.path, **opts)
 #                except urllib2.URLError:
 #                    raise Abort("Got HTTP 404 error")
                 except Exception:
                     raise
 
                 # Don't try to create if we've already cloned repo
                 create = False
             return localrepository(self.baseui, self.path, create=create)
         except (Abort, RepoError), err:
             if create:
                 msg = "Cannot create repository at %s. Original error was %s"\
                     % (self.path, err)
             else:
                 msg = "Not valid repository at %s. Original error was %s"\
                     % (self.path, err)
             raise RepositoryError(msg)
 
     @LazyProperty
     def in_memory_changeset(self):
         return MercurialInMemoryChangeset(self)
 
     @LazyProperty
     def description(self):
         undefined_description = u'unknown'
         return safe_unicode(self._repo.ui.config('web', 'description',
                                    undefined_description, untrusted=True))
 
     @LazyProperty
     def contact(self):
         undefined_contact = u'Unknown'
         return safe_unicode(get_contact(self._repo.ui.config)
                             or undefined_contact)
 
     @LazyProperty
     def last_change(self):
         """
         Returns last change made on this repository as datetime object
         """
         return date_fromtimestamp(self._get_mtime(), makedate()[1])
 
     def _get_mtime(self):
         try:
             return time.mktime(self.get_changeset().date.timetuple())
         except RepositoryError:
             #fallback to filesystem
             cl_path = os.path.join(self.path, '.hg', "00changelog.i")
             st_path = os.path.join(self.path, '.hg', "store")
             if os.path.exists(cl_path):
                 return os.stat(cl_path).st_mtime
             else:
                 return os.stat(st_path).st_mtime
 
     def _get_hidden(self):
         return self._repo.ui.configbool("web", "hidden", untrusted=True)
 
     def _get_revision(self, revision):
         """
         Get's an ID revision given as str. This will always return a fill
         40 char revision number
 
         :param revision: str or int or None
         """
 
         if self._empty:
             raise EmptyRepositoryError("There are no changesets yet")
 
         if revision in [-1, 'tip', None]:
             revision = 'tip'
 
         try:
             revision = hex(self._repo.lookup(revision))
         except (IndexError, ValueError, RepoLookupError, TypeError):
             raise ChangesetDoesNotExistError("Revision %s does not "
                                     "exist for this repository"
                                     % (revision))
         return revision
 
     def _get_archives(self, archive_name='tip'):
         allowed = self.baseui.configlist("web", "allow_archive",
                                          untrusted=True)
         for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]:
             if i[0] in allowed or self._repo.ui.configbool("web",
                                                            "allow" + i[0],
                                                            untrusted=True):
                 yield {"type": i[0], "extension": i[1], "node": archive_name}
 
     def _get_url(self, url):
         """
         Returns normalized url. If schema is not given, would fall
         to filesystem
         (``file:///``) schema.
         """
         url = str(url)
         if url != 'default' and not '://' in url:
             url = "file:" + urllib.pathname2url(url)
         return url
 
     def get_hook_location(self):
         """
         returns absolute path to location where hooks are stored
         """
         return os.path.join(self.path, '.hg', '.hgrc')
 
     def get_changeset(self, revision=None):
         """
         Returns ``MercurialChangeset`` object representing repository's
         changeset at the given ``revision``.
         """
         revision = self._get_revision(revision)
         changeset = MercurialChangeset(repository=self, revision=revision)
         return changeset
 
     def get_changesets(self, start=None, end=None, start_date=None,
                        end_date=None, branch_name=None, reverse=False):
         """
         Returns iterator of ``MercurialChangeset`` objects from start to end
         (both are inclusive)
 
         :param start: None, str, int or mercurial lookup format
         :param end:  None, str, int or mercurial lookup format
         :param start_date:
         :param end_date:
         :param branch_name:
         :param reversed: return changesets in reversed order
         """
 
         start_raw_id = self._get_revision(start)
         start_pos = self.revisions.index(start_raw_id) if start else None
         end_raw_id = self._get_revision(end)
         end_pos = self.revisions.index(end_raw_id) if end else None
 
         if None not in [start, end] and start_pos > end_pos:
             raise RepositoryError("Start revision '%s' cannot be "
                                   "after end revision '%s'" % (start, end))
 
         if branch_name and branch_name not in self.allbranches.keys():
             raise BranchDoesNotExistError('Branch %s not found in'
                                   ' this repository' % branch_name)
         if end_pos is not None:
             end_pos += 1
         #filter branches
-
+        filter_ = []
         if branch_name:
-            revisions = scmutil.revrange(self._repo,
-                                         ['branch("%s")' % (branch_name)])
+            filter_.append('branch("%s")' % (branch_name))
+
+        if start_date:
+            filter_.append('date(">%s")' % start_date)
+        if end_date:
+            filter_.append('date("<%s")' % end_date)
+        if filter_:
+            revisions = scmutil.revrange(self._repo, filter_)
         else:
             revisions = self.revisions
-
-        slice_ = reversed(revisions[start_pos:end_pos]) if reverse else \
+        revs = reversed(revisions[start_pos:end_pos]) if reverse else \
                 revisions[start_pos:end_pos]
 
-        for id_ in slice_:
-            cs = self.get_changeset(id_)
-            if start_date and cs.date < start_date:
-                continue
-            if end_date and cs.date > end_date:
-                continue
-
-            yield cs
+        return CollectionGenerator(self, revs)
 
     def pull(self, url):
         """
         Tries to pull changes from external location.
         """
         url = self._get_url(url)
         try:
             pull(self.baseui, self._repo, url)
         except Abort, err:
             # Propagate error but with vcs's type
             raise RepositoryError(str(err))
 
     @LazyProperty
     def workdir(self):
         """
         Returns ``Workdir`` instance for this repository.
         """
         return MercurialWorkdir(self)
 
     def get_config_value(self, section, name=None, config_file=None):
         """
         Returns configuration value for a given [``section``] and ``name``.
 
         :param section: Section we want to retrieve value from
         :param name: Name of configuration we want to retrieve
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         if config_file is None:
             config_file = []
         elif isinstance(config_file, basestring):
             config_file = [config_file]
 
         config = self._repo.ui
         for path in config_file:
             config.readconfig(path)
         return config.config(section, name)
 
     def get_user_name(self, config_file=None):
         """
         Returns user's name from global configuration file.
 
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         username = self.get_config_value('ui', 'username')
         if username:
             return author_name(username)
         return None
 
     def get_user_email(self, config_file=None):
         """
         Returns user's email from global configuration file.
 
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         username = self.get_config_value('ui', 'username')
         if username:
             return author_email(username)
         return None