Added VCS into rhodecode core for faster and easier deployments of new versions

rhodecode/lib/vcs/__init__.py

@@ -0,0 +1,41 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs
+    ~~~
+
+    Various version Control System (vcs) management abstraction layer for
+    Python.
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+
+VERSION = (0, 2, 3, 'dev')
+
+__version__ = '.'.join((str(each) for each in VERSION[:4]))
+
+__all__ = [
+    'get_version', 'get_repo', 'get_backend',
+    'VCSError', 'RepositoryError', 'ChangesetError']
+
+import sys
+from rhodecode.lib.vcs.backends import get_repo, get_backend
+from rhodecode.lib.vcs.exceptions import VCSError, RepositoryError, ChangesetError
+
+
+def get_version():
+    """
+    Returns shorter version (digit parts only) as string.
+    """
+    return '.'.join((str(each) for each in VERSION[:3]))
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
+    from rhodecode.lib.vcs.cli import ExecutionManager
+    manager = ExecutionManager(argv)
+    manager.execute()
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

rhodecode/lib/vcs/backends/__init__.py

@@ -0,0 +1,63 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.backends
+    ~~~~~~~~~~~~
+
+    Main package for scm backends
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+import os
+from pprint import pformat
+from rhodecode.lib.vcs.conf import settings
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.utils.helpers import get_scm
+from rhodecode.lib.vcs.utils.paths import abspath
+from rhodecode.lib.vcs.utils.imports import import_class
+
+
+def get_repo(path=None, alias=None, create=False):
+    """
+    Returns ``Repository`` object of type linked with given ``alias`` at
+    the specified ``path``. If ``alias`` is not given it will try to guess it
+    using get_scm method
+    """
+    if create:
+        if not (path or alias):
+            raise TypeError("If create is specified, we need path and scm type")
+        return get_backend(alias)(path, create=True)
+    if path is None:
+        path = abspath(os.path.curdir)
+    try:
+        scm, path = get_scm(path, search_recursively=True)
+        path = abspath(path)
+        alias = scm
+    except VCSError:
+        raise VCSError("No scm found at %s" % path)
+    if alias is None:
+        alias = get_scm(path)[0]
+
+    backend = get_backend(alias)
+    repo = backend(path, create=create)
+    return repo
+
+
+def get_backend(alias):
+    """
+    Returns ``Repository`` class identified by the given alias or raises
+    VCSError if alias is not recognized or backend class cannot be imported.
+    """
+    if alias not in settings.BACKENDS:
+        raise VCSError("Given alias '%s' is not recognized! Allowed aliases:\n"
+            "%s" % (alias, pformat(settings.BACKENDS.keys())))
+    backend_path = settings.BACKENDS[alias]
+    klass = import_class(backend_path)
+    return klass
+
+
+def get_supported_backends():
+    """
+    Returns list of aliases of supported backends.
+    """
+    return settings.BACKENDS.keys()

rhodecode/lib/vcs/backends/base.py

@@ -0,0 +1,911 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.backends.base
+    ~~~~~~~~~~~~~~~~~
+
+    Base for all available scm backends
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+
+
+from itertools import chain
+from rhodecode.lib.vcs.utils import author_name, author_email
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.helpers import get_dict_for_attrs
+from rhodecode.lib.vcs.conf import settings
+
+from rhodecode.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, \
+    NodeAlreadyAddedError, NodeAlreadyChangedError, NodeAlreadyExistsError, \
+    NodeAlreadyRemovedError, NodeDoesNotExistError, NodeNotChangedError, \
+    RepositoryError
+
+
+class BaseRepository(object):
+    """
+    Base Repository for final backends
+
+    **Attributes**
+
+        ``DEFAULT_BRANCH_NAME``
+            name of default branch (i.e. "trunk" for svn, "master" for git etc.
+
+        ``scm``
+            alias of scm, i.e. *git* or *hg*
+
+        ``repo``
+            object from external api
+
+        ``revisions``
+            list of all available revisions' ids, in ascending order
+
+        ``changesets``
+            storage dict caching returned changesets
+
+        ``path``
+            absolute path to the repository
+
+        ``branches``
+            branches as list of changesets
+
+        ``tags``
+            tags as list of changesets
+    """
+    scm = None
+    DEFAULT_BRANCH_NAME = None
+    EMPTY_CHANGESET = '0' * 40
+
+    def __init__(self, repo_path, create=False, **kwargs):
+        """
+        Initializes repository. Raises RepositoryError if repository could
+        not be find at the given ``repo_path`` or directory at ``repo_path``
+        exists and ``create`` is set to True.
+
+        :param repo_path: local path of the repository
+        :param create=False: if set to True, would try to craete repository.
+        :param src_url=None: if set, should be proper url from which repository
+          would be cloned; requires ``create`` parameter to be set to True -
+          raises RepositoryError if src_url is set and create evaluates to
+          False
+        """
+        raise NotImplementedError
+
+    def __str__(self):
+        return '<%s at %s>' % (self.__class__.__name__, self.path)
+
+    def __repr__(self):
+        return self.__str__()
+
+    def __len__(self):
+        return self.count()
+
+    @LazyProperty
+    def alias(self):
+        for k, v in settings.BACKENDS.items():
+            if v.split('.')[-1] == str(self.__class__.__name__):
+                return k
+
+    @LazyProperty
+    def name(self):
+        raise NotImplementedError
+
+    @LazyProperty
+    def owner(self):
+        raise NotImplementedError
+
+    @LazyProperty
+    def description(self):
+        raise NotImplementedError
+
+    @LazyProperty
+    def size(self):
+        """
+        Returns combined size in bytes for all repository files
+        """
+
+        size = 0
+        try:
+            tip = self.get_changeset()
+            for topnode, dirs, files in tip.walk('/'):
+                for f in files:
+                    size += tip.get_file_size(f.path)
+                for dir in dirs:
+                    for f in files:
+                        size += tip.get_file_size(f.path)
+
+        except RepositoryError, e:
+            pass
+        return size
+
+    def is_valid(self):
+        """
+        Validates repository.
+        """
+        raise NotImplementedError
+
+    def get_last_change(self):
+        self.get_changesets()
+
+    #==========================================================================
+    # CHANGESETS
+    #==========================================================================
+
+    def get_changeset(self, revision=None):
+        """
+        Returns instance of ``Changeset`` class. If ``revision`` is None, most
+        recent changeset is returned.
+
+        :raises ``EmptyRepositoryError``: if there are no revisions
+        """
+        raise NotImplementedError
+
+    def __iter__(self):
+        """
+        Allows Repository objects to be iterated.
+
+        *Requires* implementation of ``__getitem__`` method.
+        """
+        for revision in self.revisions:
+            yield self.get_changeset(revision)
+
+    def get_changesets(self, start=None, end=None, start_date=None,
+                       end_date=None, branch_name=None, reverse=False):
+        """
+        Returns iterator of ``MercurialChangeset`` objects from start to end
+        not inclusive This should behave just like a list, ie. end is not
+        inclusive
+
+        :param start: None or str
+        :param end: None or str
+        :param start_date:
+        :param end_date:
+        :param branch_name:
+        :param reversed:
+        """
+        raise NotImplementedError
+
+    def __getslice__(self, i, j):
+        """
+        Returns a iterator of sliced repository
+        """
+        for rev in self.revisions[i:j]:
+            yield self.get_changeset(rev)
+
+    def __getitem__(self, key):
+        return self.get_changeset(key)
+
+    def count(self):
+        return len(self.revisions)
+
+    def tag(self, name, user, revision=None, message=None, date=None, **opts):
+        """
+        Creates and returns a tag for the given ``revision``.
+
+        :param name: name for new tag
+        :param user: full username, i.e.: "Joe Doe <[e-mail]>"
+        :param revision: changeset id for which new tag would be created
+        :param message: message of the tag's commit
+        :param date: date of tag's commit
+
+        :raises TagAlreadyExistError: if tag with same name already exists
+        """
+        raise NotImplementedError
+
+    def remove_tag(self, name, user, message=None, date=None):
+        """
+        Removes tag with the given ``name``.
+
+        :param name: name of the tag to be removed
+        :param user: full username, i.e.: "Joe Doe <[e-mail]>"
+        :param message: message of the tag's removal commit
+        :param date: date of tag's removal commit
+
+        :raises TagDoesNotExistError: if tag with given name does not exists
+        """
+        raise NotImplementedError
+
+    def get_diff(self, rev1, rev2, path=None, ignore_whitespace=False,
+            context=3):
+        """
+        Returns (git like) *diff*, as plain text. Shows changes introduced by
+        ``rev2`` since ``rev1``.
+
+        :param rev1: Entry point from which diff is shown. Can be
+          ``self.EMPTY_CHANGESET`` - in this case, patch showing all
+          the changes since empty state of the repository until ``rev2``
+        :param rev2: Until which revision changes should be shown.
+        :param ignore_whitespace: If set to ``True``, would not show whitespace
+          changes. Defaults to ``False``.
+        :param context: How many lines before/after changed lines should be
+          shown. Defaults to ``3``.
+        """
+        raise NotImplementedError
+
+    # ========== #
+    # COMMIT API #
+    # ========== #
+
+    @LazyProperty
+    def in_memory_changeset(self):
+        """
+        Returns ``InMemoryChangeset`` object for this repository.
+        """
+        raise NotImplementedError
+
+    def add(self, filenode, **kwargs):
+        """
+        Commit api function that will add given ``FileNode`` into this
+        repository.
+
+        :raises ``NodeAlreadyExistsError``: if there is a file with same path
+          already in repository
+        :raises ``NodeAlreadyAddedError``: if given node is already marked as
+          *added*
+        """
+        raise NotImplementedError
+
+    def remove(self, filenode, **kwargs):
+        """
+        Commit api function that will remove given ``FileNode`` into this
+        repository.
+
+        :raises ``EmptyRepositoryError``: if there are no changesets yet
+        :raises ``NodeDoesNotExistError``: if there is no file with given path
+        """
+        raise NotImplementedError
+
+    def commit(self, message, **kwargs):
+        """
+        Persists current changes made on this repository and returns newly
+        created changeset.
+
+        :raises ``NothingChangedError``: if no changes has been made
+        """
+        raise NotImplementedError
+
+    def get_state(self):
+        """
+        Returns dictionary with ``added``, ``changed`` and ``removed`` lists
+        containing ``FileNode`` objects.
+        """
+        raise NotImplementedError
+
+    def get_config_value(self, section, name, config_file=None):
+        """
+        Returns configuration value for a given [``section``] and ``name``.
+
+        :param section: Section we want to retrieve value from
+        :param name: Name of configuration we want to retrieve
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        raise NotImplementedError
+
+    def get_user_name(self, config_file=None):
+        """
+        Returns user's name from global configuration file.
+
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        raise NotImplementedError
+
+    def get_user_email(self, config_file=None):
+        """
+        Returns user's email from global configuration file.
+
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        raise NotImplementedError
+
+    # =========== #
+    # WORKDIR API #
+    # =========== #
+
+    @LazyProperty
+    def workdir(self):
+        """
+        Returns ``Workdir`` instance for this repository.
+        """
+        raise NotImplementedError
+
+
+class BaseChangeset(object):
+    """
+    Each backend should implement it's changeset representation.
+
+    **Attributes**
+
+        ``repository``
+            repository object within which changeset exists
+
+        ``id``
+            may be ``raw_id`` or i.e. for mercurial's tip just ``tip``
+
+        ``raw_id``
+            raw changeset representation (i.e. full 40 length sha for git
+            backend)
+
+        ``short_id``
+            shortened (if apply) version of ``raw_id``; it would be simple
+            shortcut for ``raw_id[:12]`` for git/mercurial backends or same
+            as ``raw_id`` for subversion
+
+        ``revision``
+            revision number as integer
+
+        ``files``
+            list of ``FileNode`` (``Node`` with NodeKind.FILE) objects
+
+        ``dirs``
+            list of ``DirNode`` (``Node`` with NodeKind.DIR) objects
+
+        ``nodes``
+            combined list of ``Node`` objects
+
+        ``author``
+            author of the changeset, as unicode
+
+        ``message``
+            message of the changeset, as unicode
+
+        ``parents``
+            list of parent changesets
+
+        ``last``
+            ``True`` if this is last changeset in repository, ``False``
+            otherwise; trying to access this attribute while there is no
+            changesets would raise ``EmptyRepositoryError``
+    """
+    def __str__(self):
+        return '<%s at %s:%s>' % (self.__class__.__name__, self.revision,
+            self.short_id)
+
+    def __repr__(self):
+        return self.__str__()
+
+    def __unicode__(self):
+        return u'%s:%s' % (self.revision, self.short_id)
+
+    def __eq__(self, other):
+        return self.raw_id == other.raw_id
+
+    @LazyProperty
+    def last(self):
+        if self.repository is None:
+            raise ChangesetError("Cannot check if it's most recent revision")
+        return self.raw_id == self.repository.revisions[-1]
+
+    @LazyProperty
+    def parents(self):
+        """
+        Returns list of parents changesets.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def id(self):
+        """
+        Returns string identifying this changeset.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def raw_id(self):
+        """
+        Returns raw string identifying this changeset.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def short_id(self):
+        """
+        Returns shortened version of ``raw_id`` attribute, as string,
+        identifying this changeset, useful for web representation.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def revision(self):
+        """
+        Returns integer identifying this changeset.
+
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def author(self):
+        """
+        Returns Author for given commit
+        """
+
+        raise NotImplementedError
+
+    @LazyProperty
+    def author_name(self):
+        """
+        Returns Author name for given commit
+        """
+
+        return author_name(self.author)
+
+    @LazyProperty
+    def author_email(self):
+        """
+        Returns Author email address for given commit
+        """
+
+        return author_email(self.author)
+
+    def get_file_mode(self, path):
+        """
+        Returns stat mode of the file at the given ``path``.
+        """
+        raise NotImplementedError
+
+    def get_file_content(self, path):
+        """
+        Returns content of the file at the given ``path``.
+        """
+        raise NotImplementedError
+
+    def get_file_size(self, path):
+        """
+        Returns size of the file at the given ``path``.
+        """
+        raise NotImplementedError
+
+    def get_file_changeset(self, path):
+        """
+        Returns last commit of the file at the given ``path``.
+        """
+        raise NotImplementedError
+
+    def get_file_history(self, path):
+        """
+        Returns history of file as reversed list of ``Changeset`` objects for
+        which file at given ``path`` has been modified.
+        """
+        raise NotImplementedError
+
+    def get_nodes(self, path):
+        """
+        Returns combined ``DirNode`` and ``FileNode`` objects list representing
+        state of changeset at the given ``path``.
+
+        :raises ``ChangesetError``: if node at the given ``path`` is not
+          instance of ``DirNode``
+        """
+        raise NotImplementedError
+
+    def get_node(self, path):
+        """
+        Returns ``Node`` object from the given ``path``.
+
+        :raises ``NodeDoesNotExistError``: if there is no node at the given
+          ``path``
+        """
+        raise NotImplementedError
+
+    def fill_archive(self, stream=None, kind='tgz', prefix=None):
+        """
+        Fills up given stream.
+
+        :param stream: file like object.
+        :param kind: one of following: ``zip``, ``tar``, ``tgz``
+            or ``tbz2``. Default: ``tgz``.
+        :param prefix: name of root directory in archive.
+            Default is repository name and changeset's raw_id joined with dash.
+
+            repo-tip.<kind>
+        """
+
+        raise NotImplementedError
+
+    def get_chunked_archive(self, **kwargs):
+        """
+        Returns iterable archive. Tiny wrapper around ``fill_archive`` method.
+
+        :param chunk_size: extra parameter which controls size of returned
+            chunks. Default:8k.
+        """
+
+        chunk_size = kwargs.pop('chunk_size', 8192)
+        stream = kwargs.get('stream')
+        self.fill_archive(**kwargs)
+        while True:
+            data = stream.read(chunk_size)
+            if not data:
+                break
+            yield data
+
+    @LazyProperty
+    def root(self):
+        """
+        Returns ``RootNode`` object for this changeset.
+        """
+        return self.get_node('')
+
+    def next(self, branch=None):
+        """
+        Returns next changeset from current, if branch is gives it will return
+        next changeset belonging to this branch
+
+        :param branch: show changesets within the given named branch
+        """
+        raise NotImplementedError
+
+    def prev(self, branch=None):
+        """
+        Returns previous changeset from current, if branch is gives it will
+        return previous changeset belonging to this branch
+
+        :param branch: show changesets within the given named branch
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def added(self):
+        """
+        Returns list of added ``FileNode`` objects.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def changed(self):
+        """
+        Returns list of modified ``FileNode`` objects.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def removed(self):
+        """
+        Returns list of removed ``FileNode`` objects.
+        """
+        raise NotImplementedError
+
+    @LazyProperty
+    def size(self):
+        """
+        Returns total number of bytes from contents of all filenodes.
+        """
+        return sum((node.size for node in self.get_filenodes_generator()))
+
+    def walk(self, topurl=''):
+        """
+        Similar to os.walk method. Insted of filesystem it walks through
+        changeset starting at given ``topurl``.  Returns generator of tuples
+        (topnode, dirnodes, filenodes).
+        """
+        topnode = self.get_node(topurl)
+        yield (topnode, topnode.dirs, topnode.files)
+        for dirnode in topnode.dirs:
+            for tup in self.walk(dirnode.path):
+                yield tup
+
+    def get_filenodes_generator(self):
+        """
+        Returns generator that yields *all* file nodes.
+        """
+        for topnode, dirs, files in self.walk():
+            for node in files:
+                yield node
+
+    def as_dict(self):
+        """
+        Returns dictionary with changeset's attributes and their values.
+        """
+        data = get_dict_for_attrs(self, ['id', 'raw_id', 'short_id',
+            'revision', 'date', 'message'])
+        data['author'] = {'name': self.author_name, 'email': self.author_email}
+        data['added'] = [node.path for node in self.added]
+        data['changed'] = [node.path for node in self.changed]
+        data['removed'] = [node.path for node in self.removed]
+        return data
+
+
+class BaseWorkdir(object):
+    """
+    Working directory representation of single repository.
+
+    :attribute: repository: repository object of working directory
+    """
+
+    def __init__(self, repository):
+        self.repository = repository
+
+    def get_branch(self):
+        """
+        Returns name of current branch.
+        """
+        raise NotImplementedError
+
+    def get_changeset(self):
+        """
+        Returns current changeset.
+        """
+        raise NotImplementedError
+
+    def get_added(self):
+        """
+        Returns list of ``FileNode`` objects marked as *new* in working
+        directory.
+        """
+        raise NotImplementedError
+
+    def get_changed(self):
+        """
+        Returns list of ``FileNode`` objects *changed* in working directory.
+        """
+        raise NotImplementedError
+
+    def get_removed(self):
+        """
+        Returns list of ``RemovedFileNode`` objects marked as *removed* in
+        working directory.
+        """
+        raise NotImplementedError
+
+    def get_untracked(self):
+        """
+        Returns list of ``FileNode`` objects which are present within working
+        directory however are not tracked by repository.
+        """
+        raise NotImplementedError
+
+    def get_status(self):
+        """
+        Returns dict with ``added``, ``changed``, ``removed`` and ``untracked``
+        lists.
+        """
+        raise NotImplementedError
+
+    def commit(self, message, **kwargs):
+        """
+        Commits local (from working directory) changes and returns newly
+        created
+        ``Changeset``. Updates repository's ``revisions`` list.
+
+        :raises ``CommitError``: if any error occurs while committing
+        """
+        raise NotImplementedError
+
+    def update(self, revision=None):
+        """
+        Fetches content of the given revision and populates it within working
+        directory.
+        """
+        raise NotImplementedError
+
+    def checkout_branch(self, branch=None):
+        """
+        Checks out ``branch`` or the backend's default branch.
+
+        Raises ``BranchDoesNotExistError`` if the branch does not exist.
+        """
+        raise NotImplementedError
+
+
+class BaseInMemoryChangeset(object):
+    """
+    Represents differences between repository's state (most recent head) and
+    changes made *in place*.
+
+    **Attributes**
+
+        ``repository``
+            repository object for this in-memory-changeset
+
+        ``added``
+            list of ``FileNode`` objects marked as *added*
+
+        ``changed``
+            list of ``FileNode`` objects marked as *changed*
+
+        ``removed``
+            list of ``FileNode`` or ``RemovedFileNode`` objects marked to be
+            *removed*
+
+        ``parents``
+            list of ``Changeset`` representing parents of in-memory changeset.
+            Should always be 2-element sequence.
+
+    """
+
+    def __init__(self, repository):
+        self.repository = repository
+        self.added = []
+        self.changed = []
+        self.removed = []
+        self.parents = []
+
+    def add(self, *filenodes):
+        """
+        Marks given ``FileNode`` objects as *to be committed*.
+
+        :raises ``NodeAlreadyExistsError``: if node with same path exists at
+          latest changeset
+        :raises ``NodeAlreadyAddedError``: if node with same path is already
+          marked as *added*
+        """
+        # Check if not already marked as *added* first
+        for node in filenodes:
+            if node.path in (n.path for n in self.added):
+                raise NodeAlreadyAddedError("Such FileNode %s is already "
+                    "marked for addition" % node.path)
+        for node in filenodes:
+            self.added.append(node)
+
+    def change(self, *filenodes):
+        """
+        Marks given ``FileNode`` objects to be *changed* in next commit.
+
+        :raises ``EmptyRepositoryError``: if there are no changesets yet
+        :raises ``NodeAlreadyExistsError``: if node with same path is already
+          marked to be *changed*
+        :raises ``NodeAlreadyRemovedError``: if node with same path is already
+          marked to be *removed*
+        :raises ``NodeDoesNotExistError``: if node doesn't exist in latest
+          changeset
+        :raises ``NodeNotChangedError``: if node hasn't really be changed
+        """
+        for node in filenodes:
+            if node.path in (n.path for n in self.removed):
+                raise NodeAlreadyRemovedError("Node at %s is already marked "
+                    "as removed" % node.path)
+        try:
+            self.repository.get_changeset()
+        except EmptyRepositoryError:
+            raise EmptyRepositoryError("Nothing to change - try to *add* new "
+                "nodes rather than changing them")
+        for node in filenodes:
+            if node.path in (n.path for n in self.changed):
+                raise NodeAlreadyChangedError("Node at '%s' is already "
+                    "marked as changed" % node.path)
+            self.changed.append(node)
+
+    def remove(self, *filenodes):
+        """
+        Marks given ``FileNode`` (or ``RemovedFileNode``) objects to be
+        *removed* in next commit.
+
+        :raises ``NodeAlreadyRemovedError``: if node has been already marked to
+          be *removed*
+        :raises ``NodeAlreadyChangedError``: if node has been already marked to
+          be *changed*
+        """
+        for node in filenodes:
+            if node.path in (n.path for n in self.removed):
+                raise NodeAlreadyRemovedError("Node is already marked to "
+                    "for removal at %s" % node.path)
+            if node.path in (n.path for n in self.changed):
+                raise NodeAlreadyChangedError("Node is already marked to "
+                    "be changed at %s" % node.path)
+            # We only mark node as *removed* - real removal is done by
+            # commit method
+            self.removed.append(node)
+
+    def reset(self):
+        """
+        Resets this instance to initial state (cleans ``added``, ``changed``
+        and ``removed`` lists).
+        """
+        self.added = []
+        self.changed = []
+        self.removed = []
+        self.parents = []
+
+    def get_ipaths(self):
+        """
+        Returns generator of paths from nodes marked as added, changed or
+        removed.
+        """
+        for node in chain(self.added, self.changed, self.removed):
+            yield node.path
+
+    def get_paths(self):
+        """
+        Returns list of paths from nodes marked as added, changed or removed.
+        """
+        return list(self.get_ipaths())
+
+    def check_integrity(self, parents=None):
+        """
+        Checks in-memory changeset's integrity. Also, sets parents if not
+        already set.
+
+        :raises CommitError: if any error occurs (i.e.
+          ``NodeDoesNotExistError``).
+        """
+        if not self.parents:
+            parents = parents or []
+            if len(parents) == 0:
+                try:
+                    parents = [self.repository.get_changeset(), None]
+                except EmptyRepositoryError:
+                    parents = [None, None]
+            elif len(parents) == 1:
+                parents += [None]
+            self.parents = parents
+
+        # Local parents, only if not None
+        parents = [p for p in self.parents if p]
+
+        # Check nodes marked as added
+        for p in parents:
+            for node in self.added:
+                try:
+                    p.get_node(node.path)
+                except NodeDoesNotExistError:
+                    pass
+                else:
+                    raise NodeAlreadyExistsError("Node at %s already exists "
+                        "at %s" % (node.path, p))
+
+        # Check nodes marked as changed
+        missing = set(self.changed)
+        not_changed = set(self.changed)
+        if self.changed and not parents:
+            raise NodeDoesNotExistError(str(self.changed[0].path))
+        for p in parents:
+            for node in self.changed:
+                try:
+                    old = p.get_node(node.path)
+                    missing.remove(node)
+                    if old.content != node.content:
+                        not_changed.remove(node)
+                except NodeDoesNotExistError:
+                    pass
+        if self.changed and missing:
+            raise NodeDoesNotExistError("Node at %s is missing "
+                "(parents: %s)" % (node.path, parents))
+
+        if self.changed and not_changed:
+            raise NodeNotChangedError("Node at %s wasn't actually changed "
+                "since parents' changesets: %s" % (not_changed.pop().path,
+                    parents)
+            )
+
+        # Check nodes marked as removed
+        if self.removed and not parents:
+            raise NodeDoesNotExistError("Cannot remove node at %s as there "
+                "were no parents specified" % self.removed[0].path)
+        really_removed = set()
+        for p in parents:
+            for node in self.removed:
+                try:
+                    p.get_node(node.path)
+                    really_removed.add(node)
+                except ChangesetError:
+                    pass
+        not_removed = set(self.removed) - really_removed
+        if not_removed:
+            raise NodeDoesNotExistError("Cannot remove node at %s from "
+                "following parents: %s" % (not_removed[0], parents))
+
+    def commit(self, message, author, parents=None, branch=None, date=None,
+            **kwargs):
+        """
+        Performs in-memory commit (doesn't check workdir in any way) and
+        returns newly created ``Changeset``. Updates repository's
+        ``revisions``.
+
+        .. note::
+            While overriding this method each backend's should call
+            ``self.check_integrity(parents)`` in the first place.
+
+        :param message: message of the commit
+        :param author: full username, i.e. "Joe Doe <[e-mail]>"
+        :param parents: single parent or sequence of parents from which commit
+          would be derieved
+        :param date: ``datetime.datetime`` instance. Defaults to
+          ``datetime.datetime.now()``.
+        :param branch: branch name, as string. If none given, default backend's
+          branch would be used.
+
+        :raises ``CommitError``: if any error occurs while committing
+        """
+        raise NotImplementedError

rhodecode/lib/vcs/backends/git/__init__.py

@@ -0,0 +1,9 b''
+from .repository import GitRepository
+from .changeset import GitChangeset
+from .inmemory import GitInMemoryChangeset
+from .workdir import GitWorkdir
+
+
+__all__ = [
+    'GitRepository', 'GitChangeset', 'GitInMemoryChangeset', 'GitWorkdir',
+]

rhodecode/lib/vcs/backends/git/changeset.py

@@ -0,0 +1,450 b''
+import re
+from itertools import chain
+from dulwich import objects
+from subprocess import Popen, PIPE
+from rhodecode.lib.vcs.conf import settings
+from rhodecode.lib.vcs.exceptions import RepositoryError
+from rhodecode.lib.vcs.exceptions import ChangesetError
+from rhodecode.lib.vcs.exceptions import NodeDoesNotExistError
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.exceptions import ChangesetDoesNotExistError
+from rhodecode.lib.vcs.exceptions import ImproperArchiveTypeError
+from rhodecode.lib.vcs.backends.base import BaseChangeset
+from rhodecode.lib.vcs.nodes import FileNode, DirNode, NodeKind, RootNode, RemovedFileNode
+from rhodecode.lib.vcs.utils import safe_unicode
+from rhodecode.lib.vcs.utils import date_fromtimestamp
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+
+
+class GitChangeset(BaseChangeset):
+    """
+    Represents state of the repository at single revision.
+    """
+
+    def __init__(self, repository, revision):
+        self._stat_modes = {}
+        self.repository = repository
+        self.raw_id = revision
+        self.revision = repository.revisions.index(revision)
+
+        self.short_id = self.raw_id[:12]
+        self.id = self.raw_id
+        try:
+            commit = self.repository._repo.get_object(self.raw_id)
+        except KeyError:
+            raise RepositoryError("Cannot get object with id %s" % self.raw_id)
+        self._commit = commit
+        self._tree_id = commit.tree
+
+        try:
+            self.message = safe_unicode(commit.message[:-1])
+            # Always strip last eol
+        except UnicodeDecodeError:
+            self.message = commit.message[:-1].decode(commit.encoding
+                or 'utf-8')
+        #self.branch = None
+        self.tags = []
+        #tree = self.repository.get_object(self._tree_id)
+        self.nodes = {}
+        self._paths = {}
+
+    @LazyProperty
+    def author(self):
+        return safe_unicode(self._commit.committer)
+
+    @LazyProperty
+    def date(self):
+        return date_fromtimestamp(self._commit.commit_time,
+                                  self._commit.commit_timezone)
+
+    @LazyProperty
+    def status(self):
+        """
+        Returns modified, added, removed, deleted files for current changeset
+        """
+        return self.changed, self.added, self.removed
+
+    @LazyProperty
+    def branch(self):
+        # TODO: Cache as we walk (id <-> branch name mapping)
+        refs = self.repository._repo.get_refs()
+        heads = [(key[len('refs/heads/'):], val) for key, val in refs.items()
+            if key.startswith('refs/heads/')]
+
+        for name, id in heads:
+            walker = self.repository._repo.object_store.get_graph_walker([id])
+            while True:
+                id = walker.next()
+                if not id:
+                    break
+                if id == self.id:
+                    return safe_unicode(name)
+        raise ChangesetError("This should not happen... Have you manually "
+            "change id of the changeset?")
+
+    def _fix_path(self, path):
+        """
+        Paths are stored without trailing slash so we need to get rid off it if
+        needed.
+        """
+        if path.endswith('/'):
+            path = path.rstrip('/')
+        return path
+
+    def _get_id_for_path(self, path):
+        # FIXME: Please, spare a couple of minutes and make those codes cleaner;
+        if not path in self._paths:
+            path = path.strip('/')
+            # set root tree
+            tree = self.repository._repo[self._commit.tree]
+            if path == '':
+                self._paths[''] = tree.id
+                return tree.id
+            splitted = path.split('/')
+            dirs, name = splitted[:-1], splitted[-1]
+            curdir = ''
+            for dir in dirs:
+                if curdir:
+                    curdir = '/'.join((curdir, dir))
+                else:
+                    curdir = dir
+                #if curdir in self._paths:
+                    ## This path have been already traversed
+                    ## Update tree and continue
+                    #tree = self.repository._repo[self._paths[curdir]]
+                    #continue
+                dir_id = None
+                for item, stat, id in tree.iteritems():
+                    if curdir:
+                        item_path = '/'.join((curdir, item))
+                    else:
+                        item_path = item
+                    self._paths[item_path] = id
+                    self._stat_modes[item_path] = stat
+                    if dir == item:
+                        dir_id = id
+                if dir_id:
+                    # Update tree
+                    tree = self.repository._repo[dir_id]
+                    if not isinstance(tree, objects.Tree):
+                        raise ChangesetError('%s is not a directory' % curdir)
+                else:
+                    raise ChangesetError('%s have not been found' % curdir)
+            for item, stat, id in tree.iteritems():
+                if curdir:
+                    name = '/'.join((curdir, item))
+                else:
+                    name = item
+                self._paths[name] = id
+                self._stat_modes[name] = stat
+            if not path in self._paths:
+                raise NodeDoesNotExistError("There is no file nor directory "
+                    "at the given path %r at revision %r"
+                    % (path, self.short_id))
+        return self._paths[path]
+
+    def _get_kind(self, path):
+        id = self._get_id_for_path(path)
+        obj = self.repository._repo[id]
+        if isinstance(obj, objects.Blob):
+            return NodeKind.FILE
+        elif isinstance(obj, objects.Tree):
+            return NodeKind.DIR
+
+    def _get_file_nodes(self):
+        return chain(*(t[2] for t in self.walk()))
+
+    @LazyProperty
+    def parents(self):
+        """
+        Returns list of parents changesets.
+        """
+        return [self.repository.get_changeset(parent)
+            for parent in self._commit.parents]
+
+    def next(self, branch=None):
+
+        if branch and self.branch != branch:
+            raise VCSError('Branch option used on changeset not belonging '
+                           'to that branch')
+
+        def _next(changeset, branch):
+            try:
+                next_ = changeset.revision + 1
+                next_rev = changeset.repository.revisions[next_]
+            except IndexError:
+                raise ChangesetDoesNotExistError
+            cs = changeset.repository.get_changeset(next_rev)
+
+            if branch and branch != cs.branch:
+                return _next(cs, branch)
+
+            return cs
+
+        return _next(self, branch)
+
+    def prev(self, branch=None):
+        if branch and self.branch != branch:
+            raise VCSError('Branch option used on changeset not belonging '
+                           'to that branch')
+
+        def _prev(changeset, branch):
+            try:
+                prev_ = changeset.revision - 1
+                if prev_ < 0:
+                    raise IndexError
+                prev_rev = changeset.repository.revisions[prev_]
+            except IndexError:
+                raise ChangesetDoesNotExistError
+
+            cs = changeset.repository.get_changeset(prev_rev)
+
+            if branch and branch != cs.branch:
+                return _prev(cs, branch)
+
+            return cs
+
+        return _prev(self, branch)
+
+    def get_file_mode(self, path):
+        """
+        Returns stat mode of the file at the given ``path``.
+        """
+        # ensure path is traversed
+        self._get_id_for_path(path)
+        return self._stat_modes[path]
+
+    def get_file_content(self, path):
+        """
+        Returns content of the file at given ``path``.
+        """
+        id = self._get_id_for_path(path)
+        blob = self.repository._repo[id]
+        return blob.as_pretty_string()
+
+    def get_file_size(self, path):
+        """
+        Returns size of the file at given ``path``.
+        """
+        id = self._get_id_for_path(path)
+        blob = self.repository._repo[id]
+        return blob.raw_length()
+
+    def get_file_changeset(self, path):
+        """
+        Returns last commit of the file at the given ``path``.
+        """
+        node = self.get_node(path)
+        return node.history[0]
+
+    def get_file_history(self, path):
+        """
+        Returns history of file as reversed list of ``Changeset`` objects for
+        which file at given ``path`` has been modified.
+
+        TODO: This function now uses os underlying 'git' and 'grep' commands
+        which is generally not good. Should be replaced with algorithm
+        iterating commits.
+        """
+        cmd = 'log --name-status -p %s -- "%s" | grep "^commit"' \
+            % (self.id, path)
+        so, se = self.repository.run_git_command(cmd)
+        ids = re.findall(r'\w{40}', so)
+        return [self.repository.get_changeset(id) for id in ids]
+
+    def get_file_annotate(self, path):
+        """
+        Returns a list of three element tuples with lineno,changeset and line
+
+        TODO: This function now uses os underlying 'git' command which is
+        generally not good. Should be replaced with algorithm iterating
+        commits.
+        """
+        cmd = 'blame -l --root -r %s -- "%s"' % (self.id, path)
+        # -l     ==> outputs long shas (and we need all 40 characters)
+        # --root ==> doesn't put '^' character for bounderies
+        # -r sha ==> blames for the given revision
+        so, se = self.repository.run_git_command(cmd)
+        annotate = []
+        for i, blame_line in enumerate(so.split('\n')[:-1]):
+            ln_no = i + 1
+            id, line = re.split(r' \(.+?\) ', blame_line, 1)
+            annotate.append((ln_no, self.repository.get_changeset(id), line))
+        return annotate
+
+    def fill_archive(self, stream=None, kind='tgz', prefix=None,
+                     subrepos=False):
+        """
+        Fills up given stream.
+
+        :param stream: file like object.
+        :param kind: one of following: ``zip``, ``tgz`` or ``tbz2``.
+            Default: ``tgz``.
+        :param prefix: name of root directory in archive.
+            Default is repository name and changeset's raw_id joined with dash
+            (``repo-tip.<KIND>``).
+        :param subrepos: include subrepos in this archive.
+
+        :raise ImproperArchiveTypeError: If given kind is wrong.
+        :raise VcsError: If given stream is None
+
+        """
+        allowed_kinds = settings.ARCHIVE_SPECS.keys()
+        if kind not in allowed_kinds:
+            raise ImproperArchiveTypeError('Archive kind not supported use one'
+                'of %s', allowed_kinds)
+
+        if prefix is None:
+            prefix = '%s-%s' % (self.repository.name, self.short_id)
+        elif prefix.startswith('/'):
+            raise VCSError("Prefix cannot start with leading slash")
+        elif prefix.strip() == '':
+            raise VCSError("Prefix cannot be empty")
+
+        if kind == 'zip':
+            frmt = 'zip'
+        else:
+            frmt = 'tar'
+        cmd = 'git archive --format=%s --prefix=%s/ %s' % (frmt, prefix,
+            self.raw_id)
+        if kind == 'tgz':
+            cmd += ' | gzip -9'
+        elif kind == 'tbz2':
+            cmd += ' | bzip2 -9'
+
+        if stream is None:
+            raise VCSError('You need to pass in a valid stream for filling'
+                           ' with archival data')
+        popen = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True,
+            cwd=self.repository.path)
+
+        buffer_size = 1024 * 8
+        chunk = popen.stdout.read(buffer_size)
+        while chunk:
+            stream.write(chunk)
+            chunk = popen.stdout.read(buffer_size)
+        # Make sure all descriptors would be read
+        popen.communicate()
+
+    def get_nodes(self, path):
+        if self._get_kind(path) != NodeKind.DIR:
+            raise ChangesetError("Directory does not exist for revision %r at "
+                " %r" % (self.revision, path))
+        path = self._fix_path(path)
+        id = self._get_id_for_path(path)
+        tree = self.repository._repo[id]
+        dirnodes = []
+        filenodes = []
+        for name, stat, id in tree.iteritems():
+            obj = self.repository._repo.get_object(id)
+            if path != '':
+                obj_path = '/'.join((path, name))
+            else:
+                obj_path = name
+            if obj_path not in self._stat_modes:
+                self._stat_modes[obj_path] = stat
+            if isinstance(obj, objects.Tree):
+                dirnodes.append(DirNode(obj_path, changeset=self))
+            elif isinstance(obj, objects.Blob):
+                filenodes.append(FileNode(obj_path, changeset=self, mode=stat))
+            else:
+                raise ChangesetError("Requested object should be Tree "
+                                     "or Blob, is %r" % type(obj))
+        nodes = dirnodes + filenodes
+        for node in nodes:
+            if not node.path in self.nodes:
+                self.nodes[node.path] = node
+        nodes.sort()
+        return nodes
+
+    def get_node(self, path):
+        if isinstance(path, unicode):
+            path = path.encode('utf-8')
+        path = self._fix_path(path)
+        if not path in self.nodes:
+            try:
+                id = self._get_id_for_path(path)
+            except ChangesetError:
+                raise NodeDoesNotExistError("Cannot find one of parents' "
+                    "directories for a given path: %s" % path)
+            obj = self.repository._repo.get_object(id)
+            if isinstance(obj, objects.Tree):
+                if path == '':
+                    node = RootNode(changeset=self)
+                else:
+                    node = DirNode(path, changeset=self)
+                node._tree = obj
+            elif isinstance(obj, objects.Blob):
+                node = FileNode(path, changeset=self)
+                node._blob = obj
+            else:
+                raise NodeDoesNotExistError("There is no file nor directory "
+                    "at the given path %r at revision %r"
+                    % (path, self.short_id))
+            # cache node
+            self.nodes[path] = node
+        return self.nodes[path]
+
+    @LazyProperty
+    def affected_files(self):
+        """
+        Get's a fast accessible file changes for given changeset
+        """
+
+        return self.added + self.changed
+
+    @LazyProperty
+    def _diff_name_status(self):
+        output = []
+        for parent in self.parents:
+            cmd = 'diff --name-status %s %s' % (parent.raw_id, self.raw_id)
+            so, se = self.repository.run_git_command(cmd)
+            output.append(so.strip())
+        return '\n'.join(output)
+
+    def _get_paths_for_status(self, status):
+        """
+        Returns sorted list of paths for given ``status``.
+
+        :param status: one of: *added*, *modified* or *deleted*
+        """
+        paths = set()
+        char = status[0].upper()
+        for line in self._diff_name_status.splitlines():
+            if not line:
+                continue
+            if line.startswith(char):
+                splitted = line.split(char,1)
+                if not len(splitted) == 2:
+                    raise VCSError("Couldn't parse diff result:\n%s\n\n and "
+                        "particularly that line: %s" % (self._diff_name_status,
+                        line))
+                paths.add(splitted[1].strip())
+        return sorted(paths)
+
+    @LazyProperty
+    def added(self):
+        """
+        Returns list of added ``FileNode`` objects.
+        """
+        if not self.parents:
+            return list(self._get_file_nodes())
+        return [self.get_node(path) for path in self._get_paths_for_status('added')]
+
+    @LazyProperty
+    def changed(self):
+        """
+        Returns list of modified ``FileNode`` objects.
+        """
+        if not self.parents:
+            return []
+        return [self.get_node(path) for path in self._get_paths_for_status('modified')]
+
+    @LazyProperty
+    def removed(self):
+        """
+        Returns list of removed ``FileNode`` objects.
+        """
+        if not self.parents:
+            return []
+        return [RemovedFileNode(path) for path in self._get_paths_for_status('deleted')]

rhodecode/lib/vcs/backends/git/config.py

@@ -0,0 +1,347 b''
+# config.py - Reading and writing Git config files
+# Copyright (C) 2011 Jelmer Vernooij <[e-mail]>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; version 2
+# of the License or (at your option) a later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA  02110-1301, USA.
+
+"""Reading and writing Git configuration files.
+
+TODO:
+ * preserve formatting when updating configuration files
+ * treat subsection names as case-insensitive for [branch.foo] style
+   subsections
+"""
+
+# Taken from dulwich not yet released 0.8.3 version (until it is actually
+# released)
+
+import errno
+import os
+import re
+
+from dulwich.file import GitFile
+
+
+class Config(object):
+    """A Git configuration."""
+
+    def get(self, section, name):
+        """Retrieve the contents of a configuration setting.
+
+        :param section: Tuple with section name and optional subsection namee
+        :param subsection: Subsection name
+        :return: Contents of the setting
+        :raise KeyError: if the value is not set
+        """
+        raise NotImplementedError(self.get)
+
+    def get_boolean(self, section, name, default=None):
+        """Retrieve a configuration setting as boolean.
+
+        :param section: Tuple with section name and optional subsection namee
+        :param name: Name of the setting, including section and possible
+            subsection.
+        :return: Contents of the setting
+        :raise KeyError: if the value is not set
+        """
+        try:
+            value = self.get(section, name)
+        except KeyError:
+            return default
+        if value.lower() == "true":
+            return True
+        elif value.lower() == "false":
+            return False
+        raise ValueError("not a valid boolean string: %r" % value)
+
+    def set(self, section, name, value):
+        """Set a configuration value.
+
+        :param name: Name of the configuration value, including section
+            and optional subsection
+        :param: Value of the setting
+        """
+        raise NotImplementedError(self.set)
+
+
+class ConfigDict(Config):
+    """Git configuration stored in a dictionary."""
+
+    def __init__(self, values=None):
+        """Create a new ConfigDict."""
+        if values is None:
+            values = {}
+        self._values = values
+
+    def __repr__(self):
+        return "%s(%r)" % (self.__class__.__name__, self._values)
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, self.__class__) and
+            other._values == self._values)
+
+    @classmethod
+    def _parse_setting(cls, name):
+        parts = name.split(".")
+        if len(parts) == 3:
+            return (parts[0], parts[1], parts[2])
+        else:
+            return (parts[0], None, parts[1])
+
+    def get(self, section, name):
+        if isinstance(section, basestring):
+            section = (section, )
+        if len(section) > 1:
+            try:
+                return self._values[section][name]
+            except KeyError:
+                pass
+        return self._values[(section[0],)][name]
+
+    def set(self, section, name, value):
+        if isinstance(section, basestring):
+            section = (section, )
+        self._values.setdefault(section, {})[name] = value
+
+
+def _format_string(value):
+    if (value.startswith(" ") or
+        value.startswith("\t") or
+        value.endswith(" ") or
+        value.endswith("\t")):
+        return '"%s"' % _escape_value(value)
+    return _escape_value(value)
+
+
+def _parse_string(value):
+    value = value.strip()
+    ret = []
+    block = []
+    in_quotes = False
+    for c in value:
+        if c == "\"":
+            in_quotes = (not in_quotes)
+            ret.append(_unescape_value("".join(block)))
+            block = []
+        elif c in ("#", ";") and not in_quotes:
+            # the rest of the line is a comment
+            break
+        else:
+            block.append(c)
+
+    if in_quotes:
+        raise ValueError("value starts with quote but lacks end quote")
+
+    ret.append(_unescape_value("".join(block)).rstrip())
+
+    return "".join(ret)
+
+
+def _unescape_value(value):
+    """Unescape a value."""
+    def unescape(c):
+        return {
+            "\\\\": "\\",
+            "\\\"": "\"",
+            "\\n": "\n",
+            "\\t": "\t",
+            "\\b": "\b",
+            }[c.group(0)]
+    return re.sub(r"(\\.)", unescape, value)
+
+
+def _escape_value(value):
+    """Escape a value."""
+    return value.replace("\\", "\\\\").replace("\n", "\\n")\
+            .replace("\t", "\\t").replace("\"", "\\\"")
+
+
+def _check_variable_name(name):
+    for c in name:
+        if not c.isalnum() and c != '-':
+            return False
+    return True
+
+
+def _check_section_name(name):
+    for c in name:
+        if not c.isalnum() and c not in ('-', '.'):
+            return False
+    return True
+
+
+def _strip_comments(line):
+    line = line.split("#")[0]
+    line = line.split(";")[0]
+    return line
+
+
+class ConfigFile(ConfigDict):
+    """A Git configuration file, like .git/config or ~/.gitconfig.
+    """
+
+    @classmethod
+    def from_file(cls, f):
+        """Read configuration from a file-like object."""
+        ret = cls()
+        section = None
+        setting = None
+        for lineno, line in enumerate(f.readlines()):
+            line = line.lstrip()
+            if setting is None:
+                if _strip_comments(line).strip() == "":
+                    continue
+                if line[0] == "[":
+                    line = _strip_comments(line).rstrip()
+                    if line[-1] != "]":
+                        raise ValueError("expected trailing ]")
+                    key = line.strip()
+                    pts = key[1:-1].split(" ", 1)
+                    pts[0] = pts[0].lower()
+                    if len(pts) == 2:
+                        if pts[1][0] != "\"" or pts[1][-1] != "\"":
+                            raise ValueError(
+                                "Invalid subsection " + pts[1])
+                        else:
+                            pts[1] = pts[1][1:-1]
+                        if not _check_section_name(pts[0]):
+                            raise ValueError("invalid section name %s" %
+                                             pts[0])
+                        section = (pts[0], pts[1])
+                    else:
+                        if not _check_section_name(pts[0]):
+                            raise ValueError("invalid section name %s" %
+                                    pts[0])
+                        pts = pts[0].split(".", 1)
+                        if len(pts) == 2:
+                            section = (pts[0], pts[1])
+                        else:
+                            section = (pts[0], )
+                    ret._values[section] = {}
+                else:
+                    if section is None:
+                        raise ValueError("setting %r without section" % line)
+                    try:
+                        setting, value = line.split("=", 1)
+                    except ValueError:
+                        setting = line
+                        value = "true"
+                    setting = setting.strip().lower()
+                    if not _check_variable_name(setting):
+                        raise ValueError("invalid variable name %s" % setting)
+                    if value.endswith("\\\n"):
+                        value = value[:-2]
+                        continuation = True
+                    else:
+                        continuation = False
+                    value = _parse_string(value)
+                    ret._values[section][setting] = value
+                    if not continuation:
+                        setting = None
+            else:  # continuation line
+                if line.endswith("\\\n"):
+                    line = line[:-2]
+                    continuation = True
+                else:
+                    continuation = False
+                value = _parse_string(line)
+                ret._values[section][setting] += value
+                if not continuation:
+                    setting = None
+        return ret
+
+    @classmethod
+    def from_path(cls, path):
+        """Read configuration from a file on disk."""
+        f = GitFile(path, 'rb')
+        try:
+            ret = cls.from_file(f)
+            ret.path = path
+            return ret
+        finally:
+            f.close()
+
+    def write_to_path(self, path=None):
+        """Write configuration to a file on disk."""
+        if path is None:
+            path = self.path
+        f = GitFile(path, 'wb')
+        try:
+            self.write_to_file(f)
+        finally:
+            f.close()
+
+    def write_to_file(self, f):
+        """Write configuration to a file-like object."""
+        for section, values in self._values.iteritems():
+            try:
+                section_name, subsection_name = section
+            except ValueError:
+                (section_name, ) = section
+                subsection_name = None
+            if subsection_name is None:
+                f.write("[%s]\n" % section_name)
+            else:
+                f.write("[%s \"%s\"]\n" % (section_name, subsection_name))
+            for key, value in values.iteritems():
+                f.write("%s = %s\n" % (key, _escape_value(value)))
+
+
+class StackedConfig(Config):
+    """Configuration which reads from multiple config files.."""
+
+    def __init__(self, backends, writable=None):
+        self.backends = backends
+        self.writable = writable
+
+    def __repr__(self):
+        return "<%s for %r>" % (self.__class__.__name__, self.backends)
+
+    @classmethod
+    def default_backends(cls):
+        """Retrieve the default configuration.
+
+        This will look in the repository configuration (if for_path is
+        specified), the users' home directory and the system
+        configuration.
+        """
+        paths = []
+        paths.append(os.path.expanduser("~/.gitconfig"))
+        paths.append("/etc/gitconfig")
+        backends = []
+        for path in paths:
+            try:
+                cf = ConfigFile.from_path(path)
+            except (IOError, OSError), e:
+                if e.errno != errno.ENOENT:
+                    raise
+                else:
+                    continue
+            backends.append(cf)
+        return backends
+
+    def get(self, section, name):
+        for backend in self.backends:
+            try:
+                return backend.get(section, name)
+            except KeyError:
+                pass
+        raise KeyError(name)
+
+    def set(self, section, name, value):
+        if self.writable is None:
+            raise NotImplementedError(self.set)
+        return self.writable.set(section, name, value)

rhodecode/lib/vcs/backends/git/inmemory.py

@@ -0,0 +1,192 b''
+import time
+import datetime
+import posixpath
+from dulwich import objects
+from dulwich.repo import Repo
+from rhodecode.lib.vcs.backends.base import BaseInMemoryChangeset
+from rhodecode.lib.vcs.exceptions import RepositoryError
+
+
+class GitInMemoryChangeset(BaseInMemoryChangeset):
+
+    def commit(self, message, author, parents=None, branch=None, date=None,
+            **kwargs):
+        """
+        Performs in-memory commit (doesn't check workdir in any way) and
+        returns newly created ``Changeset``. Updates repository's
+        ``revisions``.
+
+        :param message: message of the commit
+        :param author: full username, i.e. "Joe Doe <[e-mail]>"
+        :param parents: single parent or sequence of parents from which commit
+          would be derieved
+        :param date: ``datetime.datetime`` instance. Defaults to
+          ``datetime.datetime.now()``.
+        :param branch: branch name, as string. If none given, default backend's
+          branch would be used.
+
+        :raises ``CommitError``: if any error occurs while committing
+        """
+        self.check_integrity(parents)
+
+        from .repository import GitRepository
+        if branch is None:
+            branch = GitRepository.DEFAULT_BRANCH_NAME
+
+        repo = self.repository._repo
+        object_store = repo.object_store
+
+        ENCODING = "UTF-8"
+        DIRMOD = 040000
+
+        # Create tree and populates it with blobs
+        commit_tree = self.parents[0] and repo[self.parents[0]._commit.tree] or\
+            objects.Tree()
+        for node in self.added + self.changed:
+            # Compute subdirs if needed
+            dirpath, nodename = posixpath.split(node.path)
+            dirnames = dirpath and dirpath.split('/') or []
+            parent = commit_tree
+            ancestors = [('', parent)]
+
+            # Tries to dig for the deepest existing tree
+            while dirnames:
+                curdir = dirnames.pop(0)
+                try:
+                    dir_id = parent[curdir][1]
+                except KeyError:
+                    # put curdir back into dirnames and stops
+                    dirnames.insert(0, curdir)
+                    break
+                else:
+                    # If found, updates parent
+                    parent = self.repository._repo[dir_id]
+                    ancestors.append((curdir, parent))
+            # Now parent is deepest exising tree and we need to create subtrees
+            # for dirnames (in reverse order) [this only applies for nodes from added]
+            new_trees = []
+            blob = objects.Blob.from_string(node.content.encode(ENCODING))
+            node_path = node.name.encode(ENCODING)
+            if dirnames:
+                # If there are trees which should be created we need to build
+                # them now (in reverse order)
+                reversed_dirnames = list(reversed(dirnames))
+                curtree = objects.Tree()
+                curtree[node_path] = node.mode, blob.id
+                new_trees.append(curtree)
+                for dirname in reversed_dirnames[:-1]:
+                    newtree = objects.Tree()
+                    #newtree.add(DIRMOD, dirname, curtree.id)
+                    newtree[dirname] = DIRMOD, curtree.id
+                    new_trees.append(newtree)
+                    curtree = newtree
+                parent[reversed_dirnames[-1]] = DIRMOD, curtree.id
+            else:
+                parent.add(node.mode, node_path, blob.id)
+            new_trees.append(parent)
+            # Update ancestors
+            for parent, tree, path in reversed([(a[1], b[1], b[0]) for a, b in
+                zip(ancestors, ancestors[1:])]):
+                parent[path] = DIRMOD, tree.id
+                object_store.add_object(tree)
+
+            object_store.add_object(blob)
+            for tree in new_trees:
+                object_store.add_object(tree)
+        for node in self.removed:
+            paths = node.path.split('/')
+            tree = commit_tree
+            trees = [tree]
+            # Traverse deep into the forest...
+            for path in paths:
+                try:
+                    obj = self.repository._repo[tree[path][1]]
+                    if isinstance(obj, objects.Tree):
+                        trees.append(obj)
+                        tree = obj
+                except KeyError:
+                    break
+            # Cut down the blob and all rotten trees on the way back...
+            for path, tree in reversed(zip(paths, trees)):
+                del tree[path]
+                if tree:
+                    # This tree still has elements - don't remove it or any
+                    # of it's parents
+                    break
+
+        object_store.add_object(commit_tree)
+
+        # Create commit
+        commit = objects.Commit()
+        commit.tree = commit_tree.id
+        commit.parents = [p._commit.id for p in self.parents if p]
+        commit.author = commit.committer = author
+        commit.encoding = ENCODING
+        commit.message = message + ' '
+
+        # Compute date
+        if date is None:
+            date = time.time()
+        elif isinstance(date, datetime.datetime):
+            date = time.mktime(date.timetuple())
+
+        author_time = kwargs.pop('author_time', date)
+        commit.commit_time = int(date)
+        commit.author_time = int(author_time)
+        tz = time.timezone
+        author_tz = kwargs.pop('author_timezone', tz)
+        commit.commit_timezone = tz
+        commit.author_timezone = author_tz
+
+        object_store.add_object(commit)
+
+        ref = 'refs/heads/%s' % branch
+        repo.refs[ref] = commit.id
+        repo.refs.set_symbolic_ref('HEAD', ref)
+
+        # Update vcs repository object & recreate dulwich repo
+        self.repository.revisions.append(commit.id)
+        self.repository._repo = Repo(self.repository.path)
+        tip = self.repository.get_changeset()
+        self.reset()
+        return tip
+
+    def _get_missing_trees(self, path, root_tree):
+        """
+        Creates missing ``Tree`` objects for the given path.
+
+        :param path: path given as a string. It may be a path to a file node
+          (i.e. ``foo/bar/baz.txt``) or directory path - in that case it must
+          end with slash (i.e. ``foo/bar/``).
+        :param root_tree: ``dulwich.objects.Tree`` object from which we start
+          traversing (should be commit's root tree)
+        """
+        dirpath = posixpath.split(path)[0]
+        dirs = dirpath.split('/')
+        if not dirs or dirs == ['']:
+            return []
+
+        def get_tree_for_dir(tree, dirname):
+            for name, mode, id in tree.iteritems():
+                if name == dirname:
+                    obj = self.repository._repo[id]
+                    if isinstance(obj, objects.Tree):
+                        return obj
+                    else:
+                        raise RepositoryError("Cannot create directory %s "
+                        "at tree %s as path is occupied and is not a "
+                        "Tree" % (dirname, tree))
+            return None
+
+        trees = []
+        parent = root_tree
+        for dirname in dirs:
+            tree = get_tree_for_dir(parent, dirname)
+            if tree is None:
+                tree = objects.Tree()
+                dirmode = 040000
+                parent.add(dirmode, dirname, tree.id)
+                parent = tree
+            # Always append tree
+            trees.append(tree)
+        return trees

rhodecode/lib/vcs/backends/git/repository.py

@@ -0,0 +1,508 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.backends.git
+    ~~~~~~~~~~~~~~~~
+
+    Git backend implementation.
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+
+import os
+import re
+import time
+import posixpath
+from dulwich.repo import Repo, NotGitRepository
+#from dulwich.config import ConfigFile
+from string import Template
+from subprocess import Popen, PIPE
+from rhodecode.lib.vcs.backends.base import BaseRepository
+from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError
+from rhodecode.lib.vcs.exceptions import ChangesetDoesNotExistError
+from rhodecode.lib.vcs.exceptions import EmptyRepositoryError
+from rhodecode.lib.vcs.exceptions import RepositoryError
+from rhodecode.lib.vcs.exceptions import TagAlreadyExistError
+from rhodecode.lib.vcs.exceptions import TagDoesNotExistError
+from rhodecode.lib.vcs.utils import safe_unicode, makedate, date_fromtimestamp
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.ordered_dict import OrderedDict
+from rhodecode.lib.vcs.utils.paths import abspath
+from rhodecode.lib.vcs.utils.paths import get_user_home
+from .workdir import GitWorkdir
+from .changeset import GitChangeset
+from .inmemory import GitInMemoryChangeset
+from .config import ConfigFile
+
+
+class GitRepository(BaseRepository):
+    """
+    Git repository backend.
+    """
+    DEFAULT_BRANCH_NAME = 'master'
+    scm = 'git'
+
+    def __init__(self, repo_path, create=False, src_url=None,
+                 update_after_clone=False, bare=False):
+
+        self.path = abspath(repo_path)
+        self._repo = self._get_repo(create, src_url, update_after_clone, bare)
+        try:
+            self.head = self._repo.head()
+        except KeyError:
+            self.head = None
+
+        self._config_files = [
+            bare and abspath(self.path, 'config') or abspath(self.path, '.git',
+                'config'),
+            abspath(get_user_home(), '.gitconfig'),
+        ]
+
+    @LazyProperty
+    def revisions(self):
+        """
+        Returns list of revisions' ids, in ascending order.  Being lazy
+        attribute allows external tools to inject shas from cache.
+        """
+        return self._get_all_revisions()
+
+    def run_git_command(self, cmd):
+        """
+        Runs given ``cmd`` as git command and returns tuple
+        (returncode, stdout, stderr).
+
+        .. note::
+           This method exists only until log/blame functionality is implemented
+           at Dulwich (see https://bugs.launchpad.net/bugs/645142). Parsing
+           os command's output is road to hell...
+
+        :param cmd: git command to be executed
+        """
+        #cmd = '(cd %s && git %s)' % (self.path, cmd)
+        if isinstance(cmd, basestring):
+            cmd = 'git %s' % cmd
+        else:
+            cmd = ['git'] + cmd
+        try:
+            opts = dict(
+                shell=isinstance(cmd, basestring),
+                stdout=PIPE,
+                stderr=PIPE)
+            if os.path.isdir(self.path):
+                opts['cwd'] = self.path
+            p = Popen(cmd, **opts)
+        except OSError, err:
+            raise RepositoryError("Couldn't run git command (%s).\n"
+                "Original error was:%s" % (cmd, err))
+        so, se = p.communicate()
+        if not se.startswith("fatal: bad default revision 'HEAD'") and \
+            p.returncode != 0:
+            raise RepositoryError("Couldn't run git command (%s).\n"
+                "stderr:\n%s" % (cmd, se))
+        return so, se
+
+    def _check_url(self, url):
+        """
+        Functon will check given url and try to verify if it's a valid
+        link. Sometimes it may happened that mercurial will issue basic
+        auth request that can cause whole API to hang when used from python
+        or other external calls.
+
+        On failures it'll raise urllib2.HTTPError
+        """
+
+        #TODO: implement this
+        pass
+
+    def _get_repo(self, create, src_url=None, update_after_clone=False,
+            bare=False):
+        if create and os.path.exists(self.path):
+            raise RepositoryError("Location already exist")
+        if src_url and not create:
+            raise RepositoryError("Create should be set to True if src_url is "
+                                  "given (clone operation creates repository)")
+        try:
+            if create and src_url:
+                self._check_url(src_url)
+                self.clone(src_url, update_after_clone, bare)
+                return Repo(self.path)
+            elif create:
+                os.mkdir(self.path)
+                if bare:
+                    return Repo.init_bare(self.path)
+                else:
+                    return Repo.init(self.path)
+            else:
+                return Repo(self.path)
+        except (NotGitRepository, OSError), err:
+            raise RepositoryError(err)
+
+    def _get_all_revisions(self):
+        cmd = 'rev-list --all --date-order'
+        try:
+            so, se = self.run_git_command(cmd)
+        except RepositoryError:
+            # Can be raised for empty repositories
+            return []
+        revisions = so.splitlines()
+        revisions.reverse()
+        return revisions
+
+    def _get_revision(self, revision):
+        """
+        For git backend we always return integer here. This way we ensure
+        that changset's revision attribute would become integer.
+        """
+        pattern = re.compile(r'^[[0-9a-fA-F]{12}|[0-9a-fA-F]{40}]$')
+        is_bstr = lambda o: isinstance(o, (str, unicode))
+        is_null = lambda o: len(o) == revision.count('0')
+
+        if len(self.revisions) == 0:
+            raise EmptyRepositoryError("There are no changesets yet")
+
+        if revision in (None, '', 'tip', 'HEAD', 'head', -1):
+            revision = self.revisions[-1]
+
+        if ((is_bstr(revision) and revision.isdigit() and len(revision) < 12)
+            or isinstance(revision, int) or is_null(revision)):
+            try:
+                revision = self.revisions[int(revision)]
+            except:
+                raise ChangesetDoesNotExistError("Revision %r does not exist "
+                    "for this repository %s" % (revision, self))
+
+        elif is_bstr(revision):
+            if not pattern.match(revision) or revision not in self.revisions:
+                raise ChangesetDoesNotExistError("Revision %r does not exist "
+                    "for this repository %s" % (revision, self))
+
+        # Ensure we return full id
+        if not pattern.match(str(revision)):
+            raise ChangesetDoesNotExistError("Given revision %r not recognized"
+                % revision)
+        return revision
+
+    def _get_archives(self, archive_name='tip'):
+
+        for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]:
+                yield {"type": i[0], "extension": i[1], "node": archive_name}
+
+    def _get_url(self, url):
+        """
+        Returns normalized url. If schema is not given, would fall to
+        filesystem (``file:///``) schema.
+        """
+        url = str(url)
+        if url != 'default' and not '://' in url:
+            url = ':///'.join(('file', url))
+        return url
+
+    @LazyProperty
+    def name(self):
+        return os.path.basename(self.path)
+
+    @LazyProperty
+    def last_change(self):
+        """
+        Returns last change made on this repository as datetime object
+        """
+        return date_fromtimestamp(self._get_mtime(), makedate()[1])
+
+    def _get_mtime(self):
+        try:
+            return time.mktime(self.get_changeset().date.timetuple())
+        except RepositoryError:
+            # fallback to filesystem
+            in_path = os.path.join(self.path, '.git', "index")
+            he_path = os.path.join(self.path, '.git', "HEAD")
+            if os.path.exists(in_path):
+                return os.stat(in_path).st_mtime
+            else:
+                return os.stat(he_path).st_mtime
+
+    @LazyProperty
+    def description(self):
+        undefined_description = u'unknown'
+        description_path = os.path.join(self.path, '.git', 'description')
+        if os.path.isfile(description_path):
+            return safe_unicode(open(description_path).read())
+        else:
+            return undefined_description
+
+    @LazyProperty
+    def contact(self):
+        undefined_contact = u'Unknown'
+        return undefined_contact
+
+    @property
+    def branches(self):
+        if not self.revisions:
+            return {}
+        refs = self._repo.refs.as_dict()
+        sortkey = lambda ctx: ctx[0]
+        _branches = [('/'.join(ref.split('/')[2:]), head)
+            for ref, head in refs.items()
+            if ref.startswith('refs/heads/') or
+            ref.startswith('refs/remotes/') and not ref.endswith('/HEAD')]
+        return OrderedDict(sorted(_branches, key=sortkey, reverse=False))
+
+    def _get_tags(self):
+        if not self.revisions:
+            return {}
+        sortkey = lambda ctx: ctx[0]
+        _tags = [('/'.join(ref.split('/')[2:]), head) for ref, head in
+            self._repo.get_refs().items() if ref.startswith('refs/tags/')]
+        return OrderedDict(sorted(_tags, key=sortkey, reverse=True))
+
+    @LazyProperty
+    def tags(self):
+        return self._get_tags()
+
+    def tag(self, name, user, revision=None, message=None, date=None,
+            **kwargs):
+        """
+        Creates and returns a tag for the given ``revision``.
+
+        :param name: name for new tag
+        :param user: full username, i.e.: "Joe Doe <[e-mail]>"
+        :param revision: changeset id for which new tag would be created
+        :param message: message of the tag's commit
+        :param date: date of tag's commit
+
+        :raises TagAlreadyExistError: if tag with same name already exists
+        """
+        if name in self.tags:
+            raise TagAlreadyExistError("Tag %s already exists" % name)
+        changeset = self.get_changeset(revision)
+        message = message or "Added tag %s for commit %s" % (name,
+            changeset.raw_id)
+        self._repo.refs["refs/tags/%s" % name] = changeset._commit.id
+
+        self.tags = self._get_tags()
+        return changeset
+
+    def remove_tag(self, name, user, message=None, date=None):
+        """
+        Removes tag with the given ``name``.
+
+        :param name: name of the tag to be removed
+        :param user: full username, i.e.: "Joe Doe <[e-mail]>"
+        :param message: message of the tag's removal commit
+        :param date: date of tag's removal commit
+
+        :raises TagDoesNotExistError: if tag with given name does not exists
+        """
+        if name not in self.tags:
+            raise TagDoesNotExistError("Tag %s does not exist" % name)
+        tagpath = posixpath.join(self._repo.refs.path, 'refs', 'tags', name)
+        try:
+            os.remove(tagpath)
+            self.tags = self._get_tags()
+        except OSError, e:
+            raise RepositoryError(e.strerror)
+
+    def get_changeset(self, revision=None):
+        """
+        Returns ``GitChangeset`` object representing commit from git repository
+        at the given revision or head (most recent commit) if None given.
+        """
+        if isinstance(revision, GitChangeset):
+            return revision
+        revision = self._get_revision(revision)
+        changeset = GitChangeset(repository=self, revision=revision)
+        return changeset
+
+    def get_changesets(self, start=None, end=None, start_date=None,
+           end_date=None, branch_name=None, reverse=False):
+        """
+        Returns iterator of ``GitChangeset`` objects from start to end (both
+        are inclusive), in ascending date order (unless ``reverse`` is set).
+
+        :param start: changeset ID, as str; first returned changeset
+        :param end: changeset ID, as str; last returned changeset
+        :param start_date: if specified, changesets with commit date less than
+          ``start_date`` would be filtered out from returned set
+        :param end_date: if specified, changesets with commit date greater than
+          ``end_date`` would be filtered out from returned set
+        :param branch_name: if specified, changesets not reachable from given
+          branch would be filtered out from returned set
+        :param reverse: if ``True``, returned generator would be reversed
+          (meaning that returned changesets would have descending date order)
+
+        :raise BranchDoesNotExistError: If given ``branch_name`` does not
+            exist.
+        :raise ChangesetDoesNotExistError: If changeset for given ``start`` or
+          ``end`` could not be found.
+
+        """
+        if branch_name and branch_name not in self.branches:
+            raise BranchDoesNotExistError("Branch '%s' not found" \
+                                          % branch_name)
+        # %H at format means (full) commit hash, initial hashes are retrieved
+        # in ascending date order
+        cmd_template = 'log --date-order --reverse --pretty=format:"%H"'
+        cmd_params = {}
+        if start_date:
+            cmd_template += ' --since "$since"'
+            cmd_params['since'] = start_date.strftime('%m/%d/%y %H:%M:%S')
+        if end_date:
+            cmd_template += ' --until "$until"'
+            cmd_params['until'] = end_date.strftime('%m/%d/%y %H:%M:%S')
+        if branch_name:
+            cmd_template += ' $branch_name'
+            cmd_params['branch_name'] = branch_name
+        else:
+            cmd_template += ' --all'
+
+        cmd = Template(cmd_template).safe_substitute(**cmd_params)
+        revs = self.run_git_command(cmd)[0].splitlines()
+        start_pos = 0
+        end_pos = len(revs)
+        if start:
+            _start = self._get_revision(start)
+            try:
+                start_pos = revs.index(_start)
+            except ValueError:
+                pass
+
+        if end is not None:
+            _end = self._get_revision(end)
+            try:
+                end_pos = revs.index(_end)
+            except ValueError:
+                pass
+
+        if None not in [start, end] and start_pos > end_pos:
+            raise RepositoryError('start cannot be after end')
+
+        if end_pos is not None:
+            end_pos += 1
+
+        revs = revs[start_pos:end_pos]
+        if reverse:
+            revs = reversed(revs)
+        for rev in revs:
+            yield self.get_changeset(rev)
+
+    def get_diff(self, rev1, rev2, path=None, ignore_whitespace=False,
+            context=3):
+        """
+        Returns (git like) *diff*, as plain text. Shows changes introduced by
+        ``rev2`` since ``rev1``.
+
+        :param rev1: Entry point from which diff is shown. Can be
+          ``self.EMPTY_CHANGESET`` - in this case, patch showing all
+          the changes since empty state of the repository until ``rev2``
+        :param rev2: Until which revision changes should be shown.
+        :param ignore_whitespace: If set to ``True``, would not show whitespace
+          changes. Defaults to ``False``.
+        :param context: How many lines before/after changed lines should be
+          shown. Defaults to ``3``.
+        """
+        flags = ['-U%s' % context]
+        if ignore_whitespace:
+            flags.append('-w')
+
+        if rev1 == self.EMPTY_CHANGESET:
+            rev2 = self.get_changeset(rev2).raw_id
+            cmd = ' '.join(['show'] + flags + [rev2])
+        else:
+            rev1 = self.get_changeset(rev1).raw_id
+            rev2 = self.get_changeset(rev2).raw_id
+            cmd = ' '.join(['diff'] + flags + [rev1, rev2])
+
+        if path:
+            cmd += ' -- "%s"' % path
+        stdout, stderr = self.run_git_command(cmd)
+        # If we used 'show' command, strip first few lines (until actual diff
+        # starts)
+        if rev1 == self.EMPTY_CHANGESET:
+            lines = stdout.splitlines()
+            x = 0
+            for line in lines:
+                if line.startswith('diff'):
+                    break
+                x += 1
+            # Append new line just like 'diff' command do
+            stdout = '\n'.join(lines[x:]) + '\n'
+        return stdout
+
+    @LazyProperty
+    def in_memory_changeset(self):
+        """
+        Returns ``GitInMemoryChangeset`` object for this repository.
+        """
+        return GitInMemoryChangeset(self)
+
+    def clone(self, url, update_after_clone=True, bare=False):
+        """
+        Tries to clone changes from external location.
+
+        :param update_after_clone: If set to ``False``, git won't checkout
+          working directory
+        :param bare: If set to ``True``, repository would be cloned into
+          *bare* git repository (no working directory at all).
+        """
+        url = self._get_url(url)
+        cmd = ['clone']
+        if bare:
+            cmd.append('--bare')
+        elif not update_after_clone:
+            cmd.append('--no-checkout')
+        cmd += ['--', '"%s"' % url, '"%s"' % self.path]
+        cmd = ' '.join(cmd)
+        # If error occurs run_git_command raises RepositoryError already
+        self.run_git_command(cmd)
+
+    @LazyProperty
+    def workdir(self):
+        """
+        Returns ``Workdir`` instance for this repository.
+        """
+        return GitWorkdir(self)
+
+    def get_config_value(self, section, name, config_file=None):
+        """
+        Returns configuration value for a given [``section``] and ``name``.
+
+        :param section: Section we want to retrieve value from
+        :param name: Name of configuration we want to retrieve
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        if config_file is None:
+            config_file = []
+        elif isinstance(config_file, basestring):
+            config_file = [config_file]
+
+        def gen_configs():
+            for path in config_file + self._config_files:
+                try:
+                    yield ConfigFile.from_path(path)
+                except (IOError, OSError, ValueError):
+                    continue
+
+        for config in gen_configs():
+            try:
+                return config.get(section, name)
+            except KeyError:
+                continue
+        return None
+
+    def get_user_name(self, config_file=None):
+        """
+        Returns user's name from global configuration file.
+
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        return self.get_config_value('user', 'name', config_file)
+
+    def get_user_email(self, config_file=None):
+        """
+        Returns user's email from global configuration file.
+
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        return self.get_config_value('user', 'email', config_file)

rhodecode/lib/vcs/backends/git/workdir.py

@@ -0,0 +1,31 b''
+import re
+from rhodecode.lib.vcs.backends.base import BaseWorkdir
+from rhodecode.lib.vcs.exceptions import RepositoryError
+from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError
+
+
+class GitWorkdir(BaseWorkdir):
+
+    def get_branch(self):
+        headpath = self.repository._repo.refs.refpath('HEAD')
+        try:
+            content = open(headpath).read()
+            match = re.match(r'^ref: refs/heads/(?P<branch>.+)\n$', content)
+            if match:
+                return match.groupdict()['branch']
+            else:
+                raise RepositoryError("Couldn't compute workdir's branch")
+        except IOError:
+            # Try naive way...
+            raise RepositoryError("Couldn't compute workdir's branch")
+
+    def get_changeset(self):
+        return self.repository.get_changeset(
+            self.repository._repo.refs.as_dict().get('HEAD'))
+
+    def checkout_branch(self, branch=None):
+        if branch is None:
+            branch = self.repository.DEFAULT_BRANCH_NAME
+        if branch not in self.repository.branches:
+            raise BranchDoesNotExistError
+        self.repository.run_git_command(['checkout', branch])

rhodecode/lib/vcs/backends/hg/__init__.py

@@ -0,0 +1,21 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.backends.hg
+    ~~~~~~~~~~~~~~~~
+
+    Mercurial backend implementation.
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+
+from .repository import MercurialRepository
+from .changeset import MercurialChangeset
+from .inmemory import MercurialInMemoryChangeset
+from .workdir import MercurialWorkdir
+
+
+__all__ = [
+    'MercurialRepository', 'MercurialChangeset',
+    'MercurialInMemoryChangeset', 'MercurialWorkdir',
+]

rhodecode/lib/vcs/backends/hg/changeset.py

@@ -0,0 +1,338 b''
+import os
+import posixpath
+
+from rhodecode.lib.vcs.backends.base import BaseChangeset
+from rhodecode.lib.vcs.conf import settings
+from rhodecode.lib.vcs.exceptions import  ChangesetDoesNotExistError, \
+    ChangesetError, ImproperArchiveTypeError, NodeDoesNotExistError, VCSError
+from rhodecode.lib.vcs.nodes import AddedFileNodesGenerator, ChangedFileNodesGenerator, \
+    DirNode, FileNode, NodeKind, RemovedFileNodesGenerator, RootNode
+
+from rhodecode.lib.vcs.utils import safe_str, safe_unicode, date_fromtimestamp
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.paths import get_dirs_for_path
+
+from ...utils.hgcompat import archival, hex
+
+
+class MercurialChangeset(BaseChangeset):
+    """
+    Represents state of the repository at the single revision.
+    """
+
+    def __init__(self, repository, revision):
+        self.repository = repository
+        self.raw_id = revision
+        self._ctx = repository._repo[revision]
+        self.revision = self._ctx._rev
+        self.nodes = {}
+
+    @LazyProperty
+    def tags(self):
+        return map(safe_unicode, self._ctx.tags())
+
+    @LazyProperty
+    def branch(self):
+        return  safe_unicode(self._ctx.branch())
+
+    @LazyProperty
+    def message(self):
+        return safe_unicode(self._ctx.description())
+
+    @LazyProperty
+    def author(self):
+        return safe_unicode(self._ctx.user())
+
+    @LazyProperty
+    def date(self):
+        return date_fromtimestamp(*self._ctx.date())
+
+    @LazyProperty
+    def status(self):
+        """
+        Returns modified, added, removed, deleted files for current changeset
+        """
+        return self.repository._repo.status(self._ctx.p1().node(),
+                                            self._ctx.node())
+
+    @LazyProperty
+    def _file_paths(self):
+        return list(self._ctx)
+
+    @LazyProperty
+    def _dir_paths(self):
+        p = list(set(get_dirs_for_path(*self._file_paths)))
+        p.insert(0, '')
+        return p
+
+    @LazyProperty
+    def _paths(self):
+        return self._dir_paths + self._file_paths
+
+    @LazyProperty
+    def id(self):
+        if self.last:
+            return u'tip'
+        return self.short_id
+
+    @LazyProperty
+    def short_id(self):
+        return self.raw_id[:12]
+
+    @LazyProperty
+    def parents(self):
+        """
+        Returns list of parents changesets.
+        """
+        return [self.repository.get_changeset(parent.rev())
+                for parent in self._ctx.parents() if parent.rev() >= 0]
+
+    def next(self, branch=None):
+
+        if branch and self.branch != branch:
+            raise VCSError('Branch option used on changeset not belonging '
+                           'to that branch')
+
+        def _next(changeset, branch):
+            try:
+                next_ = changeset.revision + 1
+                next_rev = changeset.repository.revisions[next_]
+            except IndexError:
+                raise ChangesetDoesNotExistError
+            cs = changeset.repository.get_changeset(next_rev)
+
+            if branch and branch != cs.branch:
+                return _next(cs, branch)
+
+            return cs
+
+        return _next(self, branch)
+
+    def prev(self, branch=None):
+        if branch and self.branch != branch:
+            raise VCSError('Branch option used on changeset not belonging '
+                           'to that branch')
+
+        def _prev(changeset, branch):
+            try:
+                prev_ = changeset.revision - 1
+                if prev_ < 0:
+                    raise IndexError
+                prev_rev = changeset.repository.revisions[prev_]
+            except IndexError:
+                raise ChangesetDoesNotExistError
+
+            cs = changeset.repository.get_changeset(prev_rev)
+
+            if branch and branch != cs.branch:
+                return _prev(cs, branch)
+
+            return cs
+
+        return _prev(self, branch)
+
+    def _fix_path(self, path):
+        """
+        Paths are stored without trailing slash so we need to get rid off it if
+        needed. Also mercurial keeps filenodes as str so we need to decode
+        from unicode to str
+        """
+        if path.endswith('/'):
+            path = path.rstrip('/')
+
+        return safe_str(path)
+
+    def _get_kind(self, path):
+        path = self._fix_path(path)
+        if path in self._file_paths:
+            return NodeKind.FILE
+        elif path in self._dir_paths:
+            return NodeKind.DIR
+        else:
+            raise ChangesetError("Node does not exist at the given path %r"
+                % (path))
+
+    def _get_filectx(self, path):
+        path = self._fix_path(path)
+        if self._get_kind(path) != NodeKind.FILE:
+            raise ChangesetError("File does not exist for revision %r at "
+                " %r" % (self.revision, path))
+        return self._ctx.filectx(path)
+
+    def get_file_mode(self, path):
+        """
+        Returns stat mode of the file at the given ``path``.
+        """
+        fctx = self._get_filectx(path)
+        if 'x' in fctx.flags():
+            return 0100755
+        else:
+            return 0100644
+
+    def get_file_content(self, path):
+        """
+        Returns content of the file at given ``path``.
+        """
+        fctx = self._get_filectx(path)
+        return fctx.data()
+
+    def get_file_size(self, path):
+        """
+        Returns size of the file at given ``path``.
+        """
+        fctx = self._get_filectx(path)
+        return fctx.size()
+
+    def get_file_changeset(self, path):
+        """
+        Returns last commit of the file at the given ``path``.
+        """
+        fctx = self._get_filectx(path)
+        changeset = self.repository.get_changeset(fctx.linkrev())
+        return changeset
+
+    def get_file_history(self, path):
+        """
+        Returns history of file as reversed list of ``Changeset`` objects for
+        which file at given ``path`` has been modified.
+        """
+        fctx = self._get_filectx(path)
+        nodes = [fctx.filectx(x).node() for x in fctx.filelog()]
+        changesets = [self.repository.get_changeset(hex(node))
+            for node in reversed(nodes)]
+        return changesets
+
+    def get_file_annotate(self, path):
+        """
+        Returns a list of three element tuples with lineno,changeset and line
+        """
+        fctx = self._get_filectx(path)
+        annotate = []
+        for i, annotate_data in enumerate(fctx.annotate()):
+            ln_no = i + 1
+            annotate.append((ln_no, self.repository\
+                             .get_changeset(hex(annotate_data[0].node())),
+                             annotate_data[1],))
+
+        return annotate
+
+    def fill_archive(self, stream=None, kind='tgz', prefix=None,
+                     subrepos=False):
+        """
+        Fills up given stream.
+
+        :param stream: file like object.
+        :param kind: one of following: ``zip``, ``tgz`` or ``tbz2``.
+            Default: ``tgz``.
+        :param prefix: name of root directory in archive.
+            Default is repository name and changeset's raw_id joined with dash
+            (``repo-tip.<KIND>``).
+        :param subrepos: include subrepos in this archive.
+
+        :raise ImproperArchiveTypeError: If given kind is wrong.
+        :raise VcsError: If given stream is None
+        """
+
+        allowed_kinds = settings.ARCHIVE_SPECS.keys()
+        if kind not in allowed_kinds:
+            raise ImproperArchiveTypeError('Archive kind not supported use one'
+                'of %s', allowed_kinds)
+
+        if stream is None:
+            raise VCSError('You need to pass in a valid stream for filling'
+                           ' with archival data')
+
+        if prefix is None:
+            prefix = '%s-%s' % (self.repository.name, self.short_id)
+        elif prefix.startswith('/'):
+            raise VCSError("Prefix cannot start with leading slash")
+        elif prefix.strip() == '':
+            raise VCSError("Prefix cannot be empty")
+
+        archival.archive(self.repository._repo, stream, self.raw_id,
+                         kind, prefix=prefix, subrepos=subrepos)
+
+        #stream.close()
+
+        if stream.closed and hasattr(stream, 'name'):
+            stream = open(stream.name, 'rb')
+        elif hasattr(stream, 'mode') and 'r' not in stream.mode:
+            stream = open(stream.name, 'rb')
+        else:
+            stream.seek(0)
+
+    def get_nodes(self, path):
+        """
+        Returns combined ``DirNode`` and ``FileNode`` objects list representing
+        state of changeset at the given ``path``. If node at the given ``path``
+        is not instance of ``DirNode``, ChangesetError would be raised.
+        """
+
+        if self._get_kind(path) != NodeKind.DIR:
+            raise ChangesetError("Directory does not exist for revision %r at "
+                " %r" % (self.revision, path))
+        path = self._fix_path(path)
+        filenodes = [FileNode(f, changeset=self) for f in self._file_paths
+            if os.path.dirname(f) == path]
+        dirs = path == '' and '' or [d for d in self._dir_paths
+            if d and posixpath.dirname(d) == path]
+        dirnodes = [DirNode(d, changeset=self) for d in dirs
+            if os.path.dirname(d) == path]
+        nodes = dirnodes + filenodes
+        # cache nodes
+        for node in nodes:
+            self.nodes[node.path] = node
+        nodes.sort()
+        return nodes
+
+    def get_node(self, path):
+        """
+        Returns ``Node`` object from the given ``path``. If there is no node at
+        the given ``path``, ``ChangesetError`` would be raised.
+        """
+
+        path = self._fix_path(path)
+
+        if not path in self.nodes:
+            if path in self._file_paths:
+                node = FileNode(path, changeset=self)
+            elif path in self._dir_paths or path in self._dir_paths:
+                if path == '':
+                    node = RootNode(changeset=self)
+                else:
+                    node = DirNode(path, changeset=self)
+            else:
+                raise NodeDoesNotExistError("There is no file nor directory "
+                    "at the given path: %r at revision %r"
+                    % (path, self.short_id))
+            # cache node
+            self.nodes[path] = node
+        return self.nodes[path]
+
+    @LazyProperty
+    def affected_files(self):
+        """
+        Get's a fast accessible file changes for given changeset
+        """
+        return self._ctx.files()
+
+    @property
+    def added(self):
+        """
+        Returns list of added ``FileNode`` objects.
+        """
+        return AddedFileNodesGenerator([n for n in self.status[1]], self)
+
+    @property
+    def changed(self):
+        """
+        Returns list of modified ``FileNode`` objects.
+        """
+        return ChangedFileNodesGenerator([n for n in  self.status[0]], self)
+
+    @property
+    def removed(self):
+        """
+        Returns list of removed ``FileNode`` objects.
+        """
+        return RemovedFileNodesGenerator([n for n in self.status[2]], self)

rhodecode/lib/vcs/backends/hg/inmemory.py

@@ -0,0 +1,110 b''
+import datetime
+import errno
+
+from rhodecode.lib.vcs.backends.base import BaseInMemoryChangeset
+from rhodecode.lib.vcs.exceptions import RepositoryError
+
+from ...utils.hgcompat import memfilectx, memctx, hex
+
+
+class MercurialInMemoryChangeset(BaseInMemoryChangeset):
+
+    def commit(self, message, author, parents=None, branch=None, date=None,
+            **kwargs):
+        """
+        Performs in-memory commit (doesn't check workdir in any way) and
+        returns newly created ``Changeset``. Updates repository's
+        ``revisions``.
+
+        :param message: message of the commit
+        :param author: full username, i.e. "Joe Doe <[e-mail]>"
+        :param parents: single parent or sequence of parents from which commit
+          would be derieved
+        :param date: ``datetime.datetime`` instance. Defaults to
+          ``datetime.datetime.now()``.
+        :param branch: branch name, as string. If none given, default backend's
+          branch would be used.
+
+        :raises ``CommitError``: if any error occurs while committing
+        """
+        self.check_integrity(parents)
+
+        from .repository import MercurialRepository
+        if not isinstance(message, str) or not isinstance(author, str):
+            raise RepositoryError('Given message and author needs to be '
+                                  'an <str> instance')
+
+        if branch is None:
+            branch = MercurialRepository.DEFAULT_BRANCH_NAME
+        kwargs['branch'] = branch
+
+        def filectxfn(_repo, memctx, path):
+            """
+            Marks given path as added/changed/removed in a given _repo. This is
+            for internal mercurial commit function.
+            """
+
+            # check if this path is removed
+            if path in (node.path for node in self.removed):
+                # Raising exception is a way to mark node for removal
+                raise IOError(errno.ENOENT, '%s is deleted' % path)
+
+            # check if this path is added
+            for node in self.added:
+                if node.path == path:
+                    return memfilectx(path=node.path,
+                        data=(node.content.encode('utf8')
+                              if not node.is_binary else node.content),
+                        islink=False,
+                        isexec=node.is_executable,
+                        copied=False)
+
+            # or changed
+            for node in self.changed:
+                if node.path == path:
+                    return memfilectx(path=node.path,
+                        data=(node.content.encode('utf8')
+                              if not node.is_binary else node.content),
+                        islink=False,
+                        isexec=node.is_executable,
+                        copied=False)
+
+            raise RepositoryError("Given path haven't been marked as added,"
+                "changed or removed (%s)" % path)
+
+        parents = [None, None]
+        for i, parent in enumerate(self.parents):
+            if parent is not None:
+                parents[i] = parent._ctx.node()
+
+        if date and isinstance(date, datetime.datetime):
+            date = date.ctime()
+
+        commit_ctx = memctx(repo=self.repository._repo,
+            parents=parents,
+            text='',
+            files=self.get_paths(),
+            filectxfn=filectxfn,
+            user=author,
+            date=date,
+            extra=kwargs)
+
+        # injecting given _repo params
+        commit_ctx._text = message
+        commit_ctx._user = author
+        commit_ctx._date = date
+
+        # TODO: Catch exceptions!
+        n = self.repository._repo.commitctx(commit_ctx)
+        # Returns mercurial node
+        self._commit_ctx = commit_ctx  # For reference
+        # Update vcs repository object & recreate mercurial _repo
+        # new_ctx = self.repository._repo[node]
+        # new_tip = self.repository.get_changeset(new_ctx.hex())
+        new_id = hex(n)
+        self.repository.revisions.append(new_id)
+        self._repo = self.repository._get_repo(create=False)
+        self.repository.branches = self.repository._get_branches()
+        tip = self.repository.get_changeset()
+        self.reset()
+        return tip

rhodecode/lib/vcs/backends/hg/repository.py

@@ -0,0 +1,521 b''
+import os
+import time
+import datetime
+import urllib
+import urllib2
+
+from rhodecode.lib.vcs.backends.base import BaseRepository
+from .workdir import MercurialWorkdir
+from .changeset import MercurialChangeset
+from .inmemory import MercurialInMemoryChangeset
+
+from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError, \
+    ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, \
+    VCSError, TagAlreadyExistError, TagDoesNotExistError
+from rhodecode.lib.vcs.utils import author_email, author_name, date_fromtimestamp, \
+    makedate, safe_unicode
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.ordered_dict import OrderedDict
+from rhodecode.lib.vcs.utils.paths import abspath
+
+from ...utils.hgcompat import ui, nullid, match, patch, diffopts, clone, \
+    get_contact, pull, localrepository, RepoLookupError, Abort, RepoError, hex
+
+
+class MercurialRepository(BaseRepository):
+    """
+    Mercurial repository backend
+    """
+    DEFAULT_BRANCH_NAME = 'default'
+    scm = 'hg'
+
+    def __init__(self, repo_path, create=False, baseui=None, src_url=None,
+                 update_after_clone=False):
+        """
+        Raises RepositoryError if repository could not be find at the given
+        ``repo_path``.
+
+        :param repo_path: local path of the repository
+        :param create=False: if set to True, would try to create repository if
+           it does not exist rather than raising exception
+        :param baseui=None: user data
+        :param src_url=None: would try to clone repository from given location
+        :param update_after_clone=False: sets update of working copy after
+          making a clone
+        """
+
+        if not isinstance(repo_path, str):
+            raise VCSError('Mercurial backend requires repository path to '
+                           'be instance of <str> got %s instead' %
+                           type(repo_path))
+
+        self.path = abspath(repo_path)
+        self.baseui = baseui or ui.ui()
+        # We've set path and ui, now we can set _repo itself
+        self._repo = self._get_repo(create, src_url, update_after_clone)
+
+    @property
+    def _empty(self):
+        """
+        Checks if repository is empty without any changesets
+        """
+        # TODO: Following raises errors when using InMemoryChangeset...
+        # return len(self._repo.changelog) == 0
+        return len(self.revisions) == 0
+
+    @LazyProperty
+    def revisions(self):
+        """
+        Returns list of revisions' ids, in ascending order.  Being lazy
+        attribute allows external tools to inject shas from cache.
+        """
+        return self._get_all_revisions()
+
+    @LazyProperty
+    def name(self):
+        return os.path.basename(self.path)
+
+    @LazyProperty
+    def branches(self):
+        return self._get_branches()
+
+    def _get_branches(self, closed=False):
+        """
+        Get's branches for this repository
+        Returns only not closed branches by default
+
+        :param closed: return also closed branches for mercurial
+        """
+
+        if self._empty:
+            return {}
+
+        def _branchtags(localrepo):
+            """
+            Patched version of mercurial branchtags to not return the closed
+            branches
+
+            :param localrepo: locarepository instance
+            """
+
+            bt = {}
+            bt_closed = {}
+            for bn, heads in localrepo.branchmap().iteritems():
+                tip = heads[-1]
+                if 'close' in localrepo.changelog.read(tip)[5]:
+                    bt_closed[bn] = tip
+                else:
+                    bt[bn] = tip
+
+            if closed:
+                bt.update(bt_closed)
+            return bt
+
+        sortkey = lambda ctx: ctx[0]  # sort by name
+        _branches = [(safe_unicode(n), hex(h),) for n, h in
+                     _branchtags(self._repo).items()]
+
+        return OrderedDict(sorted(_branches, key=sortkey, reverse=False))
+
+    @LazyProperty
+    def tags(self):
+        """
+        Get's tags for this repository
+        """
+        return self._get_tags()
+
+    def _get_tags(self):
+        if self._empty:
+            return {}
+
+        sortkey = lambda ctx: ctx[0]  # sort by name
+        _tags = [(safe_unicode(n), hex(h),) for n, h in
+                 self._repo.tags().items()]
+
+        return OrderedDict(sorted(_tags, key=sortkey, reverse=True))
+
+    def tag(self, name, user, revision=None, message=None, date=None,
+            **kwargs):
+        """
+        Creates and returns a tag for the given ``revision``.
+
+        :param name: name for new tag
+        :param user: full username, i.e.: "Joe Doe <[e-mail]>"
+        :param revision: changeset id for which new tag would be created
+        :param message: message of the tag's commit
+        :param date: date of tag's commit
+
+        :raises TagAlreadyExistError: if tag with same name already exists
+        """
+        if name in self.tags:
+            raise TagAlreadyExistError("Tag %s already exists" % name)
+        changeset = self.get_changeset(revision)
+        local = kwargs.setdefault('local', False)
+
+        if message is None:
+            message = "Added tag %s for changeset %s" % (name,
+                changeset.short_id)
+
+        if date is None:
+            date = datetime.datetime.now().ctime()
+
+        try:
+            self._repo.tag(name, changeset._ctx.node(), message, local, user,
+                date)
+        except Abort, e:
+            raise RepositoryError(e.message)
+
+        # Reinitialize tags
+        self.tags = self._get_tags()
+        tag_id = self.tags[name]
+
+        return self.get_changeset(revision=tag_id)
+
+    def remove_tag(self, name, user, message=None, date=None):
+        """
+        Removes tag with the given ``name``.
+
+        :param name: name of the tag to be removed
+        :param user: full username, i.e.: "Joe Doe <[e-mail]>"
+        :param message: message of the tag's removal commit
+        :param date: date of tag's removal commit
+
+        :raises TagDoesNotExistError: if tag with given name does not exists
+        """
+        if name not in self.tags:
+            raise TagDoesNotExistError("Tag %s does not exist" % name)
+        if message is None:
+            message = "Removed tag %s" % name
+        if date is None:
+            date = datetime.datetime.now().ctime()
+        local = False
+
+        try:
+            self._repo.tag(name, nullid, message, local, user, date)
+            self.tags = self._get_tags()
+        except Abort, e:
+            raise RepositoryError(e.message)
+
+    @LazyProperty
+    def bookmarks(self):
+        """
+        Get's bookmarks for this repository
+        """
+        return self._get_bookmarks()
+
+    def _get_bookmarks(self):
+        if self._empty:
+            return {}
+
+        sortkey = lambda ctx: ctx[0]  # sort by name
+        _bookmarks = [(safe_unicode(n), hex(h),) for n, h in
+                 self._repo._bookmarks.items()]
+        return OrderedDict(sorted(_bookmarks, key=sortkey, reverse=True))
+
+    def _get_all_revisions(self):
+
+        return map(lambda x: hex(x[7]), self._repo.changelog.index)[:-1]
+
+    def get_diff(self, rev1, rev2, path='', ignore_whitespace=False,
+                  context=3):
+        """
+        Returns (git like) *diff*, as plain text. Shows changes introduced by
+        ``rev2`` since ``rev1``.
+
+        :param rev1: Entry point from which diff is shown. Can be
+          ``self.EMPTY_CHANGESET`` - in this case, patch showing all
+          the changes since empty state of the repository until ``rev2``
+        :param rev2: Until which revision changes should be shown.
+        :param ignore_whitespace: If set to ``True``, would not show whitespace
+          changes. Defaults to ``False``.
+        :param context: How many lines before/after changed lines should be
+          shown. Defaults to ``3``.
+        """
+        # Check if given revisions are present at repository (may raise
+        # ChangesetDoesNotExistError)
+        if rev1 != self.EMPTY_CHANGESET:
+            self.get_changeset(rev1)
+        self.get_changeset(rev2)
+
+        file_filter = match(self.path, '', [path])
+        return ''.join(patch.diff(self._repo, rev1, rev2, match=file_filter,
+                          opts=diffopts(git=True,
+                                        ignorews=ignore_whitespace,
+                                        context=context)))
+
+    def _check_url(self, url):
+        """
+        Function will check given url and try to verify if it's a valid
+        link. Sometimes it may happened that mercurial will issue basic
+        auth request that can cause whole API to hang when used from python
+        or other external calls.
+
+        On failures it'll raise urllib2.HTTPError, return code 200 if url
+        is valid or True if it's a local path
+        """
+
+        from mercurial.util import url as Url
+
+        # those authnadlers are patched for python 2.6.5 bug an
+        # infinit looping when given invalid resources
+        from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
+
+        # check first if it's not an local url
+        if os.path.isdir(url) or url.startswith('file:'):
+            return True
+
+        handlers = []
+        test_uri, authinfo = Url(url).authinfo()
+
+        if authinfo:
+            #create a password manager
+            passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+            passmgr.add_password(*authinfo)
+
+            handlers.extend((httpbasicauthhandler(passmgr),
+                             httpdigestauthhandler(passmgr)))
+
+        o = urllib2.build_opener(*handlers)
+        o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
+                        ('Accept', 'application/mercurial-0.1')]
+
+        q = {"cmd": 'between'}
+        q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
+        qs = '?%s' % urllib.urlencode(q)
+        cu = "%s%s" % (test_uri, qs)
+        req = urllib2.Request(cu, None, {})
+
+        try:
+            resp = o.open(req)
+            return resp.code == 200
+        except Exception, e:
+            # means it cannot be cloned
+            raise urllib2.URLError(e)
+
+    def _get_repo(self, create, src_url=None, update_after_clone=False):
+        """
+        Function will check for mercurial repository in given path and return
+        a localrepo object. If there is no repository in that path it will
+        raise an exception unless ``create`` parameter is set to True - in
+        that case repository would be created and returned.
+        If ``src_url`` is given, would try to clone repository from the
+        location at given clone_point. Additionally it'll make update to
+        working copy accordingly to ``update_after_clone`` flag
+        """
+        try:
+            if src_url:
+                url = str(self._get_url(src_url))
+                opts = {}
+                if not update_after_clone:
+                    opts.update({'noupdate': True})
+                try:
+                    self._check_url(url)
+                    clone(self.baseui, url, self.path, **opts)
+#                except urllib2.URLError:
+#                    raise Abort("Got HTTP 404 error")
+                except Exception:
+                    raise
+                # Don't try to create if we've already cloned repo
+                create = False
+            return localrepository(self.baseui, self.path, create=create)
+        except (Abort, RepoError), err:
+            if create:
+                msg = "Cannot create repository at %s. Original error was %s"\
+                    % (self.path, err)
+            else:
+                msg = "Not valid repository at %s. Original error was %s"\
+                    % (self.path, err)
+            raise RepositoryError(msg)
+
+    @LazyProperty
+    def in_memory_changeset(self):
+        return MercurialInMemoryChangeset(self)
+
+    @LazyProperty
+    def description(self):
+        undefined_description = u'unknown'
+        return safe_unicode(self._repo.ui.config('web', 'description',
+                                   undefined_description, untrusted=True))
+
+    @LazyProperty
+    def contact(self):
+        undefined_contact = u'Unknown'
+        return safe_unicode(get_contact(self._repo.ui.config)
+                            or undefined_contact)
+
+    @LazyProperty
+    def last_change(self):
+        """
+        Returns last change made on this repository as datetime object
+        """
+        return date_fromtimestamp(self._get_mtime(), makedate()[1])
+
+    def _get_mtime(self):
+        try:
+            return time.mktime(self.get_changeset().date.timetuple())
+        except RepositoryError:
+            #fallback to filesystem
+            cl_path = os.path.join(self.path, '.hg', "00changelog.i")
+            st_path = os.path.join(self.path, '.hg', "store")
+            if os.path.exists(cl_path):
+                return os.stat(cl_path).st_mtime
+            else:
+                return os.stat(st_path).st_mtime
+
+    def _get_hidden(self):
+        return self._repo.ui.configbool("web", "hidden", untrusted=True)
+
+    def _get_revision(self, revision):
+        """
+        Get's an ID revision given as str. This will always return a fill
+        40 char revision number
+
+        :param revision: str or int or None
+        """
+
+        if self._empty:
+            raise EmptyRepositoryError("There are no changesets yet")
+
+        if revision in [-1, 'tip', None]:
+            revision = 'tip'
+
+        try:
+            revision = hex(self._repo.lookup(revision))
+        except (IndexError, ValueError, RepoLookupError, TypeError):
+            raise ChangesetDoesNotExistError("Revision %r does not "
+                                    "exist for this repository %s" \
+                                    % (revision, self))
+        return revision
+
+    def _get_archives(self, archive_name='tip'):
+        allowed = self.baseui.configlist("web", "allow_archive",
+                                         untrusted=True)
+        for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]:
+            if i[0] in allowed or self._repo.ui.configbool("web",
+                                                           "allow" + i[0],
+                                                           untrusted=True):
+                yield {"type": i[0], "extension": i[1], "node": archive_name}
+
+    def _get_url(self, url):
+        """
+        Returns normalized url. If schema is not given, would fall
+        to filesystem
+        (``file:///``) schema.
+        """
+        url = str(url)
+        if url != 'default' and not '://' in url:
+            url = "file:" + urllib.pathname2url(url)
+        return url
+
+    def get_changeset(self, revision=None):
+        """
+        Returns ``MercurialChangeset`` object representing repository's
+        changeset at the given ``revision``.
+        """
+        revision = self._get_revision(revision)
+        changeset = MercurialChangeset(repository=self, revision=revision)
+        return changeset
+
+    def get_changesets(self, start=None, end=None, start_date=None,
+                       end_date=None, branch_name=None, reverse=False):
+        """
+        Returns iterator of ``MercurialChangeset`` objects from start to end
+        (both are inclusive)
+
+        :param start: None, str, int or mercurial lookup format
+        :param end:  None, str, int or mercurial lookup format
+        :param start_date:
+        :param end_date:
+        :param branch_name:
+        :param reversed: return changesets in reversed order
+        """
+
+        start_raw_id = self._get_revision(start)
+        start_pos = self.revisions.index(start_raw_id) if start else None
+        end_raw_id = self._get_revision(end)
+        end_pos = self.revisions.index(end_raw_id) if end else None
+
+        if None not in [start, end] and start_pos > end_pos:
+            raise RepositoryError("start revision '%s' cannot be "
+                                  "after end revision '%s'" % (start, end))
+
+        if branch_name and branch_name not in self.branches.keys():
+            raise BranchDoesNotExistError('Such branch %s does not exists for'
+                                  ' this repository' % branch_name)
+        if end_pos is not None:
+            end_pos += 1
+
+        slice_ = reversed(self.revisions[start_pos:end_pos]) if reverse else \
+            self.revisions[start_pos:end_pos]
+
+        for id_ in slice_:
+            cs = self.get_changeset(id_)
+            if branch_name and cs.branch != branch_name:
+                continue
+            if start_date and cs.date < start_date:
+                continue
+            if end_date and cs.date > end_date:
+                continue
+
+            yield cs
+
+    def pull(self, url):
+        """
+        Tries to pull changes from external location.
+        """
+        url = self._get_url(url)
+        try:
+            pull(self.baseui, self._repo, url)
+        except Abort, err:
+            # Propagate error but with vcs's type
+            raise RepositoryError(str(err))
+
+    @LazyProperty
+    def workdir(self):
+        """
+        Returns ``Workdir`` instance for this repository.
+        """
+        return MercurialWorkdir(self)
+
+    def get_config_value(self, section, name, config_file=None):
+        """
+        Returns configuration value for a given [``section``] and ``name``.
+
+        :param section: Section we want to retrieve value from
+        :param name: Name of configuration we want to retrieve
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        if config_file is None:
+            config_file = []
+        elif isinstance(config_file, basestring):
+            config_file = [config_file]
+
+        config = self._repo.ui
+        for path in config_file:
+            config.readconfig(path)
+        return config.config(section, name)
+
+    def get_user_name(self, config_file=None):
+        """
+        Returns user's name from global configuration file.
+
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        username = self.get_config_value('ui', 'username')
+        if username:
+            return author_name(username)
+        return None
+
+    def get_user_email(self, config_file=None):
+        """
+        Returns user's email from global configuration file.
+
+        :param config_file: A path to file which should be used to retrieve
+          configuration from (might also be a list of file paths)
+        """
+        username = self.get_config_value('ui', 'username')
+        if username:
+            return author_email(username)
+        return None

rhodecode/lib/vcs/backends/hg/workdir.py

@@ -0,0 +1,21 b''
+from rhodecode.lib.vcs.backends.base import BaseWorkdir
+from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError
+
+from ...utils.hgcompat import hg_merge
+
+
+class MercurialWorkdir(BaseWorkdir):
+
+    def get_branch(self):
+        return self.repository._repo.dirstate.branch()
+
+    def get_changeset(self):
+        return self.repository.get_changeset()
+
+    def checkout_branch(self, branch=None):
+        if branch is None:
+            branch = self.repository.DEFAULT_BRANCH_NAME
+        if branch not in self.repository.branches:
+            raise BranchDoesNotExistError
+
+        hg_merge.update(self.repository._repo, branch, False, False, None)

rhodecode/lib/vcs/conf/settings.py

@@ -0,0 +1,33 b''
+import os
+import tempfile
+from rhodecode.lib.vcs.utils.paths import get_user_home
+
+abspath = lambda * p: os.path.abspath(os.path.join(*p))
+
+VCSRC_PATH = os.environ.get('VCSRC_PATH')
+
+if not VCSRC_PATH:
+    HOME_ = get_user_home()
+    if not HOME_:
+        HOME_ = tempfile.gettempdir()
+
+VCSRC_PATH = VCSRC_PATH or abspath(HOME_, '.vcsrc')
+if os.path.isdir(VCSRC_PATH):
+    VCSRC_PATH = os.path.join(VCSRC_PATH, '__init__.py')
+
+BACKENDS = {
+    'hg': 'vcs.backends.hg.MercurialRepository',
+    'git': 'vcs.backends.git.GitRepository',
+}
+
+ARCHIVE_SPECS = {
+    'tar': ('application/x-tar', '.tar'),
+    'tbz2': ('application/x-bzip2', '.tar.bz2'),
+    'tgz': ('application/x-gzip', '.tar.gz'),
+    'zip': ('application/zip', '.zip'),
+}
+
+BACKENDS = {
+    'hg': 'rhodecode.lib.vcs.backends.hg.MercurialRepository',
+    'git': 'rhodecode.lib.vcs.backends.git.GitRepository',
+}

rhodecode/lib/vcs/exceptions.py

@@ -0,0 +1,93 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.exceptions
+    ~~~~~~~~~~~~~~
+
+    Custom exceptions module
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+
+
+class VCSError(Exception):
+    pass
+
+
+class RepositoryError(VCSError):
+    pass
+
+
+class EmptyRepositoryError(RepositoryError):
+    pass
+
+
+class TagAlreadyExistError(RepositoryError):
+    pass
+
+
+class TagDoesNotExistError(RepositoryError):
+    pass
+
+
+class BranchAlreadyExistError(RepositoryError):
+    pass
+
+
+class BranchDoesNotExistError(RepositoryError):
+    pass
+
+
+class ChangesetError(RepositoryError):
+    pass
+
+
+class ChangesetDoesNotExistError(ChangesetError):
+    pass
+
+
+class CommitError(RepositoryError):
+    pass
+
+
+class NothingChangedError(CommitError):
+    pass
+
+
+class NodeError(VCSError):
+    pass
+
+
+class RemovedFileNodeError(NodeError):
+    pass
+
+
+class NodeAlreadyExistsError(CommitError):
+    pass
+
+
+class NodeAlreadyChangedError(CommitError):
+    pass
+
+
+class NodeDoesNotExistError(CommitError):
+    pass
+
+
+class NodeNotChangedError(CommitError):
+    pass
+
+
+class NodeAlreadyAddedError(CommitError):
+    pass
+
+
+class NodeAlreadyRemovedError(CommitError):
+    pass
+
+
+class ImproperArchiveTypeError(VCSError):
+    pass
+
+class CommandError(VCSError):
+    pass

rhodecode/lib/vcs/nodes.py

@@ -0,0 +1,551 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.nodes
+    ~~~~~~~~~
+
+    Module holding everything related to vcs nodes.
+
+    :created_on: Apr 8, 2010
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+import stat
+import posixpath
+import mimetypes
+
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils import safe_unicode
+from rhodecode.lib.vcs.exceptions import NodeError
+from rhodecode.lib.vcs.exceptions import RemovedFileNodeError
+
+from pygments import lexers
+
+
+class NodeKind:
+    DIR = 1
+    FILE = 2
+
+
+class NodeState:
+    ADDED = u'added'
+    CHANGED = u'changed'
+    NOT_CHANGED = u'not changed'
+    REMOVED = u'removed'
+
+
+class NodeGeneratorBase(object):
+    """
+    Base class for removed added and changed filenodes, it's a lazy generator
+    class that will create filenodes only on iteration or call
+
+    The len method doesn't need to create filenodes at all
+    """
+
+    def __init__(self, current_paths, cs):
+        self.cs = cs
+        self.current_paths = current_paths
+
+    def __call__(self):
+        return [n for n in self]
+
+    def __getslice__(self, i, j):
+        for p in self.current_paths[i:j]:
+            yield self.cs.get_node(p)
+
+    def __len__(self):
+        return len(self.current_paths)
+
+    def __iter__(self):
+        for p in self.current_paths:
+            yield self.cs.get_node(p)
+
+
+class AddedFileNodesGenerator(NodeGeneratorBase):
+    """
+    Class holding Added files for current changeset
+    """
+    pass
+
+
+class ChangedFileNodesGenerator(NodeGeneratorBase):
+    """
+    Class holding Changed files for current changeset
+    """
+    pass
+
+
+class RemovedFileNodesGenerator(NodeGeneratorBase):
+    """
+    Class holding removed files for current changeset
+    """
+    def __iter__(self):
+        for p in self.current_paths:
+            yield RemovedFileNode(path=p)
+
+    def __getslice__(self, i, j):
+        for p in self.current_paths[i:j]:
+            yield RemovedFileNode(path=p)
+
+
+class Node(object):
+    """
+    Simplest class representing file or directory on repository.  SCM backends
+    should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node``
+    directly.
+
+    Node's ``path`` cannot start with slash as we operate on *relative* paths
+    only. Moreover, every single node is identified by the ``path`` attribute,
+    so it cannot end with slash, too. Otherwise, path could lead to mistakes.
+    """
+
+    def __init__(self, path, kind):
+        if path.startswith('/'):
+            raise NodeError("Cannot initialize Node objects with slash at "
+                "the beginning as only relative paths are supported")
+        self.path = path.rstrip('/')
+        if path == '' and kind != NodeKind.DIR:
+            raise NodeError("Only DirNode and its subclasses may be "
+                            "initialized with empty path")
+        self.kind = kind
+        #self.dirs, self.files = [], []
+        if self.is_root() and not self.is_dir():
+            raise NodeError("Root node cannot be FILE kind")
+
+    @LazyProperty
+    def parent(self):
+        parent_path = self.get_parent_path()
+        if parent_path:
+            if self.changeset:
+                return self.changeset.get_node(parent_path)
+            return DirNode(parent_path)
+        return None
+
+    @LazyProperty
+    def name(self):
+        """
+        Returns name of the node so if its path
+        then only last part is returned.
+        """
+        return safe_unicode(self.path.rstrip('/').split('/')[-1])
+
+    def _get_kind(self):
+        return self._kind
+
+    def _set_kind(self, kind):
+        if hasattr(self, '_kind'):
+            raise NodeError("Cannot change node's kind")
+        else:
+            self._kind = kind
+            # Post setter check (path's trailing slash)
+            if self.path.endswith('/'):
+                raise NodeError("Node's path cannot end with slash")
+
+    kind = property(_get_kind, _set_kind)
+
+    def __cmp__(self, other):
+        """
+        Comparator using name of the node, needed for quick list sorting.
+        """
+        kind_cmp = cmp(self.kind, other.kind)
+        if kind_cmp:
+            return kind_cmp
+        return cmp(self.name, other.name)
+
+    def __eq__(self, other):
+        for attr in ['name', 'path', 'kind']:
+            if getattr(self, attr) != getattr(other, attr):
+                return False
+        if self.is_file():
+            if self.content != other.content:
+                return False
+        else:
+            # For DirNode's check without entering each dir
+            self_nodes_paths = list(sorted(n.path for n in self.nodes))
+            other_nodes_paths = list(sorted(n.path for n in self.nodes))
+            if self_nodes_paths != other_nodes_paths:
+                return False
+        return True
+
+    def __nq__(self, other):
+        return not self.__eq__(other)
+
+    def __repr__(self):
+        return '<%s %r>' % (self.__class__.__name__, self.path)
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __unicode__(self):
+        return self.name
+
+    def get_parent_path(self):
+        """
+        Returns node's parent path or empty string if node is root.
+        """
+        if self.is_root():
+            return ''
+        return posixpath.dirname(self.path.rstrip('/')) + '/'
+
+    def is_file(self):
+        """
+        Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False``
+        otherwise.
+        """
+        return self.kind == NodeKind.FILE
+
+    def is_dir(self):
+        """
+        Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False``
+        otherwise.
+        """
+        return self.kind == NodeKind.DIR
+
+    def is_root(self):
+        """
+        Returns ``True`` if node is a root node and ``False`` otherwise.
+        """
+        return self.kind == NodeKind.DIR and self.path == ''
+
+    @LazyProperty
+    def added(self):
+        return self.state is NodeState.ADDED
+
+    @LazyProperty
+    def changed(self):
+        return self.state is NodeState.CHANGED
+
+    @LazyProperty
+    def not_changed(self):
+        return self.state is NodeState.NOT_CHANGED
+
+    @LazyProperty
+    def removed(self):
+        return self.state is NodeState.REMOVED
+
+
+class FileNode(Node):
+    """
+    Class representing file nodes.
+
+    :attribute: path: path to the node, relative to repostiory's root
+    :attribute: content: if given arbitrary sets content of the file
+    :attribute: changeset: if given, first time content is accessed, callback
+    :attribute: mode: octal stat mode for a node. Default is 0100644.
+    """
+
+    def __init__(self, path, content=None, changeset=None, mode=None):
+        """
+        Only one of ``content`` and ``changeset`` may be given. Passing both
+        would raise ``NodeError`` exception.
+
+        :param path: relative path to the node
+        :param content: content may be passed to constructor
+        :param changeset: if given, will use it to lazily fetch content
+        :param mode: octal representation of ST_MODE (i.e. 0100644)
+        """
+
+        if content and changeset:
+            raise NodeError("Cannot use both content and changeset")
+        super(FileNode, self).__init__(path, kind=NodeKind.FILE)
+        self.changeset = changeset
+        self._content = content
+        self._mode = mode or 0100644
+
+    @LazyProperty
+    def mode(self):
+        """
+        Returns lazily mode of the FileNode. If ``changeset`` is not set, would
+        use value given at initialization or 0100644 (default).
+        """
+        if self.changeset:
+            mode = self.changeset.get_file_mode(self.path)
+        else:
+            mode = self._mode
+        return mode
+
+    @property
+    def content(self):
+        """
+        Returns lazily content of the FileNode. If possible, would try to
+        decode content from UTF-8.
+        """
+        if self.changeset:
+            content = self.changeset.get_file_content(self.path)
+        else:
+            content = self._content
+
+        if bool(content and '\0' in content):
+            return content
+        return safe_unicode(content)
+
+    @LazyProperty
+    def size(self):
+        if self.changeset:
+            return self.changeset.get_file_size(self.path)
+        raise NodeError("Cannot retrieve size of the file without related "
+            "changeset attribute")
+
+    @LazyProperty
+    def message(self):
+        if self.changeset:
+            return self.last_changeset.message
+        raise NodeError("Cannot retrieve message of the file without related "
+            "changeset attribute")
+
+    @LazyProperty
+    def last_changeset(self):
+        if self.changeset:
+            return self.changeset.get_file_changeset(self.path)
+        raise NodeError("Cannot retrieve last changeset of the file without "
+            "related changeset attribute")
+
+    def get_mimetype(self):
+        """
+        Mimetype is calculated based on the file's content. If ``_mimetype``
+        attribute is available, it will be returned (backends which store
+        mimetypes or can easily recognize them, should set this private
+        attribute to indicate that type should *NOT* be calculated).
+        """
+        if hasattr(self, '_mimetype'):
+            if (isinstance(self._mimetype,(tuple,list,)) and
+                len(self._mimetype) == 2):
+                return self._mimetype
+            else:
+                raise NodeError('given _mimetype attribute must be an 2 '
+                               'element list or tuple')
+
+        mtype,encoding = mimetypes.guess_type(self.name)
+
+        if mtype is None:
+            if self.is_binary:
+                mtype = 'application/octet-stream'
+                encoding = None
+            else:
+                mtype = 'text/plain'
+                encoding = None
+        return mtype,encoding
+
+    @LazyProperty
+    def mimetype(self):
+        """
+        Wrapper around full mimetype info. It returns only type of fetched
+        mimetype without the encoding part. use get_mimetype function to fetch
+        full set of (type,encoding)
+        """
+        return self.get_mimetype()[0]
+
+    @LazyProperty
+    def mimetype_main(self):
+        return self.mimetype.split('/')[0]
+
+    @LazyProperty
+    def lexer(self):
+        """
+        Returns pygment's lexer class. Would try to guess lexer taking file's
+        content, name and mimetype.
+        """
+        try:
+            lexer = lexers.guess_lexer_for_filename(self.name, self.content)
+        except lexers.ClassNotFound:
+            lexer = lexers.TextLexer()
+        # returns first alias
+        return lexer
+
+    @LazyProperty
+    def lexer_alias(self):
+        """
+        Returns first alias of the lexer guessed for this file.
+        """
+        return self.lexer.aliases[0]
+
+    @LazyProperty
+    def history(self):
+        """
+        Returns a list of changeset for this file in which the file was changed
+        """
+        if self.changeset is None:
+            raise NodeError('Unable to get changeset for this FileNode')
+        return self.changeset.get_file_history(self.path)
+
+    @LazyProperty
+    def annotate(self):
+        """
+        Returns a list of three element tuples with lineno,changeset and line
+        """
+        if self.changeset is None:
+            raise NodeError('Unable to get changeset for this FileNode')
+        return self.changeset.get_file_annotate(self.path)
+
+    @LazyProperty
+    def state(self):
+        if not self.changeset:
+            raise NodeError("Cannot check state of the node if it's not "
+                "linked with changeset")
+        elif self.path in (node.path for node in self.changeset.added):
+            return NodeState.ADDED
+        elif self.path in (node.path for node in self.changeset.changed):
+            return NodeState.CHANGED
+        else:
+            return NodeState.NOT_CHANGED
+
+    @property
+    def is_binary(self):
+        """
+        Returns True if file has binary content.
+        """
+        bin = '\0' in self.content
+        return bin
+
+    @LazyProperty
+    def extension(self):
+        """Returns filenode extension"""
+        return self.name.split('.')[-1]
+
+    def is_executable(self):
+        """
+        Returns ``True`` if file has executable flag turned on.
+        """
+        return bool(self.mode & stat.S_IXUSR)
+
+
+class RemovedFileNode(FileNode):
+    """
+    Dummy FileNode class - trying to access any public attribute except path,
+    name, kind or state (or methods/attributes checking those two) would raise
+    RemovedFileNodeError.
+    """
+    ALLOWED_ATTRIBUTES = ['name', 'path', 'state', 'is_root', 'is_file',
+        'is_dir', 'kind', 'added', 'changed', 'not_changed', 'removed']
+
+    def __init__(self, path):
+        """
+        :param path: relative path to the node
+        """
+        super(RemovedFileNode, self).__init__(path=path)
+
+    def __getattribute__(self, attr):
+        if attr.startswith('_') or attr in RemovedFileNode.ALLOWED_ATTRIBUTES:
+            return super(RemovedFileNode, self).__getattribute__(attr)
+        raise RemovedFileNodeError("Cannot access attribute %s on "
+            "RemovedFileNode" % attr)
+
+    @LazyProperty
+    def state(self):
+        return NodeState.REMOVED
+
+
+class DirNode(Node):
+    """
+    DirNode stores list of files and directories within this node.
+    Nodes may be used standalone but within repository context they
+    lazily fetch data within same repositorty's changeset.
+    """
+
+    def __init__(self, path, nodes=(), changeset=None):
+        """
+        Only one of ``nodes`` and ``changeset`` may be given. Passing both
+        would raise ``NodeError`` exception.
+
+        :param path: relative path to the node
+        :param nodes: content may be passed to constructor
+        :param changeset: if given, will use it to lazily fetch content
+        :param size: always 0 for ``DirNode``
+        """
+        if nodes and changeset:
+            raise NodeError("Cannot use both nodes and changeset")
+        super(DirNode, self).__init__(path, NodeKind.DIR)
+        self.changeset = changeset
+        self._nodes = nodes
+
+    @LazyProperty
+    def content(self):
+        raise NodeError("%s represents a dir and has no ``content`` attribute"
+            % self)
+
+    @LazyProperty
+    def nodes(self):
+        if self.changeset:
+            nodes = self.changeset.get_nodes(self.path)
+        else:
+            nodes = self._nodes
+        self._nodes_dict = dict((node.path, node) for node in nodes)
+        return sorted(nodes)
+
+    @LazyProperty
+    def files(self):
+        return sorted((node for node in self.nodes if node.is_file()))
+
+    @LazyProperty
+    def dirs(self):
+        return sorted((node for node in self.nodes if node.is_dir()))
+
+    def __iter__(self):
+        for node in self.nodes:
+            yield node
+
+    def get_node(self, path):
+        """
+        Returns node from within this particular ``DirNode``, so it is now
+        allowed to fetch, i.e. node located at 'docs/api/index.rst' from node
+        'docs'. In order to access deeper nodes one must fetch nodes between
+        them first - this would work::
+
+           docs = root.get_node('docs')
+           docs.get_node('api').get_node('index.rst')
+
+        :param: path - relative to the current node
+
+        .. note::
+           To access lazily (as in example above) node have to be initialized
+           with related changeset object - without it node is out of
+           context and may know nothing about anything else than nearest
+           (located at same level) nodes.
+        """
+        try:
+            path = path.rstrip('/')
+            if path == '':
+                raise NodeError("Cannot retrieve node without path")
+            self.nodes  # access nodes first in order to set _nodes_dict
+            paths = path.split('/')
+            if len(paths) == 1:
+                if not self.is_root():
+                    path = '/'.join((self.path, paths[0]))
+                else:
+                    path = paths[0]
+                return self._nodes_dict[path]
+            elif len(paths) > 1:
+                if self.changeset is None:
+                    raise NodeError("Cannot access deeper "
+                                    "nodes without changeset")
+                else:
+                    path1, path2 = paths[0], '/'.join(paths[1:])
+                    return self.get_node(path1).get_node(path2)
+            else:
+                raise KeyError
+        except KeyError:
+            raise NodeError("Node does not exist at %s" % path)
+
+    @LazyProperty
+    def state(self):
+        raise NodeError("Cannot access state of DirNode")
+
+    @LazyProperty
+    def size(self):
+        size = 0
+        for root, dirs, files in self.changeset.walk(self.path):
+            for f in files:
+                size += f.size
+
+        return size
+
+
+class RootNode(DirNode):
+    """
+    DirNode being the root node of the repository.
+    """
+
+    def __init__(self, nodes=(), changeset=None):
+        super(RootNode, self).__init__(path='', nodes=nodes,
+            changeset=changeset)
+
+    def __repr__(self):
+        return '<%s>' % self.__class__.__name__

rhodecode/lib/vcs/utils/__init__.py

@@ -0,0 +1,133 b''
+"""
+This module provides some useful tools for ``vcs`` like annotate/diff html
+output. It also includes some internal helpers.
+"""
+import sys
+import time
+import datetime
+
+
+def makedate():
+    lt = time.localtime()
+    if lt[8] == 1 and time.daylight:
+        tz = time.altzone
+    else:
+        tz = time.timezone
+    return time.mktime(lt), tz
+
+
+def date_fromtimestamp(unixts, tzoffset=0):
+    """
+    Makes a local datetime object out of unix timestamp
+
+    :param unixts:
+    :param tzoffset:
+    """
+
+    return datetime.datetime.fromtimestamp(float(unixts))
+
+
+def safe_unicode(str_, from_encoding='utf8'):
+    """
+    safe unicode function. Does few trick to turn str_ into unicode
+
+    In case of UnicodeDecode error we try to return it with encoding detected
+    by chardet library if it fails fallback to unicode with errors replaced
+
+    :param str_: string to decode
+    :rtype: unicode
+    :returns: unicode object
+    """
+    if isinstance(str_, unicode):
+        return str_
+
+    try:
+        return unicode(str_)
+    except UnicodeDecodeError:
+        pass
+
+    try:
+        return unicode(str_, from_encoding)
+    except UnicodeDecodeError:
+        pass
+
+    try:
+        import chardet
+        encoding = chardet.detect(str_)['encoding']
+        if encoding is None:
+            raise Exception()
+        return str_.decode(encoding)
+    except (ImportError, UnicodeDecodeError, Exception):
+        return unicode(str_, from_encoding, 'replace')
+
+
+def safe_str(unicode_, to_encoding='utf8'):
+    """
+    safe str function. Does few trick to turn unicode_ into string
+
+    In case of UnicodeEncodeError we try to return it with encoding detected
+    by chardet library if it fails fallback to string with errors replaced
+
+    :param unicode_: unicode to encode
+    :rtype: str
+    :returns: str object
+    """
+
+    if isinstance(unicode_, str):
+        return unicode_
+
+    try:
+        return unicode_.encode(to_encoding)
+    except UnicodeEncodeError:
+        pass
+
+    try:
+        import chardet
+        encoding = chardet.detect(unicode_)['encoding']
+        print encoding
+        if encoding is None:
+            raise UnicodeEncodeError()
+
+        return unicode_.encode(encoding)
+    except (ImportError, UnicodeEncodeError):
+        return unicode_.encode(to_encoding, 'replace')
+
+    return safe_str
+
+
+def author_email(author):
+    """
+    returns email address of given author.
+    If any of <,> sign are found, it fallbacks to regex findall()
+    and returns first found result or empty string
+
+    Regex taken from http://www.regular-expressions.info/email.html
+    """
+    import re
+    r = author.find('>')
+    l = author.find('<')
+
+    if l == -1 or r == -1:
+        # fallback to regex match of email out of a string
+        email_re = re.compile(r"""[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!"""
+                              r"""#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z"""
+                              r"""0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]"""
+                              r"""*[a-z0-9])?""", re.IGNORECASE)
+        m = re.findall(email_re, author)
+        return m[0] if m else ''
+
+    return author[l + 1:r].strip()
+
+
+def author_name(author):
+    """
+    get name of author, or else username.
+    It'll try to find an email in the author string and just cut it off
+    to get the username
+    """
+
+    if not '@' in author:
+        return author
+    else:
+        return author.replace(author_email(author), '').replace('<', '')\
+            .replace('>', '').strip()

rhodecode/lib/vcs/utils/annotate.py

@@ -0,0 +1,177 b''
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.nodes import FileNode
+from pygments.formatters import HtmlFormatter
+from pygments import highlight
+
+import StringIO
+
+
+def annotate_highlight(filenode, annotate_from_changeset_func=None,
+        order=None, headers=None, **options):
+    """
+    Returns html portion containing annotated table with 3 columns: line
+    numbers, changeset information and pygmentized line of code.
+
+    :param filenode: FileNode object
+    :param annotate_from_changeset_func: function taking changeset and
+      returning single annotate cell; needs break line at the end
+    :param order: ordered sequence of ``ls`` (line numbers column),
+      ``annotate`` (annotate column), ``code`` (code column); Default is
+      ``['ls', 'annotate', 'code']``
+    :param headers: dictionary with headers (keys are whats in ``order``
+      parameter)
+    """
+    options['linenos'] = True
+    formatter = AnnotateHtmlFormatter(filenode=filenode, order=order,
+        headers=headers,
+        annotate_from_changeset_func=annotate_from_changeset_func, **options)
+    lexer = filenode.lexer
+    highlighted = highlight(filenode.content, lexer, formatter)
+    return highlighted
+
+
+class AnnotateHtmlFormatter(HtmlFormatter):
+
+    def __init__(self, filenode, annotate_from_changeset_func=None,
+            order=None, **options):
+        """
+        If ``annotate_from_changeset_func`` is passed it should be a function
+        which returns string from the given changeset. For example, we may pass
+        following function as ``annotate_from_changeset_func``::
+
+            def changeset_to_anchor(changeset):
+                return '<a href="/changesets/%s/">%s</a>\n' %\
+                       (changeset.id, changeset.id)
+
+        :param annotate_from_changeset_func: see above
+        :param order: (default: ``['ls', 'annotate', 'code']``); order of
+          columns;
+        :param options: standard pygment's HtmlFormatter options, there is
+          extra option tough, ``headers``. For instance we can pass::
+
+             formatter = AnnotateHtmlFormatter(filenode, headers={
+                'ls': '#',
+                'annotate': 'Annotate',
+                'code': 'Code',
+             })
+
+        """
+        super(AnnotateHtmlFormatter, self).__init__(**options)
+        self.annotate_from_changeset_func = annotate_from_changeset_func
+        self.order = order or ('ls', 'annotate', 'code')
+        headers = options.pop('headers', None)
+        if headers and not ('ls' in headers and 'annotate' in headers and
+            'code' in headers):
+            raise ValueError("If headers option dict is specified it must "
+                "all 'ls', 'annotate' and 'code' keys")
+        self.headers = headers
+        if isinstance(filenode, FileNode):
+            self.filenode = filenode
+        else:
+            raise VCSError("This formatter expect FileNode parameter, not %r"
+                % type(filenode))
+
+    def annotate_from_changeset(self, changeset):
+        """
+        Returns full html line for single changeset per annotated line.
+        """
+        if self.annotate_from_changeset_func:
+            return self.annotate_from_changeset_func(changeset)
+        else:
+            return ''.join((changeset.id, '\n'))
+
+    def _wrap_tablelinenos(self, inner):
+        dummyoutfile = StringIO.StringIO()
+        lncount = 0
+        for t, line in inner:
+            if t:
+                lncount += 1
+            dummyoutfile.write(line)
+
+        fl = self.linenostart
+        mw = len(str(lncount + fl - 1))
+        sp = self.linenospecial
+        st = self.linenostep
+        la = self.lineanchors
+        aln = self.anchorlinenos
+        if sp:
+            lines = []
+
+            for i in range(fl, fl + lncount):
+                if i % st == 0:
+                    if i % sp == 0:
+                        if aln:
+                            lines.append('<a href="#%s-%d" class="special">'
+                                         '%*d</a>' %
+                                         (la, i, mw, i))
+                        else:
+                            lines.append('<span class="special">'
+                                         '%*d</span>' % (mw, i))
+                    else:
+                        if aln:
+                            lines.append('<a href="#%s-%d">'
+                                         '%*d</a>' % (la, i, mw, i))
+                        else:
+                            lines.append('%*d' % (mw, i))
+                else:
+                    lines.append('')
+            ls = '\n'.join(lines)
+        else:
+            lines = []
+            for i in range(fl, fl + lncount):
+                if i % st == 0:
+                    if aln:
+                        lines.append('<a href="#%s-%d">%*d</a>' \
+                                     % (la, i, mw, i))
+                    else:
+                        lines.append('%*d' % (mw, i))
+                else:
+                    lines.append('')
+            ls = '\n'.join(lines)
+
+        annotate_changesets = [tup[1] for tup in self.filenode.annotate]
+        # If pygments cropped last lines break we need do that too
+        ln_cs = len(annotate_changesets)
+        ln_ = len(ls.splitlines())
+        if  ln_cs > ln_:
+            annotate_changesets = annotate_changesets[:ln_ - ln_cs]
+        annotate = ''.join((self.annotate_from_changeset(changeset)
+            for changeset in annotate_changesets))
+        # in case you wonder about the seemingly redundant <div> here:
+        # since the content in the other cell also is wrapped in a div,
+        # some browsers in some configurations seem to mess up the formatting.
+        '''
+        yield 0, ('<table class="%stable">' % self.cssclass +
+                  '<tr><td class="linenos"><div class="linenodiv"><pre>' +
+                  ls + '</pre></div></td>' +
+                  '<td class="code">')
+        yield 0, dummyoutfile.getvalue()
+        yield 0, '</td></tr></table>'
+
+        '''
+        headers_row = []
+        if self.headers:
+            headers_row = ['<tr class="annotate-header">']
+            for key in self.order:
+                td = ''.join(('<td>', self.headers[key], '</td>'))
+                headers_row.append(td)
+            headers_row.append('</tr>')
+
+        body_row_start = ['<tr>']
+        for key in self.order:
+            if key == 'ls':
+                body_row_start.append(
+                    '<td class="linenos"><div class="linenodiv"><pre>' +
+                    ls + '</pre></div></td>')
+            elif key == 'annotate':
+                body_row_start.append(
+                    '<td class="annotate"><div class="annotatediv"><pre>' +
+                    annotate + '</pre></div></td>')
+            elif key == 'code':
+                body_row_start.append('<td class="code">')
+        yield 0, ('<table class="%stable">' % self.cssclass +
+                  ''.join(headers_row) +
+                  ''.join(body_row_start)
+                  )
+        yield 0, dummyoutfile.getvalue()
+        yield 0, '</td></tr></table>'

rhodecode/lib/vcs/utils/archivers.py

@@ -0,0 +1,67 b''
+# -*- coding: utf-8 -*-
+"""
+    vcs.utils.archivers
+    ~~~~~~~~~~~~~~~~~~~
+
+    set of archiver functions for creating archives from repository content
+
+    :created_on: Jan 21, 2011
+    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
+"""
+
+
+class BaseArchiver(object):
+
+    def __init__(self):
+        self.archive_file = self._get_archive_file()
+
+    def addfile(self):
+        """
+        Adds a file to archive container
+        """
+        pass
+
+    def close(self):
+        """
+        Closes and finalizes operation of archive container object
+        """
+        self.archive_file.close()
+
+    def _get_archive_file(self):
+        """
+        Returns container for specific archive
+        """
+        raise NotImplementedError()
+
+
+class TarArchiver(BaseArchiver):
+    pass
+
+
+class Tbz2Archiver(BaseArchiver):
+    pass
+
+
+class TgzArchiver(BaseArchiver):
+    pass
+
+
+class ZipArchiver(BaseArchiver):
+    pass
+
+
+def get_archiver(self, kind):
+    """
+    Returns instance of archiver class specific to given kind
+
+    :param kind: archive kind
+    """
+
+    archivers = {
+        'tar': TarArchiver,
+        'tbz2': Tbz2Archiver,
+        'tgz': TgzArchiver,
+        'zip': ZipArchiver,
+    }
+
+    return archivers[kind]()

rhodecode/lib/vcs/utils/baseui_config.py

@@ -0,0 +1,47 b''
+from mercurial import ui, config
+
+
+def make_ui(self, path='hgwebdir.config'):
+    """
+    A funcion that will read python rc files and make an ui from read options
+
+    :param path: path to mercurial config file
+    """
+    #propagated from mercurial documentation
+    sections = [
+                'alias',
+                'auth',
+                'decode/encode',
+                'defaults',
+                'diff',
+                'email',
+                'extensions',
+                'format',
+                'merge-patterns',
+                'merge-tools',
+                'hooks',
+                'http_proxy',
+                'smtp',
+                'patch',
+                'paths',
+                'profiling',
+                'server',
+                'trusted',
+                'ui',
+                'web',
+                ]
+
+    repos = path
+    baseui = ui.ui()
+    cfg = config.config()
+    cfg.read(repos)
+    self.paths = cfg.items('paths')
+    self.base_path = self.paths[0][1].replace('*', '')
+    self.check_repo_dir(self.paths)
+    self.set_statics(cfg)
+
+    for section in sections:
+        for k, v in cfg.items(section):
+            baseui.setconfig(section, k, v)
+
+    return baseui

rhodecode/lib/vcs/utils/compat.py

@@ -0,0 +1,13 b''
+"""
+Various utilities to work with Python < 2.7.
+
+Those utilities may be deleted once ``vcs`` stops support for older Python
+versions.
+"""
+import sys
+
+
+if sys.version_info >= (2, 7):
+    unittest = __import__('unittest')
+else:
+    unittest = __import__('unittest2')

rhodecode/lib/vcs/utils/diffs.py

@@ -0,0 +1,460 b''
+# -*- coding: utf-8 -*-
+# original copyright: 2007-2008 by Armin Ronacher
+# licensed under the BSD license.
+
+import re
+import difflib
+import logging
+
+from difflib import unified_diff
+from itertools import tee, imap
+
+from mercurial.match import match
+
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.nodes import FileNode, NodeError
+
+
+def get_udiff(filenode_old, filenode_new,show_whitespace=True):
+    """
+    Returns unified diff between given ``filenode_old`` and ``filenode_new``.
+    """
+    try:
+        filenode_old_date = filenode_old.last_changeset.date
+    except NodeError:
+        filenode_old_date = None
+
+    try:
+        filenode_new_date = filenode_new.last_changeset.date
+    except NodeError:
+        filenode_new_date = None
+
+    for filenode in (filenode_old, filenode_new):
+        if not isinstance(filenode, FileNode):
+            raise VCSError("Given object should be FileNode object, not %s"
+                % filenode.__class__)
+
+    if filenode_old_date and filenode_new_date:
+        if not filenode_old_date < filenode_new_date:
+            logging.debug("Generating udiff for filenodes with not increasing "
+                "dates")
+
+    vcs_udiff = unified_diff(filenode_old.content.splitlines(True),
+                               filenode_new.content.splitlines(True),
+                               filenode_old.name,
+                               filenode_new.name,
+                               filenode_old_date,
+                               filenode_old_date)
+    return vcs_udiff
+
+
+def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True):
+    """
+    Returns git style diff between given ``filenode_old`` and ``filenode_new``.
+
+    :param ignore_whitespace: ignore whitespaces in diff
+    """
+
+    for filenode in (filenode_old, filenode_new):
+        if not isinstance(filenode, FileNode):
+            raise VCSError("Given object should be FileNode object, not %s"
+                % filenode.__class__)
+
+    old_raw_id = getattr(filenode_old.changeset, 'raw_id', '0' * 40)
+    new_raw_id = getattr(filenode_new.changeset, 'raw_id', '0' * 40)
+
+    repo = filenode_new.changeset.repository
+    vcs_gitdiff = repo._get_diff(old_raw_id, new_raw_id, filenode_new.path,
+                                 ignore_whitespace)
+
+    return vcs_gitdiff
+
+
+class DiffProcessor(object):
+    """
+    Give it a unified diff and it returns a list of the files that were
+    mentioned in the diff together with a dict of meta information that
+    can be used to render it in a HTML template.
+    """
+    _chunk_re = re.compile(r'@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
+
+    def __init__(self, diff, differ='diff', format='udiff'):
+        """
+        :param diff:   a text in diff format or generator
+        :param format: format of diff passed, `udiff` or `gitdiff`
+        """
+        if isinstance(diff, basestring):
+            diff = [diff]
+
+        self.__udiff = diff
+        self.__format = format
+        self.adds = 0
+        self.removes = 0
+
+        if isinstance(self.__udiff, basestring):
+            self.lines = iter(self.__udiff.splitlines(1))
+
+        elif self.__format == 'gitdiff':
+            udiff_copy = self.copy_iterator()
+            self.lines = imap(self.escaper, self._parse_gitdiff(udiff_copy))
+        else:
+            udiff_copy = self.copy_iterator()
+            self.lines = imap(self.escaper, udiff_copy)
+
+        # Select a differ.
+        if differ == 'difflib':
+            self.differ = self._highlight_line_difflib
+        else:
+            self.differ = self._highlight_line_udiff
+
+    def escaper(self, string):
+        return string.replace('<', '&lt;').replace('>', '&gt;')
+
+    def copy_iterator(self):
+        """
+        make a fresh copy of generator, we should not iterate thru
+        an original as it's needed for repeating operations on
+        this instance of DiffProcessor
+        """
+        self.__udiff, iterator_copy = tee(self.__udiff)
+        return iterator_copy
+
+    def _extract_rev(self, line1, line2):
+        """
+        Extract the filename and revision hint from a line.
+        """
+
+        try:
+            if line1.startswith('--- ') and line2.startswith('+++ '):
+                l1 = line1[4:].split(None, 1)
+                old_filename = l1[0].lstrip('a/') if len(l1) >= 1 else None
+                old_rev = l1[1] if len(l1) == 2 else 'old'
+
+                l2 = line2[4:].split(None, 1)
+                new_filename = l2[0].lstrip('b/') if len(l1) >= 1 else None
+                new_rev = l2[1] if len(l2) == 2 else 'new'
+
+                filename = old_filename if (old_filename !=
+                                            'dev/null') else new_filename
+
+                return filename, new_rev, old_rev
+        except (ValueError, IndexError):
+            pass
+
+        return None, None, None
+
+    def _parse_gitdiff(self, diffiterator):
+        def line_decoder(l):
+            if l.startswith('+') and not l.startswith('+++'):
+                self.adds += 1
+            elif l.startswith('-') and not l.startswith('---'):
+                self.removes += 1
+            return l.decode('utf8', 'replace')
+
+        output = list(diffiterator)
+        size = len(output)
+
+        if size == 2:
+            l = []
+            l.extend([output[0]])
+            l.extend(output[1].splitlines(1))
+            return map(line_decoder, l)
+        elif size == 1:
+            return  map(line_decoder, output[0].splitlines(1))
+        elif size == 0:
+            return []
+
+        raise Exception('wrong size of diff %s' % size)
+
+    def _highlight_line_difflib(self, line, next):
+        """
+        Highlight inline changes in both lines.
+        """
+
+        if line['action'] == 'del':
+            old, new = line, next
+        else:
+            old, new = next, line
+
+        oldwords = re.split(r'(\W)', old['line'])
+        newwords = re.split(r'(\W)', new['line'])
+
+        sequence = difflib.SequenceMatcher(None, oldwords, newwords)
+
+        oldfragments, newfragments = [], []
+        for tag, i1, i2, j1, j2 in sequence.get_opcodes():
+            oldfrag = ''.join(oldwords[i1:i2])
+            newfrag = ''.join(newwords[j1:j2])
+            if tag != 'equal':
+                if oldfrag:
+                    oldfrag = '<del>%s</del>' % oldfrag
+                if newfrag:
+                    newfrag = '<ins>%s</ins>' % newfrag
+            oldfragments.append(oldfrag)
+            newfragments.append(newfrag)
+
+        old['line'] = "".join(oldfragments)
+        new['line'] = "".join(newfragments)
+
+    def _highlight_line_udiff(self, line, next):
+        """
+        Highlight inline changes in both lines.
+        """
+        start = 0
+        limit = min(len(line['line']), len(next['line']))
+        while start < limit and line['line'][start] == next['line'][start]:
+            start += 1
+        end = -1
+        limit -= start
+        while -end <= limit and line['line'][end] == next['line'][end]:
+            end -= 1
+        end += 1
+        if start or end:
+            def do(l):
+                last = end + len(l['line'])
+                if l['action'] == 'add':
+                    tag = 'ins'
+                else:
+                    tag = 'del'
+                l['line'] = '%s<%s>%s</%s>%s' % (
+                    l['line'][:start],
+                    tag,
+                    l['line'][start:last],
+                    tag,
+                    l['line'][last:]
+                )
+            do(line)
+            do(next)
+
+    def _parse_udiff(self):
+        """
+        Parse the diff an return data for the template.
+        """
+        lineiter = self.lines
+        files = []
+        try:
+            line = lineiter.next()
+            # skip first context
+            skipfirst = True
+            while 1:
+                # continue until we found the old file
+                if not line.startswith('--- '):
+                    line = lineiter.next()
+                    continue
+
+                chunks = []
+                filename, old_rev, new_rev = \
+                    self._extract_rev(line, lineiter.next())
+                files.append({
+                    'filename':         filename,
+                    'old_revision':     old_rev,
+                    'new_revision':     new_rev,
+                    'chunks':           chunks
+                })
+
+                line = lineiter.next()
+                while line:
+                    match = self._chunk_re.match(line)
+                    if not match:
+                        break
+
+                    lines = []
+                    chunks.append(lines)
+
+                    old_line, old_end, new_line, new_end = \
+                        [int(x or 1) for x in match.groups()[:-1]]
+                    old_line -= 1
+                    new_line -= 1
+                    context = len(match.groups()) == 5
+                    old_end += old_line
+                    new_end += new_line
+
+                    if context:
+                        if not skipfirst:
+                            lines.append({
+                                'old_lineno': '...',
+                                'new_lineno': '...',
+                                'action': 'context',
+                                'line': line,
+                            })
+                        else:
+                            skipfirst = False
+
+                    line = lineiter.next()
+                    while old_line < old_end or new_line < new_end:
+                        if line:
+                            command, line = line[0], line[1:]
+                        else:
+                            command = ' '
+                        affects_old = affects_new = False
+
+                        # ignore those if we don't expect them
+                        if command in '#@':
+                            continue
+                        elif command == '+':
+                            affects_new = True
+                            action = 'add'
+                        elif command == '-':
+                            affects_old = True
+                            action = 'del'
+                        else:
+                            affects_old = affects_new = True
+                            action = 'unmod'
+
+                        old_line += affects_old
+                        new_line += affects_new
+                        lines.append({
+                            'old_lineno':   affects_old and old_line or '',
+                            'new_lineno':   affects_new and new_line or '',
+                            'action':       action,
+                            'line':         line
+                        })
+                        line = lineiter.next()
+
+        except StopIteration:
+            pass
+
+        # highlight inline changes
+        for file in files:
+            for chunk in chunks:
+                lineiter = iter(chunk)
+                #first = True
+                try:
+                    while 1:
+                        line = lineiter.next()
+                        if line['action'] != 'unmod':
+                            nextline = lineiter.next()
+                            if nextline['action'] == 'unmod' or \
+                               nextline['action'] == line['action']:
+                                continue
+                            self.differ(line, nextline)
+                except StopIteration:
+                    pass
+
+        return files
+
+    def prepare(self):
+        """
+        Prepare the passed udiff for HTML rendering. It'l return a list
+        of dicts
+        """
+        return self._parse_udiff()
+
+    def _safe_id(self, idstring):
+        """Make a string safe for including in an id attribute.
+
+        The HTML spec says that id attributes 'must begin with
+        a letter ([A-Za-z]) and may be followed by any number
+        of letters, digits ([0-9]), hyphens ("-"), underscores
+        ("_"), colons (":"), and periods (".")'. These regexps
+        are slightly over-zealous, in that they remove colons
+        and periods unnecessarily.
+
+        Whitespace is transformed into underscores, and then
+        anything which is not a hyphen or a character that
+        matches \w (alphanumerics and underscore) is removed.
+
+        """
+        # Transform all whitespace to underscore
+        idstring = re.sub(r'\s', "_", '%s' % idstring)
+        # Remove everything that is not a hyphen or a member of \w
+        idstring = re.sub(r'(?!-)\W', "", idstring).lower()
+        return idstring
+
+    def raw_diff(self):
+        """
+        Returns raw string as udiff
+        """
+        udiff_copy = self.copy_iterator()
+        if self.__format == 'gitdiff':
+            udiff_copy = self._parse_gitdiff(udiff_copy)
+        return u''.join(udiff_copy)
+
+    def as_html(self, table_class='code-difftable', line_class='line',
+                new_lineno_class='lineno old', old_lineno_class='lineno new',
+                code_class='code'):
+        """
+        Return udiff as html table with customized css classes
+        """
+        def _link_to_if(condition, label, url):
+            """
+            Generates a link if condition is meet or just the label if not.
+            """
+
+            if condition:
+                return '''<a href="%(url)s">%(label)s</a>''' % {'url': url,
+                                                                'label': label}
+            else:
+                return label
+        diff_lines = self.prepare()
+        _html_empty = True
+        _html = []
+        _html.append('''<table class="%(table_class)s">\n''' \
+                                            % {'table_class': table_class})
+        for diff in diff_lines:
+            for line in diff['chunks']:
+                _html_empty = False
+                for change in line:
+                    _html.append('''<tr class="%(line_class)s %(action)s">\n''' \
+                        % {'line_class': line_class,
+                           'action': change['action']})
+                    anchor_old_id = ''
+                    anchor_new_id = ''
+                    anchor_old = "%(filename)s_o%(oldline_no)s" % \
+                                {'filename': self._safe_id(diff['filename']),
+                                 'oldline_no': change['old_lineno']}
+                    anchor_new = "%(filename)s_n%(oldline_no)s" % \
+                                {'filename': self._safe_id(diff['filename']),
+                                 'oldline_no': change['new_lineno']}
+                    cond_old = change['old_lineno'] != '...' and \
+                                                        change['old_lineno']
+                    cond_new = change['new_lineno'] != '...' and \
+                                                        change['new_lineno']
+                    if cond_old:
+                        anchor_old_id = 'id="%s"' % anchor_old
+                    if cond_new:
+                        anchor_new_id = 'id="%s"' % anchor_new
+                    ###########################################################
+                    # OLD LINE NUMBER
+                    ###########################################################
+                    _html.append('''\t<td %(a_id)s class="%(old_lineno_cls)s">''' \
+                                    % {'a_id': anchor_old_id,
+                                       'old_lineno_cls': old_lineno_class})
+
+                    _html.append('''<pre>%(link)s</pre>''' \
+                        % {'link':
+                        _link_to_if(cond_old, change['old_lineno'], '#%s' \
+                                                                % anchor_old)})
+                    _html.append('''</td>\n''')
+                    ###########################################################
+                    # NEW LINE NUMBER
+                    ###########################################################
+
+                    _html.append('''\t<td %(a_id)s class="%(new_lineno_cls)s">''' \
+                                    % {'a_id': anchor_new_id,
+                                       'new_lineno_cls': new_lineno_class})
+
+                    _html.append('''<pre>%(link)s</pre>''' \
+                        % {'link':
+                        _link_to_if(cond_new, change['new_lineno'], '#%s' \
+                                                                % anchor_new)})
+                    _html.append('''</td>\n''')
+                    ###########################################################
+                    # CODE
+                    ###########################################################
+                    _html.append('''\t<td class="%(code_class)s">''' \
+                                                % {'code_class': code_class})
+                    _html.append('''\n\t\t<pre>%(code)s</pre>\n''' \
+                                                % {'code': change['line']})
+                    _html.append('''\t</td>''')
+                    _html.append('''\n</tr>\n''')
+        _html.append('''</table>''')
+        if _html_empty:
+            return None
+        return ''.join(_html)
+
+    def stat(self):
+        """
+        Returns tuple of adde,and removed lines for this instance
+        """
+        return self.adds, self.removes

rhodecode/lib/vcs/utils/fakemod.py

@@ -0,0 +1,13 b''
+import imp
+
+
+def create_module(name, path):
+    """
+    Returns module created *on the fly*. Returned module would have name same
+    as given ``name`` and would contain code read from file at the given
+    ``path`` (it may also be a zip or package containing *__main__* module).
+    """
+    module = imp.new_module(name)
+    module.__file__ = path
+    execfile(path, module.__dict__)
+    return module

rhodecode/lib/vcs/utils/filesize.py

@@ -0,0 +1,28 b''
+def filesizeformat(bytes, sep=' '):
+    """
+    Formats the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB,
+    102 B, 2.3 GB etc).
+
+    Grabbed from Django (http://www.djangoproject.com), slightly modified.
+
+    :param bytes: size in bytes (as integer)
+    :param sep: string separator between number and abbreviation
+    """
+    try:
+        bytes = float(bytes)
+    except (TypeError, ValueError, UnicodeDecodeError):
+        return '0%sB' % sep
+
+    if bytes < 1024:
+        size = bytes
+        template = '%.0f%sB'
+    elif bytes < 1024 * 1024:
+        size = bytes / 1024
+        template = '%.0f%sKB'
+    elif bytes < 1024 * 1024 * 1024:
+        size = bytes / 1024 / 1024
+        template = '%.1f%sMB'
+    else:
+        size = bytes / 1024 / 1024 / 1024
+        template = '%.2f%sGB'
+    return template % (size, sep)

rhodecode/lib/vcs/utils/helpers.py

@@ -0,0 +1,252 b''
+"""
+Utitlites aimed to help achieve mostly basic tasks.
+"""
+from __future__ import division
+
+import re
+import time
+import datetime
+import os.path
+from subprocess import Popen, PIPE
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.exceptions import RepositoryError
+from rhodecode.lib.vcs.utils.paths import abspath
+
+ALIASES = ['hg', 'git']
+
+
+def get_scm(path, search_recursively=False, explicit_alias=None):
+    """
+    Returns one of alias from ``ALIASES`` (in order of precedence same as
+    shortcuts given in ``ALIASES``) and top working dir path for the given
+    argument. If no scm-specific directory is found or more than one scm is
+    found at that directory, ``VCSError`` is raised.
+
+    :param search_recursively: if set to ``True``, this function would try to
+      move up to parent directory every time no scm is recognized for the
+      currently checked path. Default: ``False``.
+    :param explicit_alias: can be one of available backend aliases, when given
+      it will return given explicit alias in repositories under more than one
+      version control, if explicit_alias is different than found it will raise
+      VCSError
+    """
+    if not os.path.isdir(path):
+        raise VCSError("Given path %s is not a directory" % path)
+
+    def get_scms(path):
+        return [(scm, path) for scm in get_scms_for_path(path)]
+
+    found_scms = get_scms(path)
+    while  not found_scms and search_recursively:
+        newpath = abspath(path, '..')
+        if newpath == path:
+            break
+        path = newpath
+        found_scms = get_scms(path)
+
+    if len(found_scms) > 1:
+        for scm in found_scms:
+            if scm[0] == explicit_alias:
+                return scm
+        raise VCSError('More than one [%s] scm found at given path %s'
+                       % (','.join((x[0] for x in found_scms)), path))
+
+    if len(found_scms) is 0:
+        raise VCSError('No scm found at given path %s' % path)
+
+    return found_scms[0]
+
+
+def get_scms_for_path(path):
+    """
+    Returns all scm's found at the given path. If no scm is recognized
+    - empty list is returned.
+
+    :param path: path to directory which should be checked. May be callable.
+
+    :raises VCSError: if given ``path`` is not a directory
+    """
+    from rhodecode.lib.vcs.backends import get_backend
+    if hasattr(path, '__call__'):
+        path = path()
+    if not os.path.isdir(path):
+        raise VCSError("Given path %r is not a directory" % path)
+
+    result = []
+    for key in ALIASES:
+        dirname = os.path.join(path, '.' + key)
+        if os.path.isdir(dirname):
+            result.append(key)
+            continue
+        # We still need to check if it's not bare repository as
+        # bare repos don't have working directories
+        try:
+            get_backend(key)(path)
+            result.append(key)
+            continue
+        except RepositoryError:
+            # Wrong backend
+            pass
+        except VCSError:
+            # No backend at all
+            pass
+    return result
+
+
+def get_repo_paths(path):
+    """
+    Returns path's subdirectories which seems to be a repository.
+    """
+    repo_paths = []
+    dirnames = (os.path.abspath(dirname) for dirname in os.listdir(path))
+    for dirname in dirnames:
+        try:
+            get_scm(dirname)
+            repo_paths.append(dirname)
+        except VCSError:
+            pass
+    return repo_paths
+
+
+def run_command(cmd, *args):
+    """
+    Runs command on the system with given ``args``.
+    """
+    command = ' '.join((cmd, args))
+    p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE)
+    stdout, stderr = p.communicate()
+    return p.retcode, stdout, stderr
+
+
+def get_highlighted_code(name, code, type='terminal'):
+    """
+    If pygments are available on the system
+    then returned output is colored. Otherwise
+    unchanged content is returned.
+    """
+    import logging
+    try:
+        import pygments
+        pygments
+    except ImportError:
+        return code
+    from pygments import highlight
+    from pygments.lexers import guess_lexer_for_filename, ClassNotFound
+    from pygments.formatters import TerminalFormatter
+
+    try:
+        lexer = guess_lexer_for_filename(name, code)
+        formatter = TerminalFormatter()
+        content = highlight(code, lexer, formatter)
+    except ClassNotFound:
+        logging.debug("Couldn't guess Lexer, will not use pygments.")
+        content = code
+    return content
+
+def parse_changesets(text):
+    """
+    Returns dictionary with *start*, *main* and *end* ids.
+
+    Examples::
+
+        >>> parse_changesets('aaabbb')
+        {'start': None, 'main': 'aaabbb', 'end': None}
+        >>> parse_changesets('aaabbb..cccddd')
+        {'start': 'aaabbb', 'main': None, 'end': 'cccddd'}
+
+    """
+    text = text.strip()
+    CID_RE = r'[a-zA-Z0-9]+'
+    if not '..' in text:
+        m = re.match(r'^(?P<cid>%s)$' % CID_RE, text)
+        if m:
+            return {
+                'start': None,
+                'main': text,
+                'end': None,
+            }
+    else:
+        RE = r'^(?P<start>%s)?\.{2,3}(?P<end>%s)?$' % (CID_RE, CID_RE)
+        m = re.match(RE, text)
+        if m:
+            result = m.groupdict()
+            result['main'] = None
+            return result
+    raise ValueError("IDs not recognized")
+
+def parse_datetime(text):
+    """
+    Parses given text and returns ``datetime.datetime`` instance or raises
+    ``ValueError``.
+
+    :param text: string of desired date/datetime or something more verbose,
+      like *yesterday*, *2weeks 3days*, etc.
+    """
+
+    text = text.strip().lower()
+
+    INPUT_FORMATS = (
+        '%Y-%m-%d %H:%M:%S',
+        '%Y-%m-%d %H:%M',
+        '%Y-%m-%d',
+        '%m/%d/%Y %H:%M:%S',
+        '%m/%d/%Y %H:%M',
+        '%m/%d/%Y',
+        '%m/%d/%y %H:%M:%S',
+        '%m/%d/%y %H:%M',
+        '%m/%d/%y',
+    )
+    for format in INPUT_FORMATS:
+        try:
+            return datetime.datetime(*time.strptime(text, format)[:6])
+        except ValueError:
+            pass
+
+    # Try descriptive texts
+    if text == 'tomorrow':
+        future = datetime.datetime.now() + datetime.timedelta(days=1)
+        args = future.timetuple()[:3] + (23, 59, 59)
+        return datetime.datetime(*args)
+    elif text == 'today':
+        return datetime.datetime(*datetime.datetime.today().timetuple()[:3])
+    elif text == 'now':
+        return datetime.datetime.now()
+    elif text == 'yesterday':
+        past = datetime.datetime.now() - datetime.timedelta(days=1)
+        return datetime.datetime(*past.timetuple()[:3])
+    else:
+        days = 0
+        matched = re.match(
+            r'^((?P<weeks>\d+) ?w(eeks?)?)? ?((?P<days>\d+) ?d(ays?)?)?$', text)
+        if matched:
+            groupdict = matched.groupdict()
+            if groupdict['days']:
+                days += int(matched.groupdict()['days'])
+            if groupdict['weeks']:
+                days += int(matched.groupdict()['weeks']) * 7
+            past = datetime.datetime.now() - datetime.timedelta(days=days)
+            return datetime.datetime(*past.timetuple()[:3])
+
+    raise ValueError('Wrong date: "%s"' % text)
+
+
+def get_dict_for_attrs(obj, attrs):
+    """
+    Returns dictionary for each attribute from given ``obj``.
+    """
+    data = {}
+    for attr in attrs:
+        data[attr] = getattr(obj, attr)
+    return data
+
+
+def get_total_seconds(timedelta):
+    """
+    Backported for Python 2.5.
+
+    See http://docs.python.org/library/datetime.html.
+    """
+    return ((timedelta.microseconds + (
+            timedelta.seconds +
+            timedelta.days * 24 * 60 * 60
+        ) * 10**6) / 10**6)

rhodecode/lib/vcs/utils/hgcompat.py

@@ -0,0 +1,12 b''
+"""Mercurial libs compatibility
+
+"""
+from mercurial import archival, merge as hg_merge, patch, ui
+from mercurial.commands import clone, nullid, pull
+from mercurial.context import memctx, memfilectx
+from mercurial.error import RepoError, RepoLookupError, Abort
+from mercurial.hgweb.common import get_contact
+from mercurial.localrepo import localrepository
+from mercurial.match import match
+from mercurial.mdiff import diffopts
+from mercurial.node import hex

rhodecode/lib/vcs/utils/imports.py

@@ -0,0 +1,27 b''
+from rhodecode.lib.vcs.exceptions import VCSError
+
+
+def import_class(class_path):
+    """
+    Returns class from the given path.
+
+    For example, in order to get class located at
+    ``vcs.backends.hg.MercurialRepository``:
+
+        try:
+            hgrepo = import_class('vcs.backends.hg.MercurialRepository')
+        except VCSError:
+            # hadle error
+    """
+    splitted = class_path.split('.')
+    mod_path = '.'.join(splitted[:-1])
+    class_name = splitted[-1]
+    try:
+        class_mod = __import__(mod_path, {}, {}, [class_name])
+    except ImportError, err:
+        msg = "There was problem while trying to import backend class. "\
+            "Original error was:\n%s" % err
+        raise VCSError(msg)
+    cls = getattr(class_mod, class_name)
+
+    return cls

rhodecode/lib/vcs/utils/lazy.py

@@ -0,0 +1,27 b''
+class LazyProperty(object):
+    """
+    Decorator for easier creation of ``property`` from potentially expensive to
+    calculate attribute of the class.
+
+    Usage::
+
+      class Foo(object):
+          @LazyProperty
+          def bar(self):
+              print 'Calculating self._bar'
+              return 42
+
+    Taken from http://blog.pythonisito.com/2008/08/lazy-descriptors.html and
+    used widely.
+    """
+
+    def __init__(self, func):
+        self._func = func
+        self.__name__ = func.__name__
+        self.__doc__ = func.__doc__
+
+    def __get__(self, obj, klass=None):
+        if obj is None:
+            return None
+        result = obj.__dict__[self.__name__] = self._func(obj)
+        return result

rhodecode/lib/vcs/utils/lockfiles.py

@@ -0,0 +1,72 b''
+import os
+
+
+class LockFile(object):
+	"""Provides methods to obtain, check for, and release a file based lock which
+	should be used to handle concurrent access to the same file.
+
+	As we are a utility class to be derived from, we only use protected methods.
+
+	Locks will automatically be released on destruction"""
+	__slots__ = ("_file_path", "_owns_lock")
+
+	def __init__(self, file_path):
+		self._file_path = file_path
+		self._owns_lock = False
+
+	def __del__(self):
+		self._release_lock()
+
+	def _lock_file_path(self):
+		""":return: Path to lockfile"""
+		return "%s.lock" % (self._file_path)
+
+	def _has_lock(self):
+		""":return: True if we have a lock and if the lockfile still exists
+		:raise AssertionError: if our lock-file does not exist"""
+		if not self._owns_lock:
+			return False
+
+		return True
+
+	def _obtain_lock_or_raise(self):
+		"""Create a lock file as flag for other instances, mark our instance as lock-holder
+
+		:raise IOError: if a lock was already present or a lock file could not be written"""
+		if self._has_lock():
+			return
+		lock_file = self._lock_file_path()
+		if os.path.isfile(lock_file):
+			raise IOError("Lock for file %r did already exist, delete %r in case the lock is illegal" % (self._file_path, lock_file))
+
+		try:
+			fd = os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0)
+			os.close(fd)
+		except OSError,e:
+			raise IOError(str(e))
+
+		self._owns_lock = True
+
+	def _obtain_lock(self):
+		"""The default implementation will raise if a lock cannot be obtained.
+		Subclasses may override this method to provide a different implementation"""
+		return self._obtain_lock_or_raise()
+
+	def _release_lock(self):
+		"""Release our lock if we have one"""
+		if not self._has_lock():
+			return
+
+		# if someone removed our file beforhand, lets just flag this issue
+		# instead of failing, to make it more usable.
+		lfp = self._lock_file_path()
+		try:
+			# on bloody windows, the file needs write permissions to be removable.
+			# Why ...
+			if os.name == 'nt':
+				os.chmod(lfp, 0777)
+			# END handle win32
+			os.remove(lfp)
+		except OSError:
+			pass
+		self._owns_lock = False

rhodecode/lib/vcs/utils/ordered_dict.py

@@ -0,0 +1,102 b''
+"""Ordered dict implementation"""
+from UserDict import DictMixin
+
+
+class OrderedDict(dict, DictMixin):
+
+    def __init__(self, *args, **kwds):
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        try:
+            self.__end
+        except AttributeError:
+            self.clear()
+        self.update(*args, **kwds)
+
+    def clear(self):
+        self.__end = end = []
+        end += [None, end, end]         # sentinel node for doubly linked list
+        self.__map = {}                 # key --> [key, prev, next]
+        dict.clear(self)
+
+    def __setitem__(self, key, value):
+        if key not in self:
+            end = self.__end
+            curr = end[1]
+            curr[2] = end[1] = self.__map[key] = [key, curr, end]
+        dict.__setitem__(self, key, value)
+
+    def __delitem__(self, key):
+        dict.__delitem__(self, key)
+        key, prev, next = self.__map.pop(key)
+        prev[2] = next
+        next[1] = prev
+
+    def __iter__(self):
+        end = self.__end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.__end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def popitem(self, last=True):
+        if not self:
+            raise KeyError('dictionary is empty')
+        if last:
+            key = reversed(self).next()
+        else:
+            key = iter(self).next()
+        value = self.pop(key)
+        return key, value
+
+    def __reduce__(self):
+        items = [[k, self[k]] for k in self]
+        tmp = self.__map, self.__end
+        del self.__map, self.__end
+        inst_dict = vars(self).copy()
+        self.__map, self.__end = tmp
+        if inst_dict:
+            return (self.__class__, (items,), inst_dict)
+        return self.__class__, (items,)
+
+    def keys(self):
+        return list(self)
+
+    setdefault = DictMixin.setdefault
+    update = DictMixin.update
+    pop = DictMixin.pop
+    values = DictMixin.values
+    items = DictMixin.items
+    iterkeys = DictMixin.iterkeys
+    itervalues = DictMixin.itervalues
+    iteritems = DictMixin.iteritems
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, self.items())
+
+    def copy(self):
+        return self.__class__(self)
+
+    @classmethod
+    def fromkeys(cls, iterable, value=None):
+        d = cls()
+        for key in iterable:
+            d[key] = value
+        return d
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedDict):
+            return len(self) == len(other) and self.items() == other.items()
+        return dict.__eq__(self, other)
+
+    def __ne__(self, other):
+        return not self == other

rhodecode/lib/vcs/utils/paths.py

@@ -0,0 +1,36 b''
+import os
+
+abspath = lambda * p: os.path.abspath(os.path.join(*p))
+
+
+def get_dirs_for_path(*paths):
+    """
+    Returns list of directories, including intermediate.
+    """
+    for path in paths:
+        head = path
+        while head:
+            head, tail = os.path.split(head)
+            if head:
+                yield head
+            else:
+                # We don't need to yield empty path
+                break
+
+
+def get_dir_size(path):
+    root_path = path
+    size = 0
+    for path, dirs, files in os.walk(root_path):
+        for f in files:
+            try:
+                size += os.path.getsize(os.path.join(path, f))
+            except OSError:
+                pass
+    return size
+
+def get_user_home():
+    """
+    Returns home path of the user.
+    """
+    return os.getenv('HOME', os.getenv('USERPROFILE'))

rhodecode/lib/vcs/utils/progressbar.py

@@ -0,0 +1,419 b''
+# encoding: UTF-8
+import sys
+import datetime
+from string import Template
+from rhodecode.lib.vcs.utils.filesize import filesizeformat
+from rhodecode.lib.vcs.utils.helpers import get_total_seconds
+
+
+class ProgressBarError(Exception):
+    pass
+
+class AlreadyFinishedError(ProgressBarError):
+    pass
+
+
+class ProgressBar(object):
+
+    default_elements = ['percentage', 'bar', 'steps']
+
+    def __init__(self, steps=100, stream=None, elements=None):
+        self.step = 0
+        self.steps = steps
+        self.stream = stream or sys.stderr
+        self.bar_char = '='
+        self.width = 50
+        self.separator = ' | '
+        self.elements = elements or self.default_elements
+        self.started = None
+        self.finished = False
+        self.steps_label = 'Step'
+        self.time_label = 'Time'
+        self.eta_label = 'ETA'
+        self.speed_label = 'Speed'
+        self.transfer_label = 'Transfer'
+
+    def __str__(self):
+        return self.get_line()
+
+    def __iter__(self):
+        start = self.step
+        end = self.steps + 1
+        for x in xrange(start, end):
+            self.render(x)
+            yield x
+
+    def get_separator(self):
+        return self.separator
+
+    def get_bar_char(self):
+        return self.bar_char
+
+    def get_bar(self):
+        char = self.get_bar_char()
+        perc = self.get_percentage()
+        length = int(self.width * perc / 100)
+        bar = char * length
+        bar = bar.ljust(self.width)
+        return bar
+
+    def get_elements(self):
+        return self.elements
+
+    def get_template(self):
+        separator = self.get_separator()
+        elements = self.get_elements()
+        return Template(separator.join((('$%s' % e) for e in elements)))
+
+    def get_total_time(self, current_time=None):
+        if current_time is None:
+            current_time = datetime.datetime.now()
+        if not self.started:
+            return datetime.timedelta()
+        return current_time - self.started
+
+    def get_rendered_total_time(self):
+        delta = self.get_total_time()
+        if not delta:
+            ttime = '-'
+        else:
+            ttime = str(delta)
+        return '%s %s' % (self.time_label, ttime)
+
+    def get_eta(self, current_time=None):
+        if current_time is None:
+            current_time = datetime.datetime.now()
+        if self.step == 0:
+            return datetime.timedelta()
+        total_seconds = get_total_seconds(self.get_total_time())
+        eta_seconds = total_seconds * self.steps / self.step - total_seconds
+        return datetime.timedelta(seconds=int(eta_seconds))
+
+    def get_rendered_eta(self):
+        eta = self.get_eta()
+        if not eta:
+            eta = '--:--:--'
+        else:
+            eta = str(eta).rjust(8)
+        return '%s: %s' % (self.eta_label, eta)
+
+    def get_percentage(self):
+        return float(self.step) / self.steps * 100
+
+    def get_rendered_percentage(self):
+        perc = self.get_percentage()
+        return ('%s%%' % (int(perc))).rjust(5)
+
+    def get_rendered_steps(self):
+        return '%s: %s/%s' % (self.steps_label, self.step, self.steps)
+
+    def get_rendered_speed(self, step=None, total_seconds=None):
+        if step is None:
+            step = self.step
+        if total_seconds is None:
+            total_seconds = get_total_seconds(self.get_total_time())
+        if step <= 0 or total_seconds <= 0:
+            speed = '-'
+        else:
+            speed = filesizeformat(float(step) / total_seconds)
+        return '%s: %s/s' % (self.speed_label, speed)
+
+    def get_rendered_transfer(self, step=None, steps=None):
+        if step is None:
+            step = self.step
+        if steps is None:
+            steps = self.steps
+
+        if steps <= 0:
+            return '%s: -' % self.transfer_label
+        total = filesizeformat(float(steps))
+        if step <= 0:
+            transferred = '-'
+        else:
+            transferred = filesizeformat(float(step))
+        return '%s: %s / %s' % (self.transfer_label, transferred, total)
+
+    def get_context(self):
+        return {
+            'percentage': self.get_rendered_percentage(),
+            'bar': self.get_bar(),
+            'steps': self.get_rendered_steps(),
+            'time': self.get_rendered_total_time(),
+            'eta': self.get_rendered_eta(),
+            'speed': self.get_rendered_speed(),
+            'transfer': self.get_rendered_transfer(),
+        }
+
+    def get_line(self):
+        template = self.get_template()
+        context = self.get_context()
+        return template.safe_substitute(**context)
+
+    def write(self, data):
+        self.stream.write(data)
+
+    def render(self, step):
+        if not self.started:
+            self.started = datetime.datetime.now()
+        if self.finished:
+            raise AlreadyFinishedError
+        self.step = step
+        self.write('\r%s' % self)
+        if step == self.steps:
+            self.finished = True
+        if step == self.steps:
+            self.write('\n')
+
+
+"""
+termcolors.py
+
+Grabbed from Django (http://www.djangoproject.com)
+"""
+
+color_names = ('black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white')
+foreground = dict([(color_names[x], '3%s' % x) for x in range(8)])
+background = dict([(color_names[x], '4%s' % x) for x in range(8)])
+
+RESET = '0'
+opt_dict = {'bold': '1', 'underscore': '4', 'blink': '5', 'reverse': '7', 'conceal': '8'}
+
+def colorize(text='', opts=(), **kwargs):
+    """
+    Returns your text, enclosed in ANSI graphics codes.
+
+    Depends on the keyword arguments 'fg' and 'bg', and the contents of
+    the opts tuple/list.
+
+    Returns the RESET code if no parameters are given.
+
+    Valid colors:
+        'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white'
+
+    Valid options:
+        'bold'
+        'underscore'
+        'blink'
+        'reverse'
+        'conceal'
+        'noreset' - string will not be auto-terminated with the RESET code
+
+    Examples:
+        colorize('hello', fg='red', bg='blue', opts=('blink',))
+        colorize()
+        colorize('goodbye', opts=('underscore',))
+        print colorize('first line', fg='red', opts=('noreset',))
+        print 'this should be red too'
+        print colorize('and so should this')
+        print 'this should not be red'
+    """
+    code_list = []
+    if text == '' and len(opts) == 1 and opts[0] == 'reset':
+        return '\x1b[%sm' % RESET
+    for k, v in kwargs.iteritems():
+        if k == 'fg':
+            code_list.append(foreground[v])
+        elif k == 'bg':
+            code_list.append(background[v])
+    for o in opts:
+        if o in opt_dict:
+            code_list.append(opt_dict[o])
+    if 'noreset' not in opts:
+        text = text + '\x1b[%sm' % RESET
+    return ('\x1b[%sm' % ';'.join(code_list)) + text
+
+def make_style(opts=(), **kwargs):
+    """
+    Returns a function with default parameters for colorize()
+
+    Example:
+        bold_red = make_style(opts=('bold',), fg='red')
+        print bold_red('hello')
+        KEYWORD = make_style(fg='yellow')
+        COMMENT = make_style(fg='blue', opts=('bold',))
+    """
+    return lambda text: colorize(text, opts, **kwargs)
+
+NOCOLOR_PALETTE = 'nocolor'
+DARK_PALETTE = 'dark'
+LIGHT_PALETTE = 'light'
+
+PALETTES = {
+    NOCOLOR_PALETTE: {
+        'ERROR':        {},
+        'NOTICE':       {},
+        'SQL_FIELD':    {},
+        'SQL_COLTYPE':  {},
+        'SQL_KEYWORD':  {},
+        'SQL_TABLE':    {},
+        'HTTP_INFO':         {},
+        'HTTP_SUCCESS':      {},
+        'HTTP_REDIRECT':     {},
+        'HTTP_NOT_MODIFIED': {},
+        'HTTP_BAD_REQUEST':  {},
+        'HTTP_NOT_FOUND':    {},
+        'HTTP_SERVER_ERROR': {},
+    },
+    DARK_PALETTE: {
+        'ERROR':        { 'fg': 'red', 'opts': ('bold',) },
+        'NOTICE':       { 'fg': 'red' },
+        'SQL_FIELD':    { 'fg': 'green', 'opts': ('bold',) },
+        'SQL_COLTYPE':  { 'fg': 'green' },
+        'SQL_KEYWORD':  { 'fg': 'yellow' },
+        'SQL_TABLE':    { 'opts': ('bold',) },
+        'HTTP_INFO':         { 'opts': ('bold',) },
+        'HTTP_SUCCESS':      { },
+        'HTTP_REDIRECT':     { 'fg': 'green' },
+        'HTTP_NOT_MODIFIED': { 'fg': 'cyan' },
+        'HTTP_BAD_REQUEST':  { 'fg': 'red', 'opts': ('bold',) },
+        'HTTP_NOT_FOUND':    { 'fg': 'yellow' },
+        'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) },
+    },
+    LIGHT_PALETTE: {
+        'ERROR':        { 'fg': 'red', 'opts': ('bold',) },
+        'NOTICE':       { 'fg': 'red' },
+        'SQL_FIELD':    { 'fg': 'green', 'opts': ('bold',) },
+        'SQL_COLTYPE':  { 'fg': 'green' },
+        'SQL_KEYWORD':  { 'fg': 'blue' },
+        'SQL_TABLE':    { 'opts': ('bold',) },
+        'HTTP_INFO':         { 'opts': ('bold',) },
+        'HTTP_SUCCESS':      { },
+        'HTTP_REDIRECT':     { 'fg': 'green', 'opts': ('bold',) },
+        'HTTP_NOT_MODIFIED': { 'fg': 'green' },
+        'HTTP_BAD_REQUEST':  { 'fg': 'red', 'opts': ('bold',) },
+        'HTTP_NOT_FOUND':    { 'fg': 'red' },
+        'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) },
+    }
+}
+DEFAULT_PALETTE = DARK_PALETTE
+
+# ---------------------------- #
+# --- End of termcolors.py --- #
+# ---------------------------- #
+
+
+class ColoredProgressBar(ProgressBar):
+
+    BAR_COLORS = (
+        (10, 'red'),
+        (30, 'magenta'),
+        (50, 'yellow'),
+        (99, 'green'),
+        (100, 'blue'),
+    )
+
+    def get_line(self):
+        line = super(ColoredProgressBar, self).get_line()
+        perc = self.get_percentage()
+        if perc > 100:
+            color = 'blue'
+        for max_perc, color in self.BAR_COLORS:
+            if perc <= max_perc:
+                break
+        return colorize(line, fg=color)
+
+
+class AnimatedProgressBar(ProgressBar):
+
+    def get_bar_char(self):
+        chars = '-/|\\'
+        if self.step >= self.steps:
+            return '='
+        return chars[self.step % len(chars)]
+
+
+class BarOnlyProgressBar(ProgressBar):
+
+    default_elements = ['bar', 'steps']
+
+    def get_bar(self):
+        bar = super(BarOnlyProgressBar, self).get_bar()
+        perc = self.get_percentage()
+        perc_text = '%s%%' % int(perc)
+        text = (' %s%% ' % (perc_text)).center(self.width, '=')
+        L = text.find(' ')
+        R = text.rfind(' ')
+        bar = ' '.join((bar[:L], perc_text, bar[R:]))
+        return bar
+
+
+class AnimatedColoredProgressBar(AnimatedProgressBar,
+                                 ColoredProgressBar):
+    pass
+
+
+class BarOnlyColoredProgressBar(ColoredProgressBar,
+                                BarOnlyProgressBar):
+    pass
+
+
+
+def main():
+    import time
+
+    print "Standard progress bar..."
+    bar = ProgressBar(30)
+    for x in xrange(1, 31):
+            bar.render(x)
+            time.sleep(0.02)
+    bar.stream.write('\n')
+    print
+
+    print "Empty bar..."
+    bar = ProgressBar(50)
+    bar.render(0)
+    print
+    print
+
+    print "Colored bar..."
+    bar = ColoredProgressBar(20)
+    for x in bar:
+        time.sleep(0.01)
+    print
+
+    print "Animated char bar..."
+    bar = AnimatedProgressBar(20)
+    for x in bar:
+        time.sleep(0.01)
+    print
+
+    print "Animated + colored char bar..."
+    bar = AnimatedColoredProgressBar(20)
+    for x in bar:
+        time.sleep(0.01)
+    print
+
+    print "Bar only ..."
+    bar = BarOnlyProgressBar(20)
+    for x in bar:
+        time.sleep(0.01)
+    print
+
+    print "Colored, longer bar-only, eta, total time ..."
+    bar = BarOnlyColoredProgressBar(40)
+    bar.width = 60
+    bar.elements += ['time', 'eta']
+    for x in bar:
+        time.sleep(0.01)
+    print
+    print
+
+    print "File transfer bar, breaks after 2 seconds ..."
+    total_bytes = 1024 * 1024 * 2
+    bar = ProgressBar(total_bytes)
+    bar.width = 50
+    bar.elements.remove('steps')
+    bar.elements += ['transfer', 'time', 'eta', 'speed']
+    for x in xrange(0, bar.steps, 1024):
+        bar.render(x)
+        time.sleep(0.01)
+        now = datetime.datetime.now()
+        if now - bar.started >= datetime.timedelta(seconds=2):
+            break
+    print
+    print
+
+
+
+if __name__ == '__main__':
+    main()

rhodecode/lib/vcs/utils/termcolors.py

@@ -0,0 +1,200 b''
+"""
+termcolors.py
+
+Grabbed from Django (http://www.djangoproject.com)
+"""
+
+color_names = ('black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white')
+foreground = dict([(color_names[x], '3%s' % x) for x in range(8)])
+background = dict([(color_names[x], '4%s' % x) for x in range(8)])
+
+RESET = '0'
+opt_dict = {'bold': '1', 'underscore': '4', 'blink': '5', 'reverse': '7', 'conceal': '8'}
+
+def colorize(text='', opts=(), **kwargs):
+    """
+    Returns your text, enclosed in ANSI graphics codes.
+
+    Depends on the keyword arguments 'fg' and 'bg', and the contents of
+    the opts tuple/list.
+
+    Returns the RESET code if no parameters are given.
+
+    Valid colors:
+        'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white'
+
+    Valid options:
+        'bold'
+        'underscore'
+        'blink'
+        'reverse'
+        'conceal'
+        'noreset' - string will not be auto-terminated with the RESET code
+
+    Examples:
+        colorize('hello', fg='red', bg='blue', opts=('blink',))
+        colorize()
+        colorize('goodbye', opts=('underscore',))
+        print colorize('first line', fg='red', opts=('noreset',))
+        print 'this should be red too'
+        print colorize('and so should this')
+        print 'this should not be red'
+    """
+    code_list = []
+    if text == '' and len(opts) == 1 and opts[0] == 'reset':
+        return '\x1b[%sm' % RESET
+    for k, v in kwargs.iteritems():
+        if k == 'fg':
+            code_list.append(foreground[v])
+        elif k == 'bg':
+            code_list.append(background[v])
+    for o in opts:
+        if o in opt_dict:
+            code_list.append(opt_dict[o])
+    if 'noreset' not in opts:
+        text = text + '\x1b[%sm' % RESET
+    return ('\x1b[%sm' % ';'.join(code_list)) + text
+
+def make_style(opts=(), **kwargs):
+    """
+    Returns a function with default parameters for colorize()
+
+    Example:
+        bold_red = make_style(opts=('bold',), fg='red')
+        print bold_red('hello')
+        KEYWORD = make_style(fg='yellow')
+        COMMENT = make_style(fg='blue', opts=('bold',))
+    """
+    return lambda text: colorize(text, opts, **kwargs)
+
+NOCOLOR_PALETTE = 'nocolor'
+DARK_PALETTE = 'dark'
+LIGHT_PALETTE = 'light'
+
+PALETTES = {
+    NOCOLOR_PALETTE: {
+        'ERROR':        {},
+        'NOTICE':       {},
+        'SQL_FIELD':    {},
+        'SQL_COLTYPE':  {},
+        'SQL_KEYWORD':  {},
+        'SQL_TABLE':    {},
+        'HTTP_INFO':         {},
+        'HTTP_SUCCESS':      {},
+        'HTTP_REDIRECT':     {},
+        'HTTP_NOT_MODIFIED': {},
+        'HTTP_BAD_REQUEST':  {},
+        'HTTP_NOT_FOUND':    {},
+        'HTTP_SERVER_ERROR': {},
+    },
+    DARK_PALETTE: {
+        'ERROR':        { 'fg': 'red', 'opts': ('bold',) },
+        'NOTICE':       { 'fg': 'red' },
+        'SQL_FIELD':    { 'fg': 'green', 'opts': ('bold',) },
+        'SQL_COLTYPE':  { 'fg': 'green' },
+        'SQL_KEYWORD':  { 'fg': 'yellow' },
+        'SQL_TABLE':    { 'opts': ('bold',) },
+        'HTTP_INFO':         { 'opts': ('bold',) },
+        'HTTP_SUCCESS':      { },
+        'HTTP_REDIRECT':     { 'fg': 'green' },
+        'HTTP_NOT_MODIFIED': { 'fg': 'cyan' },
+        'HTTP_BAD_REQUEST':  { 'fg': 'red', 'opts': ('bold',) },
+        'HTTP_NOT_FOUND':    { 'fg': 'yellow' },
+        'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) },
+    },
+    LIGHT_PALETTE: {
+        'ERROR':        { 'fg': 'red', 'opts': ('bold',) },
+        'NOTICE':       { 'fg': 'red' },
+        'SQL_FIELD':    { 'fg': 'green', 'opts': ('bold',) },
+        'SQL_COLTYPE':  { 'fg': 'green' },
+        'SQL_KEYWORD':  { 'fg': 'blue' },
+        'SQL_TABLE':    { 'opts': ('bold',) },
+        'HTTP_INFO':         { 'opts': ('bold',) },
+        'HTTP_SUCCESS':      { },
+        'HTTP_REDIRECT':     { 'fg': 'green', 'opts': ('bold',) },
+        'HTTP_NOT_MODIFIED': { 'fg': 'green' },
+        'HTTP_BAD_REQUEST':  { 'fg': 'red', 'opts': ('bold',) },
+        'HTTP_NOT_FOUND':    { 'fg': 'red' },
+        'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) },
+    }
+}
+DEFAULT_PALETTE = DARK_PALETTE
+
+def parse_color_setting(config_string):
+    """Parse a DJANGO_COLORS environment variable to produce the system palette
+
+    The general form of a pallete definition is:
+
+        "palette;role=fg;role=fg/bg;role=fg,option,option;role=fg/bg,option,option"
+
+    where:
+        palette is a named palette; one of 'light', 'dark', or 'nocolor'.
+        role is a named style used by Django
+        fg is a background color.
+        bg is a background color.
+        option is a display options.
+
+    Specifying a named palette is the same as manually specifying the individual
+    definitions for each role. Any individual definitions following the pallete
+    definition will augment the base palette definition.
+
+    Valid roles:
+        'error', 'notice', 'sql_field', 'sql_coltype', 'sql_keyword', 'sql_table',
+        'http_info', 'http_success', 'http_redirect', 'http_bad_request',
+        'http_not_found', 'http_server_error'
+
+    Valid colors:
+        'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white'
+
+    Valid options:
+        'bold', 'underscore', 'blink', 'reverse', 'conceal'
+
+    """
+    if not config_string:
+        return PALETTES[DEFAULT_PALETTE]
+
+    # Split the color configuration into parts
+    parts = config_string.lower().split(';')
+    palette = PALETTES[NOCOLOR_PALETTE].copy()
+    for part in parts:
+        if part in PALETTES:
+            # A default palette has been specified
+            palette.update(PALETTES[part])
+        elif '=' in part:
+            # Process a palette defining string
+            definition = {}
+
+            # Break the definition into the role,
+            # plus the list of specific instructions.
+            # The role must be in upper case
+            role, instructions = part.split('=')
+            role = role.upper()
+
+            styles = instructions.split(',')
+            styles.reverse()
+
+            # The first instruction can contain a slash
+            # to break apart fg/bg.
+            colors = styles.pop().split('/')
+            colors.reverse()
+            fg = colors.pop()
+            if fg in color_names:
+                definition['fg'] = fg
+            if colors and colors[-1] in color_names:
+                definition['bg'] = colors[-1]
+
+            # All remaining instructions are options
+            opts = tuple(s for s in styles if s in opt_dict.keys())
+            if opts:
+                definition['opts'] = opts
+
+            # The nocolor palette has all available roles.
+            # Use that palette as the basis for determining
+            # if the role is valid.
+            if role in PALETTES[NOCOLOR_PALETTE] and definition:
+                palette[role] = definition
+
+    # If there are no colors specified, return the empty palette.
+    if palette == PALETTES[NOCOLOR_PALETTE]:
+        return None
+    return palette

.hgignore

@@ -17,3 +17,4 b' syntax: regexp'
 ^test\.db$
 ^RhodeCode\.egg-info$
 ^rc\.ini$
+^fabfile.py

requires.txt

@@ -10,9 +10,8 b' celery>=2.2.5,<2.3'
 babel
 python-dateutil>=1.5.0,<2.0.0
 dulwich>=0.8.0,<0.9.0
-vcs>=0.2.3.dev
 webob==1.0.8
-markdown==2.0.3
+markdown==2.1.1
 docutils==0.8.1
 py-bcrypt
 mercurial>=2.1,<2.2
  No newline at end of file

rhodecode/__init__.py

@@ -49,9 +49,8 b' requirements = ['
     "babel",
     "python-dateutil>=1.5.0,<2.0.0",
     "dulwich>=0.8.0,<0.9.0",
-    "vcs>=0.2.3.dev",
     "webob==1.0.8",
-    "markdown==2.0.3",
+    "markdown==2.1.1",
     "docutils==0.8.1",
 ]
 

rhodecode/controllers/changelog.py

@@ -37,7 +37,7 b' from rhodecode.lib.base import BaseRepoC'
 from rhodecode.lib.helpers import RepoPage
 from rhodecode.lib.compat import json
 
-from vcs.exceptions import RepositoryError, ChangesetDoesNotExistError
+from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetDoesNotExistError
 from rhodecode.model.db import Repository
 
 log = logging.getLogger(__name__)

rhodecode/controllers/changeset.py

@@ -33,9 +33,9 b' from pylons.i18n.translation import _'
 from pylons.controllers.util import redirect
 from pylons.decorators import jsonify
 
-from vcs.exceptions import RepositoryError, ChangesetError, \
+from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetError, \
     ChangesetDoesNotExistError
-from vcs.nodes import FileNode
+from rhodecode.lib.vcs.nodes import FileNode
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator

rhodecode/controllers/files.py

@@ -32,11 +32,11 b' from pylons.i18n.translation import _'
 from pylons.controllers.util import redirect
 from pylons.decorators import jsonify
 
-from vcs.conf import settings
-from vcs.exceptions import RepositoryError, ChangesetDoesNotExistError, \
+from rhodecode.lib.vcs.conf import settings
+from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetDoesNotExistError, \
     EmptyRepositoryError, ImproperArchiveTypeError, VCSError, \
     NodeAlreadyExistsError
-from vcs.nodes import FileNode
+from rhodecode.lib.vcs.nodes import FileNode
 
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib import convert_line_endings, detect_mode, safe_str

rhodecode/controllers/summary.py

@@ -31,7 +31,7 b' from datetime import timedelta, date'
 from itertools import product
 from urlparse import urlparse
 
-from vcs.exceptions import ChangesetError, EmptyRepositoryError, \
+from rhodecode.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, \
     NodeDoesNotExistError
 
 from pylons import tmpl_context as c, request, url, config

rhodecode/lib/__init__.py

@@ -25,7 +25,7 b''
 
 import os
 import re
-from vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
 
 
 def __get_lem():
@@ -138,7 +138,6 b' def convert_line_endings(line, mode):'
             line = replace(line, '\r\n', '\r')
             line = replace(line, '\n', '\r')
     elif mode == 2:
-            import re
             line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
     return line
 
@@ -341,7 +340,8 b' def age(curdate):'
     pos = 1
     for scale in agescales:
         if scale[1] <= age_seconds:
-            if pos == 6:pos = 5
+            if pos == 6:
+                pos = 5
             return '%s %s' % (time_ago_in_words(curdate,
                                                 agescales[pos][0]), _('ago'))
         pos += 1
@@ -404,8 +404,8 b' def get_changeset_safe(repo, rev):'
     :param repo:
     :param rev:
     """
-    from vcs.backends.base import BaseRepository
-    from vcs.exceptions import RepositoryError
+    from rhodecode.lib.vcs.backends.base import BaseRepository
+    from rhodecode.lib.vcs.exceptions import RepositoryError
     if not isinstance(repo, BaseRepository):
         raise Exception('You must pass an Repository '
                         'object as first argument got %s', type(repo))
@@ -427,8 +427,8 b' def get_current_revision(quiet=False):'
     """
 
     try:
-        from vcs import get_repo
-        from vcs.utils.helpers import get_scm
+        from rhodecode.lib.vcs import get_repo
+        from rhodecode.lib.vcs.utils.helpers import get_scm
         repopath = os.path.join(os.path.dirname(__file__), '..', '..')
         scm = get_scm(repopath)[0]
         repo = get_repo(path=repopath, alias=scm)

rhodecode/lib/annotate.py

@@ -11,8 +11,8 b''
     :license: GPLv3, see COPYING for more details.
 """
 
-from vcs.exceptions import VCSError
-from vcs.nodes import FileNode
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.nodes import FileNode
 from pygments.formatters import HtmlFormatter
 from pygments import highlight
 

rhodecode/lib/celerylib/__init__.py

@@ -34,7 +34,7 b' from pylons import config'
 from hashlib import md5
 from decorator import decorator
 
-from vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
 from rhodecode import CELERY_ON
 from rhodecode.lib import str2bool, safe_str
 from rhodecode.lib.pidlock import DaemonLock, LockHeld

rhodecode/lib/celerylib/tasks.py

@@ -37,7 +37,7 b' from string import lower'
 from pylons import config, url
 from pylons.i18n.translation import _
 
-from vcs import get_backend
+from rhodecode.lib.vcs import get_backend
 
 from rhodecode import CELERY_ON
 from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP, safe_str

rhodecode/lib/dbmigrate/schema/db_1_2_0.py

@@ -34,10 +34,10 b' from sqlalchemy.ext.hybrid import hybrid'
 from sqlalchemy.orm import relationship, joinedload, class_mapper, validates
 from beaker.cache import cache_region, region_invalidate
 
-from vcs import get_backend
-from vcs.utils.helpers import get_scm
-from vcs.exceptions import VCSError
-from vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs import get_backend
+from rhodecode.lib.vcs.utils.helpers import get_scm
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
 
 from rhodecode.lib import str2bool, safe_str, get_changeset_safe, \
     generate_api_key, safe_unicode

rhodecode/lib/diffs.py

@@ -32,8 +32,8 b' from itertools import tee, imap'
 
 from pylons.i18n.translation import _
 
-from vcs.exceptions import VCSError
-from vcs.nodes import FileNode
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.nodes import FileNode
 
 from rhodecode.lib.utils import EmptyChangeset
 

rhodecode/lib/helpers.py

@@ -318,7 +318,7 b' flash = _Flash()'
 #==============================================================================
 # SCM FILTERS available via h.
 #==============================================================================
-from vcs.utils import author_name, author_email
+from rhodecode.lib.vcs.utils import author_name, author_email
 from rhodecode.lib import credentials_filter, age as _age
 from rhodecode.model.db import User
 

rhodecode/lib/indexers/daemon.py

@@ -43,7 +43,7 b' from rhodecode.model.scm import ScmModel'
 from rhodecode.lib import safe_unicode
 from rhodecode.lib.indexers import INDEX_EXTENSIONS, SCHEMA, IDX_NAME
 
-from vcs.exceptions import ChangesetError, RepositoryError, \
+from rhodecode.lib.vcs.exceptions import ChangesetError, RepositoryError, \
     NodeDoesNotExistError
 
 from whoosh.index import create_in, open_dir

rhodecode/lib/middleware/simplegit.py

@@ -30,6 +30,7 b' import traceback'
 
 from dulwich import server as dulserver
 
+
 class SimpleGitUploadPackHandler(dulserver.UploadPackHandler):
 
     def handle(self):

rhodecode/lib/middleware/simplehg.py

@@ -154,7 +154,6 b' class SimpleHg(BaseVCSController):'
         baseui = make_ui('db')
         self.__inject_extras(repo_path, baseui, extras)
 
-
         # quick check if that dir exists...
         if is_valid_repo(repo_name, self.basepath) is False:
             return HTTPNotFound()(environ, start_response)
@@ -221,7 +220,6 b' class SimpleHg(BaseVCSController):'
                 else:
                     return 'pull'
 
-
     def __inject_extras(self, repo_path, baseui, extras={}):
         """
         Injects some extra params into baseui instance

rhodecode/lib/utils.py

@@ -40,11 +40,11 b' from mercurial import ui, config'
 
 from webhelpers.text import collapse, remove_formatting, strip_tags
 
-from vcs import get_backend
-from vcs.backends.base import BaseChangeset
-from vcs.utils.lazy import LazyProperty
-from vcs.utils.helpers import get_scm
-from vcs.exceptions import VCSError
+from rhodecode.lib.vcs import get_backend
+from rhodecode.lib.vcs.backends.base import BaseChangeset
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.utils.helpers import get_scm
+from rhodecode.lib.vcs.exceptions import VCSError
 
 from rhodecode.lib.caching_query import FromCache
 

rhodecode/model/db.py

@@ -34,10 +34,10 b' from sqlalchemy.ext.hybrid import hybrid'
 from sqlalchemy.orm import relationship, joinedload, class_mapper, validates
 from beaker.cache import cache_region, region_invalidate
 
-from vcs import get_backend
-from vcs.utils.helpers import get_scm
-from vcs.exceptions import VCSError
-from vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs import get_backend
+from rhodecode.lib.vcs.utils.helpers import get_scm
+from rhodecode.lib.vcs.exceptions import VCSError
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
 
 from rhodecode.lib import str2bool, safe_str, get_changeset_safe, safe_unicode
 from rhodecode.lib.compat import json

rhodecode/model/repo.py

@@ -28,7 +28,7 b' import logging'
 import traceback
 from datetime import datetime
 
-from vcs.backends import get_backend
+from rhodecode.lib.vcs.backends import get_backend
 
 from rhodecode.lib import LazyProperty
 from rhodecode.lib import safe_str, safe_unicode

rhodecode/model/scm.py

@@ -28,10 +28,10 b' import traceback'
 import logging
 import cStringIO
 
-from vcs import get_backend
-from vcs.exceptions import RepositoryError
-from vcs.utils.lazy import LazyProperty
-from vcs.nodes import FileNode
+from rhodecode.lib.vcs import get_backend
+from rhodecode.lib.vcs.exceptions import RepositoryError
+from rhodecode.lib.vcs.utils.lazy import LazyProperty
+from rhodecode.lib.vcs.nodes import FileNode
 
 from rhodecode import BACKENDS
 from rhodecode.lib import helpers as h
@@ -359,9 +359,9 b' class ScmModel(BaseModel):'
                       content, f_path):
 
         if repo.alias == 'hg':
-            from vcs.backends.hg import MercurialInMemoryChangeset as IMC
+            from rhodecode.lib.vcs.backends.hg import MercurialInMemoryChangeset as IMC
         elif repo.alias == 'git':
-            from vcs.backends.git import GitInMemoryChangeset as IMC
+            from rhodecode.lib.vcs.backends.git import GitInMemoryChangeset as IMC
 
         # decoding here will force that we have proper encoded values
         # in any other case this will throw exceptions and deny commit
@@ -385,9 +385,9 b' class ScmModel(BaseModel):'
     def create_node(self, repo, repo_name, cs, user, author, message, content,
                       f_path):
         if repo.alias == 'hg':
-            from vcs.backends.hg import MercurialInMemoryChangeset as IMC
+            from rhodecode.lib.vcs.backends.hg import MercurialInMemoryChangeset as IMC
         elif repo.alias == 'git':
-            from vcs.backends.git import GitInMemoryChangeset as IMC
+            from rhodecode.lib.vcs.backends.git import GitInMemoryChangeset as IMC
         # decoding here will force that we have proper encoded values
         # in any other case this will throw exceptions and deny commit
 

rhodecode/tests/functional/test_admin_repos.py

@@ -1,18 +1,16 b''
 # -*- coding: utf-8 -*-
 
 import os
-import vcs
+from rhodecode.lib import vcs
 
 from rhodecode.model.db import Repository
 from rhodecode.tests import *
 
 class TestAdminReposController(TestController):
 
-
     def __make_repo(self):
         pass
 
-
     def test_index(self):
         self.log_user()
         response = self.app.get(url('repos'))

rhodecode/tests/rhodecode_crawler.py

@@ -31,11 +31,10 b''
 import cookielib
 import urllib
 import urllib2
-import vcs
 import time
 
 from os.path import join as jn
-
+from rhodecode.lib import vcs
 
 BASE_URI = 'http://127.0.0.1:5000/%s'
 PROJECT = 'CPython'
@@ -52,7 +51,6 b' o.addheaders = ['
 urllib2.install_opener(o)
 
 
-
 def test_changelog_walk(pages=100):
     total_time = 0
     for i in range(1, pages):
@@ -67,7 +65,6 b' def test_changelog_walk(pages=100):'
         total_time += e
         print 'visited %s size:%s req:%s ms' % (full_uri, size, e)
 
-
     print 'total_time', total_time
     print 'average on req', total_time / float(pages)
 
@@ -103,7 +100,7 b' def test_files_walk(limit=100):'
     repo = vcs.get_repo(jn(PROJECT_PATH, PROJECT))
 
     from rhodecode.lib.compat import OrderedSet
-    from vcs.exceptions import RepositoryError
+    from rhodecode.lib.vcs.exceptions import RepositoryError
 
     paths_ = OrderedSet([''])
     try:
@@ -141,7 +138,6 b' def test_files_walk(limit=100):'
     print 'average on req', total_time / float(cnt)
 
 
-
 test_changelog_walk(40)
 time.sleep(2)
 test_changeset_walk(limit=100)

setup.py

@@ -7,10 +7,7 b' from rhodecode import requirements'
 if __py_version__ < (2, 5):
     raise Exception('RhodeCode requires python 2.5 or later')
 
-
 dependency_links = [
- "https://secure.rhodecode.org/vcs/archive/default.zip#egg=vcs-0.2.3.dev",
- "https://bitbucket.org/marcinkuzminski/vcs/get/default.zip#egg=vcs-0.2.3.dev",
 ]
 
 classifiers = [