diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -17,3 +17,4 @@ syntax: regexp ^test\.db$ ^RhodeCode\.egg-info$ ^rc\.ini$ +^fabfile.py diff --git a/requires.txt b/requires.txt --- a/requires.txt +++ b/requires.txt @@ -10,9 +10,8 @@ celery>=2.2.5,<2.3 babel python-dateutil>=1.5.0,<2.0.0 dulwich>=0.8.0,<0.9.0 -vcs>=0.2.3.dev webob==1.0.8 -markdown==2.0.3 +markdown==2.1.1 docutils==0.8.1 py-bcrypt mercurial>=2.1,<2.2 \ No newline at end of file diff --git a/rhodecode/__init__.py b/rhodecode/__init__.py --- a/rhodecode/__init__.py +++ b/rhodecode/__init__.py @@ -49,9 +49,8 @@ requirements = [ "babel", "python-dateutil>=1.5.0,<2.0.0", "dulwich>=0.8.0,<0.9.0", - "vcs>=0.2.3.dev", "webob==1.0.8", - "markdown==2.0.3", + "markdown==2.1.1", "docutils==0.8.1", ] diff --git a/rhodecode/controllers/changelog.py b/rhodecode/controllers/changelog.py --- a/rhodecode/controllers/changelog.py +++ b/rhodecode/controllers/changelog.py @@ -37,7 +37,7 @@ from rhodecode.lib.base import BaseRepoC from rhodecode.lib.helpers import RepoPage from rhodecode.lib.compat import json -from vcs.exceptions import RepositoryError, ChangesetDoesNotExistError +from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetDoesNotExistError from rhodecode.model.db import Repository log = logging.getLogger(__name__) diff --git a/rhodecode/controllers/changeset.py b/rhodecode/controllers/changeset.py --- a/rhodecode/controllers/changeset.py +++ b/rhodecode/controllers/changeset.py @@ -33,9 +33,9 @@ from pylons.i18n.translation import _ from pylons.controllers.util import redirect from pylons.decorators import jsonify -from vcs.exceptions import RepositoryError, ChangesetError, \ +from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetError, \ ChangesetDoesNotExistError -from vcs.nodes import FileNode +from rhodecode.lib.vcs.nodes import FileNode import rhodecode.lib.helpers as h from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator diff --git a/rhodecode/controllers/files.py b/rhodecode/controllers/files.py --- a/rhodecode/controllers/files.py +++ b/rhodecode/controllers/files.py @@ -32,11 +32,11 @@ from pylons.i18n.translation import _ from pylons.controllers.util import redirect from pylons.decorators import jsonify -from vcs.conf import settings -from vcs.exceptions import RepositoryError, ChangesetDoesNotExistError, \ +from rhodecode.lib.vcs.conf import settings +from rhodecode.lib.vcs.exceptions import RepositoryError, ChangesetDoesNotExistError, \ EmptyRepositoryError, ImproperArchiveTypeError, VCSError, \ NodeAlreadyExistsError -from vcs.nodes import FileNode +from rhodecode.lib.vcs.nodes import FileNode from rhodecode.lib.compat import OrderedDict from rhodecode.lib import convert_line_endings, detect_mode, safe_str diff --git a/rhodecode/controllers/summary.py b/rhodecode/controllers/summary.py --- a/rhodecode/controllers/summary.py +++ b/rhodecode/controllers/summary.py @@ -31,7 +31,7 @@ from datetime import timedelta, date from itertools import product from urlparse import urlparse -from vcs.exceptions import ChangesetError, EmptyRepositoryError, \ +from rhodecode.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, \ NodeDoesNotExistError from pylons import tmpl_context as c, request, url, config diff --git a/rhodecode/lib/__init__.py b/rhodecode/lib/__init__.py --- a/rhodecode/lib/__init__.py +++ b/rhodecode/lib/__init__.py @@ -25,7 +25,7 @@ import os import re -from vcs.utils.lazy import LazyProperty +from rhodecode.lib.vcs.utils.lazy import LazyProperty def __get_lem(): @@ -82,7 +82,7 @@ ALL_READMES = [ # extension together with weights to search lower is first RST_EXTS = [ ('', 0), ('.rst', 1), ('.rest', 1), - ('.RST', 2) , ('.REST', 2), + ('.RST', 2), ('.REST', 2), ('.txt', 3), ('.TXT', 3) ] @@ -138,7 +138,6 @@ def convert_line_endings(line, mode): line = replace(line, '\r\n', '\r') line = replace(line, '\n', '\r') elif mode == 2: - import re line = re.sub("\r(?!\n)|(?' % (self.__class__.__name__, self.path) + + def __repr__(self): + return self.__str__() + + def __len__(self): + return self.count() + + @LazyProperty + def alias(self): + for k, v in settings.BACKENDS.items(): + if v.split('.')[-1] == str(self.__class__.__name__): + return k + + @LazyProperty + def name(self): + raise NotImplementedError + + @LazyProperty + def owner(self): + raise NotImplementedError + + @LazyProperty + def description(self): + raise NotImplementedError + + @LazyProperty + def size(self): + """ + Returns combined size in bytes for all repository files + """ + + size = 0 + try: + tip = self.get_changeset() + for topnode, dirs, files in tip.walk('/'): + for f in files: + size += tip.get_file_size(f.path) + for dir in dirs: + for f in files: + size += tip.get_file_size(f.path) + + except RepositoryError, e: + pass + return size + + def is_valid(self): + """ + Validates repository. + """ + raise NotImplementedError + + def get_last_change(self): + self.get_changesets() + + #========================================================================== + # CHANGESETS + #========================================================================== + + def get_changeset(self, revision=None): + """ + Returns instance of ``Changeset`` class. If ``revision`` is None, most + recent changeset is returned. + + :raises ``EmptyRepositoryError``: if there are no revisions + """ + raise NotImplementedError + + def __iter__(self): + """ + Allows Repository objects to be iterated. + + *Requires* implementation of ``__getitem__`` method. + """ + for revision in self.revisions: + yield self.get_changeset(revision) + + def get_changesets(self, start=None, end=None, start_date=None, + end_date=None, branch_name=None, reverse=False): + """ + Returns iterator of ``MercurialChangeset`` objects from start to end + not inclusive This should behave just like a list, ie. end is not + inclusive + + :param start: None or str + :param end: None or str + :param start_date: + :param end_date: + :param branch_name: + :param reversed: + """ + raise NotImplementedError + + def __getslice__(self, i, j): + """ + Returns a iterator of sliced repository + """ + for rev in self.revisions[i:j]: + yield self.get_changeset(rev) + + def __getitem__(self, key): + return self.get_changeset(key) + + def count(self): + return len(self.revisions) + + def tag(self, name, user, revision=None, message=None, date=None, **opts): + """ + Creates and returns a tag for the given ``revision``. + + :param name: name for new tag + :param user: full username, i.e.: "Joe Doe " + :param revision: changeset id for which new tag would be created + :param message: message of the tag's commit + :param date: date of tag's commit + + :raises TagAlreadyExistError: if tag with same name already exists + """ + raise NotImplementedError + + def remove_tag(self, name, user, message=None, date=None): + """ + Removes tag with the given ``name``. + + :param name: name of the tag to be removed + :param user: full username, i.e.: "Joe Doe " + :param message: message of the tag's removal commit + :param date: date of tag's removal commit + + :raises TagDoesNotExistError: if tag with given name does not exists + """ + raise NotImplementedError + + def get_diff(self, rev1, rev2, path=None, ignore_whitespace=False, + context=3): + """ + Returns (git like) *diff*, as plain text. Shows changes introduced by + ``rev2`` since ``rev1``. + + :param rev1: Entry point from which diff is shown. Can be + ``self.EMPTY_CHANGESET`` - in this case, patch showing all + the changes since empty state of the repository until ``rev2`` + :param rev2: Until which revision changes should be shown. + :param ignore_whitespace: If set to ``True``, would not show whitespace + changes. Defaults to ``False``. + :param context: How many lines before/after changed lines should be + shown. Defaults to ``3``. + """ + raise NotImplementedError + + # ========== # + # COMMIT API # + # ========== # + + @LazyProperty + def in_memory_changeset(self): + """ + Returns ``InMemoryChangeset`` object for this repository. + """ + raise NotImplementedError + + def add(self, filenode, **kwargs): + """ + Commit api function that will add given ``FileNode`` into this + repository. + + :raises ``NodeAlreadyExistsError``: if there is a file with same path + already in repository + :raises ``NodeAlreadyAddedError``: if given node is already marked as + *added* + """ + raise NotImplementedError + + def remove(self, filenode, **kwargs): + """ + Commit api function that will remove given ``FileNode`` into this + repository. + + :raises ``EmptyRepositoryError``: if there are no changesets yet + :raises ``NodeDoesNotExistError``: if there is no file with given path + """ + raise NotImplementedError + + def commit(self, message, **kwargs): + """ + Persists current changes made on this repository and returns newly + created changeset. + + :raises ``NothingChangedError``: if no changes has been made + """ + raise NotImplementedError + + def get_state(self): + """ + Returns dictionary with ``added``, ``changed`` and ``removed`` lists + containing ``FileNode`` objects. + """ + raise NotImplementedError + + def get_config_value(self, section, name, config_file=None): + """ + Returns configuration value for a given [``section``] and ``name``. + + :param section: Section we want to retrieve value from + :param name: Name of configuration we want to retrieve + :param config_file: A path to file which should be used to retrieve + configuration from (might also be a list of file paths) + """ + raise NotImplementedError + + def get_user_name(self, config_file=None): + """ + Returns user's name from global configuration file. + + :param config_file: A path to file which should be used to retrieve + configuration from (might also be a list of file paths) + """ + raise NotImplementedError + + def get_user_email(self, config_file=None): + """ + Returns user's email from global configuration file. + + :param config_file: A path to file which should be used to retrieve + configuration from (might also be a list of file paths) + """ + raise NotImplementedError + + # =========== # + # WORKDIR API # + # =========== # + + @LazyProperty + def workdir(self): + """ + Returns ``Workdir`` instance for this repository. + """ + raise NotImplementedError + + +class BaseChangeset(object): + """ + Each backend should implement it's changeset representation. + + **Attributes** + + ``repository`` + repository object within which changeset exists + + ``id`` + may be ``raw_id`` or i.e. for mercurial's tip just ``tip`` + + ``raw_id`` + raw changeset representation (i.e. full 40 length sha for git + backend) + + ``short_id`` + shortened (if apply) version of ``raw_id``; it would be simple + shortcut for ``raw_id[:12]`` for git/mercurial backends or same + as ``raw_id`` for subversion + + ``revision`` + revision number as integer + + ``files`` + list of ``FileNode`` (``Node`` with NodeKind.FILE) objects + + ``dirs`` + list of ``DirNode`` (``Node`` with NodeKind.DIR) objects + + ``nodes`` + combined list of ``Node`` objects + + ``author`` + author of the changeset, as unicode + + ``message`` + message of the changeset, as unicode + + ``parents`` + list of parent changesets + + ``last`` + ``True`` if this is last changeset in repository, ``False`` + otherwise; trying to access this attribute while there is no + changesets would raise ``EmptyRepositoryError`` + """ + def __str__(self): + return '<%s at %s:%s>' % (self.__class__.__name__, self.revision, + self.short_id) + + def __repr__(self): + return self.__str__() + + def __unicode__(self): + return u'%s:%s' % (self.revision, self.short_id) + + def __eq__(self, other): + return self.raw_id == other.raw_id + + @LazyProperty + def last(self): + if self.repository is None: + raise ChangesetError("Cannot check if it's most recent revision") + return self.raw_id == self.repository.revisions[-1] + + @LazyProperty + def parents(self): + """ + Returns list of parents changesets. + """ + raise NotImplementedError + + @LazyProperty + def id(self): + """ + Returns string identifying this changeset. + """ + raise NotImplementedError + + @LazyProperty + def raw_id(self): + """ + Returns raw string identifying this changeset. + """ + raise NotImplementedError + + @LazyProperty + def short_id(self): + """ + Returns shortened version of ``raw_id`` attribute, as string, + identifying this changeset, useful for web representation. + """ + raise NotImplementedError + + @LazyProperty + def revision(self): + """ + Returns integer identifying this changeset. + + """ + raise NotImplementedError + + @LazyProperty + def author(self): + """ + Returns Author for given commit + """ + + raise NotImplementedError + + @LazyProperty + def author_name(self): + """ + Returns Author name for given commit + """ + + return author_name(self.author) + + @LazyProperty + def author_email(self): + """ + Returns Author email address for given commit + """ + + return author_email(self.author) + + def get_file_mode(self, path): + """ + Returns stat mode of the file at the given ``path``. + """ + raise NotImplementedError + + def get_file_content(self, path): + """ + Returns content of the file at the given ``path``. + """ + raise NotImplementedError + + def get_file_size(self, path): + """ + Returns size of the file at the given ``path``. + """ + raise NotImplementedError + + def get_file_changeset(self, path): + """ + Returns last commit of the file at the given ``path``. + """ + raise NotImplementedError + + def get_file_history(self, path): + """ + Returns history of file as reversed list of ``Changeset`` objects for + which file at given ``path`` has been modified. + """ + raise NotImplementedError + + def get_nodes(self, path): + """ + Returns combined ``DirNode`` and ``FileNode`` objects list representing + state of changeset at the given ``path``. + + :raises ``ChangesetError``: if node at the given ``path`` is not + instance of ``DirNode`` + """ + raise NotImplementedError + + def get_node(self, path): + """ + Returns ``Node`` object from the given ``path``. + + :raises ``NodeDoesNotExistError``: if there is no node at the given + ``path`` + """ + raise NotImplementedError + + def fill_archive(self, stream=None, kind='tgz', prefix=None): + """ + Fills up given stream. + + :param stream: file like object. + :param kind: one of following: ``zip``, ``tar``, ``tgz`` + or ``tbz2``. Default: ``tgz``. + :param prefix: name of root directory in archive. + Default is repository name and changeset's raw_id joined with dash. + + repo-tip. + """ + + raise NotImplementedError + + def get_chunked_archive(self, **kwargs): + """ + Returns iterable archive. Tiny wrapper around ``fill_archive`` method. + + :param chunk_size: extra parameter which controls size of returned + chunks. Default:8k. + """ + + chunk_size = kwargs.pop('chunk_size', 8192) + stream = kwargs.get('stream') + self.fill_archive(**kwargs) + while True: + data = stream.read(chunk_size) + if not data: + break + yield data + + @LazyProperty + def root(self): + """ + Returns ``RootNode`` object for this changeset. + """ + return self.get_node('') + + def next(self, branch=None): + """ + Returns next changeset from current, if branch is gives it will return + next changeset belonging to this branch + + :param branch: show changesets within the given named branch + """ + raise NotImplementedError + + def prev(self, branch=None): + """ + Returns previous changeset from current, if branch is gives it will + return previous changeset belonging to this branch + + :param branch: show changesets within the given named branch + """ + raise NotImplementedError + + @LazyProperty + def added(self): + """ + Returns list of added ``FileNode`` objects. + """ + raise NotImplementedError + + @LazyProperty + def changed(self): + """ + Returns list of modified ``FileNode`` objects. + """ + raise NotImplementedError + + @LazyProperty + def removed(self): + """ + Returns list of removed ``FileNode`` objects. + """ + raise NotImplementedError + + @LazyProperty + def size(self): + """ + Returns total number of bytes from contents of all filenodes. + """ + return sum((node.size for node in self.get_filenodes_generator())) + + def walk(self, topurl=''): + """ + Similar to os.walk method. Insted of filesystem it walks through + changeset starting at given ``topurl``. Returns generator of tuples + (topnode, dirnodes, filenodes). + """ + topnode = self.get_node(topurl) + yield (topnode, topnode.dirs, topnode.files) + for dirnode in topnode.dirs: + for tup in self.walk(dirnode.path): + yield tup + + def get_filenodes_generator(self): + """ + Returns generator that yields *all* file nodes. + """ + for topnode, dirs, files in self.walk(): + for node in files: + yield node + + def as_dict(self): + """ + Returns dictionary with changeset's attributes and their values. + """ + data = get_dict_for_attrs(self, ['id', 'raw_id', 'short_id', + 'revision', 'date', 'message']) + data['author'] = {'name': self.author_name, 'email': self.author_email} + data['added'] = [node.path for node in self.added] + data['changed'] = [node.path for node in self.changed] + data['removed'] = [node.path for node in self.removed] + return data + + +class BaseWorkdir(object): + """ + Working directory representation of single repository. + + :attribute: repository: repository object of working directory + """ + + def __init__(self, repository): + self.repository = repository + + def get_branch(self): + """ + Returns name of current branch. + """ + raise NotImplementedError + + def get_changeset(self): + """ + Returns current changeset. + """ + raise NotImplementedError + + def get_added(self): + """ + Returns list of ``FileNode`` objects marked as *new* in working + directory. + """ + raise NotImplementedError + + def get_changed(self): + """ + Returns list of ``FileNode`` objects *changed* in working directory. + """ + raise NotImplementedError + + def get_removed(self): + """ + Returns list of ``RemovedFileNode`` objects marked as *removed* in + working directory. + """ + raise NotImplementedError + + def get_untracked(self): + """ + Returns list of ``FileNode`` objects which are present within working + directory however are not tracked by repository. + """ + raise NotImplementedError + + def get_status(self): + """ + Returns dict with ``added``, ``changed``, ``removed`` and ``untracked`` + lists. + """ + raise NotImplementedError + + def commit(self, message, **kwargs): + """ + Commits local (from working directory) changes and returns newly + created + ``Changeset``. Updates repository's ``revisions`` list. + + :raises ``CommitError``: if any error occurs while committing + """ + raise NotImplementedError + + def update(self, revision=None): + """ + Fetches content of the given revision and populates it within working + directory. + """ + raise NotImplementedError + + def checkout_branch(self, branch=None): + """ + Checks out ``branch`` or the backend's default branch. + + Raises ``BranchDoesNotExistError`` if the branch does not exist. + """ + raise NotImplementedError + + +class BaseInMemoryChangeset(object): + """ + Represents differences between repository's state (most recent head) and + changes made *in place*. + + **Attributes** + + ``repository`` + repository object for this in-memory-changeset + + ``added`` + list of ``FileNode`` objects marked as *added* + + ``changed`` + list of ``FileNode`` objects marked as *changed* + + ``removed`` + list of ``FileNode`` or ``RemovedFileNode`` objects marked to be + *removed* + + ``parents`` + list of ``Changeset`` representing parents of in-memory changeset. + Should always be 2-element sequence. + + """ + + def __init__(self, repository): + self.repository = repository + self.added = [] + self.changed = [] + self.removed = [] + self.parents = [] + + def add(self, *filenodes): + """ + Marks given ``FileNode`` objects as *to be committed*. + + :raises ``NodeAlreadyExistsError``: if node with same path exists at + latest changeset + :raises ``NodeAlreadyAddedError``: if node with same path is already + marked as *added* + """ + # Check if not already marked as *added* first + for node in filenodes: + if node.path in (n.path for n in self.added): + raise NodeAlreadyAddedError("Such FileNode %s is already " + "marked for addition" % node.path) + for node in filenodes: + self.added.append(node) + + def change(self, *filenodes): + """ + Marks given ``FileNode`` objects to be *changed* in next commit. + + :raises ``EmptyRepositoryError``: if there are no changesets yet + :raises ``NodeAlreadyExistsError``: if node with same path is already + marked to be *changed* + :raises ``NodeAlreadyRemovedError``: if node with same path is already + marked to be *removed* + :raises ``NodeDoesNotExistError``: if node doesn't exist in latest + changeset + :raises ``NodeNotChangedError``: if node hasn't really be changed + """ + for node in filenodes: + if node.path in (n.path for n in self.removed): + raise NodeAlreadyRemovedError("Node at %s is already marked " + "as removed" % node.path) + try: + self.repository.get_changeset() + except EmptyRepositoryError: + raise EmptyRepositoryError("Nothing to change - try to *add* new " + "nodes rather than changing them") + for node in filenodes: + if node.path in (n.path for n in self.changed): + raise NodeAlreadyChangedError("Node at '%s' is already " + "marked as changed" % node.path) + self.changed.append(node) + + def remove(self, *filenodes): + """ + Marks given ``FileNode`` (or ``RemovedFileNode``) objects to be + *removed* in next commit. + + :raises ``NodeAlreadyRemovedError``: if node has been already marked to + be *removed* + :raises ``NodeAlreadyChangedError``: if node has been already marked to + be *changed* + """ + for node in filenodes: + if node.path in (n.path for n in self.removed): + raise NodeAlreadyRemovedError("Node is already marked to " + "for removal at %s" % node.path) + if node.path in (n.path for n in self.changed): + raise NodeAlreadyChangedError("Node is already marked to " + "be changed at %s" % node.path) + # We only mark node as *removed* - real removal is done by + # commit method + self.removed.append(node) + + def reset(self): + """ + Resets this instance to initial state (cleans ``added``, ``changed`` + and ``removed`` lists). + """ + self.added = [] + self.changed = [] + self.removed = [] + self.parents = [] + + def get_ipaths(self): + """ + Returns generator of paths from nodes marked as added, changed or + removed. + """ + for node in chain(self.added, self.changed, self.removed): + yield node.path + + def get_paths(self): + """ + Returns list of paths from nodes marked as added, changed or removed. + """ + return list(self.get_ipaths()) + + def check_integrity(self, parents=None): + """ + Checks in-memory changeset's integrity. Also, sets parents if not + already set. + + :raises CommitError: if any error occurs (i.e. + ``NodeDoesNotExistError``). + """ + if not self.parents: + parents = parents or [] + if len(parents) == 0: + try: + parents = [self.repository.get_changeset(), None] + except EmptyRepositoryError: + parents = [None, None] + elif len(parents) == 1: + parents += [None] + self.parents = parents + + # Local parents, only if not None + parents = [p for p in self.parents if p] + + # Check nodes marked as added + for p in parents: + for node in self.added: + try: + p.get_node(node.path) + except NodeDoesNotExistError: + pass + else: + raise NodeAlreadyExistsError("Node at %s already exists " + "at %s" % (node.path, p)) + + # Check nodes marked as changed + missing = set(self.changed) + not_changed = set(self.changed) + if self.changed and not parents: + raise NodeDoesNotExistError(str(self.changed[0].path)) + for p in parents: + for node in self.changed: + try: + old = p.get_node(node.path) + missing.remove(node) + if old.content != node.content: + not_changed.remove(node) + except NodeDoesNotExistError: + pass + if self.changed and missing: + raise NodeDoesNotExistError("Node at %s is missing " + "(parents: %s)" % (node.path, parents)) + + if self.changed and not_changed: + raise NodeNotChangedError("Node at %s wasn't actually changed " + "since parents' changesets: %s" % (not_changed.pop().path, + parents) + ) + + # Check nodes marked as removed + if self.removed and not parents: + raise NodeDoesNotExistError("Cannot remove node at %s as there " + "were no parents specified" % self.removed[0].path) + really_removed = set() + for p in parents: + for node in self.removed: + try: + p.get_node(node.path) + really_removed.add(node) + except ChangesetError: + pass + not_removed = set(self.removed) - really_removed + if not_removed: + raise NodeDoesNotExistError("Cannot remove node at %s from " + "following parents: %s" % (not_removed[0], parents)) + + def commit(self, message, author, parents=None, branch=None, date=None, + **kwargs): + """ + Performs in-memory commit (doesn't check workdir in any way) and + returns newly created ``Changeset``. Updates repository's + ``revisions``. + + .. note:: + While overriding this method each backend's should call + ``self.check_integrity(parents)`` in the first place. + + :param message: message of the commit + :param author: full username, i.e. "Joe Doe " + :param parents: single parent or sequence of parents from which commit + would be derieved + :param date: ``datetime.datetime`` instance. Defaults to + ``datetime.datetime.now()``. + :param branch: branch name, as string. If none given, default backend's + branch would be used. + + :raises ``CommitError``: if any error occurs while committing + """ + raise NotImplementedError diff --git a/rhodecode/lib/vcs/backends/git/__init__.py b/rhodecode/lib/vcs/backends/git/__init__.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/backends/git/__init__.py @@ -0,0 +1,9 @@ +from .repository import GitRepository +from .changeset import GitChangeset +from .inmemory import GitInMemoryChangeset +from .workdir import GitWorkdir + + +__all__ = [ + 'GitRepository', 'GitChangeset', 'GitInMemoryChangeset', 'GitWorkdir', +] diff --git a/rhodecode/lib/vcs/backends/git/changeset.py b/rhodecode/lib/vcs/backends/git/changeset.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/backends/git/changeset.py @@ -0,0 +1,450 @@ +import re +from itertools import chain +from dulwich import objects +from subprocess import Popen, PIPE +from rhodecode.lib.vcs.conf import settings +from rhodecode.lib.vcs.exceptions import RepositoryError +from rhodecode.lib.vcs.exceptions import ChangesetError +from rhodecode.lib.vcs.exceptions import NodeDoesNotExistError +from rhodecode.lib.vcs.exceptions import VCSError +from rhodecode.lib.vcs.exceptions import ChangesetDoesNotExistError +from rhodecode.lib.vcs.exceptions import ImproperArchiveTypeError +from rhodecode.lib.vcs.backends.base import BaseChangeset +from rhodecode.lib.vcs.nodes import FileNode, DirNode, NodeKind, RootNode, RemovedFileNode +from rhodecode.lib.vcs.utils import safe_unicode +from rhodecode.lib.vcs.utils import date_fromtimestamp +from rhodecode.lib.vcs.utils.lazy import LazyProperty + + +class GitChangeset(BaseChangeset): + """ + Represents state of the repository at single revision. + """ + + def __init__(self, repository, revision): + self._stat_modes = {} + self.repository = repository + self.raw_id = revision + self.revision = repository.revisions.index(revision) + + self.short_id = self.raw_id[:12] + self.id = self.raw_id + try: + commit = self.repository._repo.get_object(self.raw_id) + except KeyError: + raise RepositoryError("Cannot get object with id %s" % self.raw_id) + self._commit = commit + self._tree_id = commit.tree + + try: + self.message = safe_unicode(commit.message[:-1]) + # Always strip last eol + except UnicodeDecodeError: + self.message = commit.message[:-1].decode(commit.encoding + or 'utf-8') + #self.branch = None + self.tags = [] + #tree = self.repository.get_object(self._tree_id) + self.nodes = {} + self._paths = {} + + @LazyProperty + def author(self): + return safe_unicode(self._commit.committer) + + @LazyProperty + def date(self): + return date_fromtimestamp(self._commit.commit_time, + self._commit.commit_timezone) + + @LazyProperty + def status(self): + """ + Returns modified, added, removed, deleted files for current changeset + """ + return self.changed, self.added, self.removed + + @LazyProperty + def branch(self): + # TODO: Cache as we walk (id <-> branch name mapping) + refs = self.repository._repo.get_refs() + heads = [(key[len('refs/heads/'):], val) for key, val in refs.items() + if key.startswith('refs/heads/')] + + for name, id in heads: + walker = self.repository._repo.object_store.get_graph_walker([id]) + while True: + id = walker.next() + if not id: + break + if id == self.id: + return safe_unicode(name) + raise ChangesetError("This should not happen... Have you manually " + "change id of the changeset?") + + def _fix_path(self, path): + """ + Paths are stored without trailing slash so we need to get rid off it if + needed. + """ + if path.endswith('/'): + path = path.rstrip('/') + return path + + def _get_id_for_path(self, path): + # FIXME: Please, spare a couple of minutes and make those codes cleaner; + if not path in self._paths: + path = path.strip('/') + # set root tree + tree = self.repository._repo[self._commit.tree] + if path == '': + self._paths[''] = tree.id + return tree.id + splitted = path.split('/') + dirs, name = splitted[:-1], splitted[-1] + curdir = '' + for dir in dirs: + if curdir: + curdir = '/'.join((curdir, dir)) + else: + curdir = dir + #if curdir in self._paths: + ## This path have been already traversed + ## Update tree and continue + #tree = self.repository._repo[self._paths[curdir]] + #continue + dir_id = None + for item, stat, id in tree.iteritems(): + if curdir: + item_path = '/'.join((curdir, item)) + else: + item_path = item + self._paths[item_path] = id + self._stat_modes[item_path] = stat + if dir == item: + dir_id = id + if dir_id: + # Update tree + tree = self.repository._repo[dir_id] + if not isinstance(tree, objects.Tree): + raise ChangesetError('%s is not a directory' % curdir) + else: + raise ChangesetError('%s have not been found' % curdir) + for item, stat, id in tree.iteritems(): + if curdir: + name = '/'.join((curdir, item)) + else: + name = item + self._paths[name] = id + self._stat_modes[name] = stat + if not path in self._paths: + raise NodeDoesNotExistError("There is no file nor directory " + "at the given path %r at revision %r" + % (path, self.short_id)) + return self._paths[path] + + def _get_kind(self, path): + id = self._get_id_for_path(path) + obj = self.repository._repo[id] + if isinstance(obj, objects.Blob): + return NodeKind.FILE + elif isinstance(obj, objects.Tree): + return NodeKind.DIR + + def _get_file_nodes(self): + return chain(*(t[2] for t in self.walk())) + + @LazyProperty + def parents(self): + """ + Returns list of parents changesets. + """ + return [self.repository.get_changeset(parent) + for parent in self._commit.parents] + + def next(self, branch=None): + + if branch and self.branch != branch: + raise VCSError('Branch option used on changeset not belonging ' + 'to that branch') + + def _next(changeset, branch): + try: + next_ = changeset.revision + 1 + next_rev = changeset.repository.revisions[next_] + except IndexError: + raise ChangesetDoesNotExistError + cs = changeset.repository.get_changeset(next_rev) + + if branch and branch != cs.branch: + return _next(cs, branch) + + return cs + + return _next(self, branch) + + def prev(self, branch=None): + if branch and self.branch != branch: + raise VCSError('Branch option used on changeset not belonging ' + 'to that branch') + + def _prev(changeset, branch): + try: + prev_ = changeset.revision - 1 + if prev_ < 0: + raise IndexError + prev_rev = changeset.repository.revisions[prev_] + except IndexError: + raise ChangesetDoesNotExistError + + cs = changeset.repository.get_changeset(prev_rev) + + if branch and branch != cs.branch: + return _prev(cs, branch) + + return cs + + return _prev(self, branch) + + def get_file_mode(self, path): + """ + Returns stat mode of the file at the given ``path``. + """ + # ensure path is traversed + self._get_id_for_path(path) + return self._stat_modes[path] + + def get_file_content(self, path): + """ + Returns content of the file at given ``path``. + """ + id = self._get_id_for_path(path) + blob = self.repository._repo[id] + return blob.as_pretty_string() + + def get_file_size(self, path): + """ + Returns size of the file at given ``path``. + """ + id = self._get_id_for_path(path) + blob = self.repository._repo[id] + return blob.raw_length() + + def get_file_changeset(self, path): + """ + Returns last commit of the file at the given ``path``. + """ + node = self.get_node(path) + return node.history[0] + + def get_file_history(self, path): + """ + Returns history of file as reversed list of ``Changeset`` objects for + which file at given ``path`` has been modified. + + TODO: This function now uses os underlying 'git' and 'grep' commands + which is generally not good. Should be replaced with algorithm + iterating commits. + """ + cmd = 'log --name-status -p %s -- "%s" | grep "^commit"' \ + % (self.id, path) + so, se = self.repository.run_git_command(cmd) + ids = re.findall(r'\w{40}', so) + return [self.repository.get_changeset(id) for id in ids] + + def get_file_annotate(self, path): + """ + Returns a list of three element tuples with lineno,changeset and line + + TODO: This function now uses os underlying 'git' command which is + generally not good. Should be replaced with algorithm iterating + commits. + """ + cmd = 'blame -l --root -r %s -- "%s"' % (self.id, path) + # -l ==> outputs long shas (and we need all 40 characters) + # --root ==> doesn't put '^' character for bounderies + # -r sha ==> blames for the given revision + so, se = self.repository.run_git_command(cmd) + annotate = [] + for i, blame_line in enumerate(so.split('\n')[:-1]): + ln_no = i + 1 + id, line = re.split(r' \(.+?\) ', blame_line, 1) + annotate.append((ln_no, self.repository.get_changeset(id), line)) + return annotate + + def fill_archive(self, stream=None, kind='tgz', prefix=None, + subrepos=False): + """ + Fills up given stream. + + :param stream: file like object. + :param kind: one of following: ``zip``, ``tgz`` or ``tbz2``. + Default: ``tgz``. + :param prefix: name of root directory in archive. + Default is repository name and changeset's raw_id joined with dash + (``repo-tip.``). + :param subrepos: include subrepos in this archive. + + :raise ImproperArchiveTypeError: If given kind is wrong. + :raise VcsError: If given stream is None + + """ + allowed_kinds = settings.ARCHIVE_SPECS.keys() + if kind not in allowed_kinds: + raise ImproperArchiveTypeError('Archive kind not supported use one' + 'of %s', allowed_kinds) + + if prefix is None: + prefix = '%s-%s' % (self.repository.name, self.short_id) + elif prefix.startswith('/'): + raise VCSError("Prefix cannot start with leading slash") + elif prefix.strip() == '': + raise VCSError("Prefix cannot be empty") + + if kind == 'zip': + frmt = 'zip' + else: + frmt = 'tar' + cmd = 'git archive --format=%s --prefix=%s/ %s' % (frmt, prefix, + self.raw_id) + if kind == 'tgz': + cmd += ' | gzip -9' + elif kind == 'tbz2': + cmd += ' | bzip2 -9' + + if stream is None: + raise VCSError('You need to pass in a valid stream for filling' + ' with archival data') + popen = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, + cwd=self.repository.path) + + buffer_size = 1024 * 8 + chunk = popen.stdout.read(buffer_size) + while chunk: + stream.write(chunk) + chunk = popen.stdout.read(buffer_size) + # Make sure all descriptors would be read + popen.communicate() + + def get_nodes(self, path): + if self._get_kind(path) != NodeKind.DIR: + raise ChangesetError("Directory does not exist for revision %r at " + " %r" % (self.revision, path)) + path = self._fix_path(path) + id = self._get_id_for_path(path) + tree = self.repository._repo[id] + dirnodes = [] + filenodes = [] + for name, stat, id in tree.iteritems(): + obj = self.repository._repo.get_object(id) + if path != '': + obj_path = '/'.join((path, name)) + else: + obj_path = name + if obj_path not in self._stat_modes: + self._stat_modes[obj_path] = stat + if isinstance(obj, objects.Tree): + dirnodes.append(DirNode(obj_path, changeset=self)) + elif isinstance(obj, objects.Blob): + filenodes.append(FileNode(obj_path, changeset=self, mode=stat)) + else: + raise ChangesetError("Requested object should be Tree " + "or Blob, is %r" % type(obj)) + nodes = dirnodes + filenodes + for node in nodes: + if not node.path in self.nodes: + self.nodes[node.path] = node + nodes.sort() + return nodes + + def get_node(self, path): + if isinstance(path, unicode): + path = path.encode('utf-8') + path = self._fix_path(path) + if not path in self.nodes: + try: + id = self._get_id_for_path(path) + except ChangesetError: + raise NodeDoesNotExistError("Cannot find one of parents' " + "directories for a given path: %s" % path) + obj = self.repository._repo.get_object(id) + if isinstance(obj, objects.Tree): + if path == '': + node = RootNode(changeset=self) + else: + node = DirNode(path, changeset=self) + node._tree = obj + elif isinstance(obj, objects.Blob): + node = FileNode(path, changeset=self) + node._blob = obj + else: + raise NodeDoesNotExistError("There is no file nor directory " + "at the given path %r at revision %r" + % (path, self.short_id)) + # cache node + self.nodes[path] = node + return self.nodes[path] + + @LazyProperty + def affected_files(self): + """ + Get's a fast accessible file changes for given changeset + """ + + return self.added + self.changed + + @LazyProperty + def _diff_name_status(self): + output = [] + for parent in self.parents: + cmd = 'diff --name-status %s %s' % (parent.raw_id, self.raw_id) + so, se = self.repository.run_git_command(cmd) + output.append(so.strip()) + return '\n'.join(output) + + def _get_paths_for_status(self, status): + """ + Returns sorted list of paths for given ``status``. + + :param status: one of: *added*, *modified* or *deleted* + """ + paths = set() + char = status[0].upper() + for line in self._diff_name_status.splitlines(): + if not line: + continue + if line.startswith(char): + splitted = line.split(char,1) + if not len(splitted) == 2: + raise VCSError("Couldn't parse diff result:\n%s\n\n and " + "particularly that line: %s" % (self._diff_name_status, + line)) + paths.add(splitted[1].strip()) + return sorted(paths) + + @LazyProperty + def added(self): + """ + Returns list of added ``FileNode`` objects. + """ + if not self.parents: + return list(self._get_file_nodes()) + return [self.get_node(path) for path in self._get_paths_for_status('added')] + + @LazyProperty + def changed(self): + """ + Returns list of modified ``FileNode`` objects. + """ + if not self.parents: + return [] + return [self.get_node(path) for path in self._get_paths_for_status('modified')] + + @LazyProperty + def removed(self): + """ + Returns list of removed ``FileNode`` objects. + """ + if not self.parents: + return [] + return [RemovedFileNode(path) for path in self._get_paths_for_status('deleted')] diff --git a/rhodecode/lib/vcs/backends/git/config.py b/rhodecode/lib/vcs/backends/git/config.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/backends/git/config.py @@ -0,0 +1,347 @@ +# config.py - Reading and writing Git config files +# Copyright (C) 2011 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; version 2 +# of the License or (at your option) a later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, +# MA 02110-1301, USA. + +"""Reading and writing Git configuration files. + +TODO: + * preserve formatting when updating configuration files + * treat subsection names as case-insensitive for [branch.foo] style + subsections +""" + +# Taken from dulwich not yet released 0.8.3 version (until it is actually +# released) + +import errno +import os +import re + +from dulwich.file import GitFile + + +class Config(object): + """A Git configuration.""" + + def get(self, section, name): + """Retrieve the contents of a configuration setting. + + :param section: Tuple with section name and optional subsection namee + :param subsection: Subsection name + :return: Contents of the setting + :raise KeyError: if the value is not set + """ + raise NotImplementedError(self.get) + + def get_boolean(self, section, name, default=None): + """Retrieve a configuration setting as boolean. + + :param section: Tuple with section name and optional subsection namee + :param name: Name of the setting, including section and possible + subsection. + :return: Contents of the setting + :raise KeyError: if the value is not set + """ + try: + value = self.get(section, name) + except KeyError: + return default + if value.lower() == "true": + return True + elif value.lower() == "false": + return False + raise ValueError("not a valid boolean string: %r" % value) + + def set(self, section, name, value): + """Set a configuration value. + + :param name: Name of the configuration value, including section + and optional subsection + :param: Value of the setting + """ + raise NotImplementedError(self.set) + + +class ConfigDict(Config): + """Git configuration stored in a dictionary.""" + + def __init__(self, values=None): + """Create a new ConfigDict.""" + if values is None: + values = {} + self._values = values + + def __repr__(self): + return "%s(%r)" % (self.__class__.__name__, self._values) + + def __eq__(self, other): + return ( + isinstance(other, self.__class__) and + other._values == self._values) + + @classmethod + def _parse_setting(cls, name): + parts = name.split(".") + if len(parts) == 3: + return (parts[0], parts[1], parts[2]) + else: + return (parts[0], None, parts[1]) + + def get(self, section, name): + if isinstance(section, basestring): + section = (section, ) + if len(section) > 1: + try: + return self._values[section][name] + except KeyError: + pass + return self._values[(section[0],)][name] + + def set(self, section, name, value): + if isinstance(section, basestring): + section = (section, ) + self._values.setdefault(section, {})[name] = value + + +def _format_string(value): + if (value.startswith(" ") or + value.startswith("\t") or + value.endswith(" ") or + value.endswith("\t")): + return '"%s"' % _escape_value(value) + return _escape_value(value) + + +def _parse_string(value): + value = value.strip() + ret = [] + block = [] + in_quotes = False + for c in value: + if c == "\"": + in_quotes = (not in_quotes) + ret.append(_unescape_value("".join(block))) + block = [] + elif c in ("#", ";") and not in_quotes: + # the rest of the line is a comment + break + else: + block.append(c) + + if in_quotes: + raise ValueError("value starts with quote but lacks end quote") + + ret.append(_unescape_value("".join(block)).rstrip()) + + return "".join(ret) + + +def _unescape_value(value): + """Unescape a value.""" + def unescape(c): + return { + "\\\\": "\\", + "\\\"": "\"", + "\\n": "\n", + "\\t": "\t", + "\\b": "\b", + }[c.group(0)] + return re.sub(r"(\\.)", unescape, value) + + +def _escape_value(value): + """Escape a value.""" + return value.replace("\\", "\\\\").replace("\n", "\\n")\ + .replace("\t", "\\t").replace("\"", "\\\"") + + +def _check_variable_name(name): + for c in name: + if not c.isalnum() and c != '-': + return False + return True + + +def _check_section_name(name): + for c in name: + if not c.isalnum() and c not in ('-', '.'): + return False + return True + + +def _strip_comments(line): + line = line.split("#")[0] + line = line.split(";")[0] + return line + + +class ConfigFile(ConfigDict): + """A Git configuration file, like .git/config or ~/.gitconfig. + """ + + @classmethod + def from_file(cls, f): + """Read configuration from a file-like object.""" + ret = cls() + section = None + setting = None + for lineno, line in enumerate(f.readlines()): + line = line.lstrip() + if setting is None: + if _strip_comments(line).strip() == "": + continue + if line[0] == "[": + line = _strip_comments(line).rstrip() + if line[-1] != "]": + raise ValueError("expected trailing ]") + key = line.strip() + pts = key[1:-1].split(" ", 1) + pts[0] = pts[0].lower() + if len(pts) == 2: + if pts[1][0] != "\"" or pts[1][-1] != "\"": + raise ValueError( + "Invalid subsection " + pts[1]) + else: + pts[1] = pts[1][1:-1] + if not _check_section_name(pts[0]): + raise ValueError("invalid section name %s" % + pts[0]) + section = (pts[0], pts[1]) + else: + if not _check_section_name(pts[0]): + raise ValueError("invalid section name %s" % + pts[0]) + pts = pts[0].split(".", 1) + if len(pts) == 2: + section = (pts[0], pts[1]) + else: + section = (pts[0], ) + ret._values[section] = {} + else: + if section is None: + raise ValueError("setting %r without section" % line) + try: + setting, value = line.split("=", 1) + except ValueError: + setting = line + value = "true" + setting = setting.strip().lower() + if not _check_variable_name(setting): + raise ValueError("invalid variable name %s" % setting) + if value.endswith("\\\n"): + value = value[:-2] + continuation = True + else: + continuation = False + value = _parse_string(value) + ret._values[section][setting] = value + if not continuation: + setting = None + else: # continuation line + if line.endswith("\\\n"): + line = line[:-2] + continuation = True + else: + continuation = False + value = _parse_string(line) + ret._values[section][setting] += value + if not continuation: + setting = None + return ret + + @classmethod + def from_path(cls, path): + """Read configuration from a file on disk.""" + f = GitFile(path, 'rb') + try: + ret = cls.from_file(f) + ret.path = path + return ret + finally: + f.close() + + def write_to_path(self, path=None): + """Write configuration to a file on disk.""" + if path is None: + path = self.path + f = GitFile(path, 'wb') + try: + self.write_to_file(f) + finally: + f.close() + + def write_to_file(self, f): + """Write configuration to a file-like object.""" + for section, values in self._values.iteritems(): + try: + section_name, subsection_name = section + except ValueError: + (section_name, ) = section + subsection_name = None + if subsection_name is None: + f.write("[%s]\n" % section_name) + else: + f.write("[%s \"%s\"]\n" % (section_name, subsection_name)) + for key, value in values.iteritems(): + f.write("%s = %s\n" % (key, _escape_value(value))) + + +class StackedConfig(Config): + """Configuration which reads from multiple config files..""" + + def __init__(self, backends, writable=None): + self.backends = backends + self.writable = writable + + def __repr__(self): + return "<%s for %r>" % (self.__class__.__name__, self.backends) + + @classmethod + def default_backends(cls): + """Retrieve the default configuration. + + This will look in the repository configuration (if for_path is + specified), the users' home directory and the system + configuration. + """ + paths = [] + paths.append(os.path.expanduser("~/.gitconfig")) + paths.append("/etc/gitconfig") + backends = [] + for path in paths: + try: + cf = ConfigFile.from_path(path) + except (IOError, OSError), e: + if e.errno != errno.ENOENT: + raise + else: + continue + backends.append(cf) + return backends + + def get(self, section, name): + for backend in self.backends: + try: + return backend.get(section, name) + except KeyError: + pass + raise KeyError(name) + + def set(self, section, name, value): + if self.writable is None: + raise NotImplementedError(self.set) + return self.writable.set(section, name, value) diff --git a/rhodecode/lib/vcs/backends/git/inmemory.py b/rhodecode/lib/vcs/backends/git/inmemory.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/backends/git/inmemory.py @@ -0,0 +1,192 @@ +import time +import datetime +import posixpath +from dulwich import objects +from dulwich.repo import Repo +from rhodecode.lib.vcs.backends.base import BaseInMemoryChangeset +from rhodecode.lib.vcs.exceptions import RepositoryError + + +class GitInMemoryChangeset(BaseInMemoryChangeset): + + def commit(self, message, author, parents=None, branch=None, date=None, + **kwargs): + """ + Performs in-memory commit (doesn't check workdir in any way) and + returns newly created ``Changeset``. Updates repository's + ``revisions``. + + :param message: message of the commit + :param author: full username, i.e. "Joe Doe " + :param parents: single parent or sequence of parents from which commit + would be derieved + :param date: ``datetime.datetime`` instance. Defaults to + ``datetime.datetime.now()``. + :param branch: branch name, as string. If none given, default backend's + branch would be used. + + :raises ``CommitError``: if any error occurs while committing + """ + self.check_integrity(parents) + + from .repository import GitRepository + if branch is None: + branch = GitRepository.DEFAULT_BRANCH_NAME + + repo = self.repository._repo + object_store = repo.object_store + + ENCODING = "UTF-8" + DIRMOD = 040000 + + # Create tree and populates it with blobs + commit_tree = self.parents[0] and repo[self.parents[0]._commit.tree] or\ + objects.Tree() + for node in self.added + self.changed: + # Compute subdirs if needed + dirpath, nodename = posixpath.split(node.path) + dirnames = dirpath and dirpath.split('/') or [] + parent = commit_tree + ancestors = [('', parent)] + + # Tries to dig for the deepest existing tree + while dirnames: + curdir = dirnames.pop(0) + try: + dir_id = parent[curdir][1] + except KeyError: + # put curdir back into dirnames and stops + dirnames.insert(0, curdir) + break + else: + # If found, updates parent + parent = self.repository._repo[dir_id] + ancestors.append((curdir, parent)) + # Now parent is deepest exising tree and we need to create subtrees + # for dirnames (in reverse order) [this only applies for nodes from added] + new_trees = [] + blob = objects.Blob.from_string(node.content.encode(ENCODING)) + node_path = node.name.encode(ENCODING) + if dirnames: + # If there are trees which should be created we need to build + # them now (in reverse order) + reversed_dirnames = list(reversed(dirnames)) + curtree = objects.Tree() + curtree[node_path] = node.mode, blob.id + new_trees.append(curtree) + for dirname in reversed_dirnames[:-1]: + newtree = objects.Tree() + #newtree.add(DIRMOD, dirname, curtree.id) + newtree[dirname] = DIRMOD, curtree.id + new_trees.append(newtree) + curtree = newtree + parent[reversed_dirnames[-1]] = DIRMOD, curtree.id + else: + parent.add(node.mode, node_path, blob.id) + new_trees.append(parent) + # Update ancestors + for parent, tree, path in reversed([(a[1], b[1], b[0]) for a, b in + zip(ancestors, ancestors[1:])]): + parent[path] = DIRMOD, tree.id + object_store.add_object(tree) + + object_store.add_object(blob) + for tree in new_trees: + object_store.add_object(tree) + for node in self.removed: + paths = node.path.split('/') + tree = commit_tree + trees = [tree] + # Traverse deep into the forest... + for path in paths: + try: + obj = self.repository._repo[tree[path][1]] + if isinstance(obj, objects.Tree): + trees.append(obj) + tree = obj + except KeyError: + break + # Cut down the blob and all rotten trees on the way back... + for path, tree in reversed(zip(paths, trees)): + del tree[path] + if tree: + # This tree still has elements - don't remove it or any + # of it's parents + break + + object_store.add_object(commit_tree) + + # Create commit + commit = objects.Commit() + commit.tree = commit_tree.id + commit.parents = [p._commit.id for p in self.parents if p] + commit.author = commit.committer = author + commit.encoding = ENCODING + commit.message = message + ' ' + + # Compute date + if date is None: + date = time.time() + elif isinstance(date, datetime.datetime): + date = time.mktime(date.timetuple()) + + author_time = kwargs.pop('author_time', date) + commit.commit_time = int(date) + commit.author_time = int(author_time) + tz = time.timezone + author_tz = kwargs.pop('author_timezone', tz) + commit.commit_timezone = tz + commit.author_timezone = author_tz + + object_store.add_object(commit) + + ref = 'refs/heads/%s' % branch + repo.refs[ref] = commit.id + repo.refs.set_symbolic_ref('HEAD', ref) + + # Update vcs repository object & recreate dulwich repo + self.repository.revisions.append(commit.id) + self.repository._repo = Repo(self.repository.path) + tip = self.repository.get_changeset() + self.reset() + return tip + + def _get_missing_trees(self, path, root_tree): + """ + Creates missing ``Tree`` objects for the given path. + + :param path: path given as a string. It may be a path to a file node + (i.e. ``foo/bar/baz.txt``) or directory path - in that case it must + end with slash (i.e. ``foo/bar/``). + :param root_tree: ``dulwich.objects.Tree`` object from which we start + traversing (should be commit's root tree) + """ + dirpath = posixpath.split(path)[0] + dirs = dirpath.split('/') + if not dirs or dirs == ['']: + return [] + + def get_tree_for_dir(tree, dirname): + for name, mode, id in tree.iteritems(): + if name == dirname: + obj = self.repository._repo[id] + if isinstance(obj, objects.Tree): + return obj + else: + raise RepositoryError("Cannot create directory %s " + "at tree %s as path is occupied and is not a " + "Tree" % (dirname, tree)) + return None + + trees = [] + parent = root_tree + for dirname in dirs: + tree = get_tree_for_dir(parent, dirname) + if tree is None: + tree = objects.Tree() + dirmode = 040000 + parent.add(dirmode, dirname, tree.id) + parent = tree + # Always append tree + trees.append(tree) + return trees diff --git a/rhodecode/lib/vcs/backends/git/repository.py b/rhodecode/lib/vcs/backends/git/repository.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/backends/git/repository.py @@ -0,0 +1,508 @@ +# -*- coding: utf-8 -*- +""" + vcs.backends.git + ~~~~~~~~~~~~~~~~ + + Git backend implementation. + + :created_on: Apr 8, 2010 + :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak. +""" + +import os +import re +import time +import posixpath +from dulwich.repo import Repo, NotGitRepository +#from dulwich.config import ConfigFile +from string import Template +from subprocess import Popen, PIPE +from rhodecode.lib.vcs.backends.base import BaseRepository +from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError +from rhodecode.lib.vcs.exceptions import ChangesetDoesNotExistError +from rhodecode.lib.vcs.exceptions import EmptyRepositoryError +from rhodecode.lib.vcs.exceptions import RepositoryError +from rhodecode.lib.vcs.exceptions import TagAlreadyExistError +from rhodecode.lib.vcs.exceptions import TagDoesNotExistError +from rhodecode.lib.vcs.utils import safe_unicode, makedate, date_fromtimestamp +from rhodecode.lib.vcs.utils.lazy import LazyProperty +from rhodecode.lib.vcs.utils.ordered_dict import OrderedDict +from rhodecode.lib.vcs.utils.paths import abspath +from rhodecode.lib.vcs.utils.paths import get_user_home +from .workdir import GitWorkdir +from .changeset import GitChangeset +from .inmemory import GitInMemoryChangeset +from .config import ConfigFile + + +class GitRepository(BaseRepository): + """ + Git repository backend. + """ + DEFAULT_BRANCH_NAME = 'master' + scm = 'git' + + def __init__(self, repo_path, create=False, src_url=None, + update_after_clone=False, bare=False): + + self.path = abspath(repo_path) + self._repo = self._get_repo(create, src_url, update_after_clone, bare) + try: + self.head = self._repo.head() + except KeyError: + self.head = None + + self._config_files = [ + bare and abspath(self.path, 'config') or abspath(self.path, '.git', + 'config'), + abspath(get_user_home(), '.gitconfig'), + ] + + @LazyProperty + def revisions(self): + """ + Returns list of revisions' ids, in ascending order. Being lazy + attribute allows external tools to inject shas from cache. + """ + return self._get_all_revisions() + + def run_git_command(self, cmd): + """ + Runs given ``cmd`` as git command and returns tuple + (returncode, stdout, stderr). + + .. note:: + This method exists only until log/blame functionality is implemented + at Dulwich (see https://bugs.launchpad.net/bugs/645142). Parsing + os command's output is road to hell... + + :param cmd: git command to be executed + """ + #cmd = '(cd %s && git %s)' % (self.path, cmd) + if isinstance(cmd, basestring): + cmd = 'git %s' % cmd + else: + cmd = ['git'] + cmd + try: + opts = dict( + shell=isinstance(cmd, basestring), + stdout=PIPE, + stderr=PIPE) + if os.path.isdir(self.path): + opts['cwd'] = self.path + p = Popen(cmd, **opts) + except OSError, err: + raise RepositoryError("Couldn't run git command (%s).\n" + "Original error was:%s" % (cmd, err)) + so, se = p.communicate() + if not se.startswith("fatal: bad default revision 'HEAD'") and \ + p.returncode != 0: + raise RepositoryError("Couldn't run git command (%s).\n" + "stderr:\n%s" % (cmd, se)) + return so, se + + def _check_url(self, url): + """ + Functon will check given url and try to verify if it's a valid + link. Sometimes it may happened that mercurial will issue basic + auth request that can cause whole API to hang when used from python + or other external calls. + + On failures it'll raise urllib2.HTTPError + """ + + #TODO: implement this + pass + + def _get_repo(self, create, src_url=None, update_after_clone=False, + bare=False): + if create and os.path.exists(self.path): + raise RepositoryError("Location already exist") + if src_url and not create: + raise RepositoryError("Create should be set to True if src_url is " + "given (clone operation creates repository)") + try: + if create and src_url: + self._check_url(src_url) + self.clone(src_url, update_after_clone, bare) + return Repo(self.path) + elif create: + os.mkdir(self.path) + if bare: + return Repo.init_bare(self.path) + else: + return Repo.init(self.path) + else: + return Repo(self.path) + except (NotGitRepository, OSError), err: + raise RepositoryError(err) + + def _get_all_revisions(self): + cmd = 'rev-list --all --date-order' + try: + so, se = self.run_git_command(cmd) + except RepositoryError: + # Can be raised for empty repositories + return [] + revisions = so.splitlines() + revisions.reverse() + return revisions + + def _get_revision(self, revision): + """ + For git backend we always return integer here. This way we ensure + that changset's revision attribute would become integer. + """ + pattern = re.compile(r'^[[0-9a-fA-F]{12}|[0-9a-fA-F]{40}]$') + is_bstr = lambda o: isinstance(o, (str, unicode)) + is_null = lambda o: len(o) == revision.count('0') + + if len(self.revisions) == 0: + raise EmptyRepositoryError("There are no changesets yet") + + if revision in (None, '', 'tip', 'HEAD', 'head', -1): + revision = self.revisions[-1] + + if ((is_bstr(revision) and revision.isdigit() and len(revision) < 12) + or isinstance(revision, int) or is_null(revision)): + try: + revision = self.revisions[int(revision)] + except: + raise ChangesetDoesNotExistError("Revision %r does not exist " + "for this repository %s" % (revision, self)) + + elif is_bstr(revision): + if not pattern.match(revision) or revision not in self.revisions: + raise ChangesetDoesNotExistError("Revision %r does not exist " + "for this repository %s" % (revision, self)) + + # Ensure we return full id + if not pattern.match(str(revision)): + raise ChangesetDoesNotExistError("Given revision %r not recognized" + % revision) + return revision + + def _get_archives(self, archive_name='tip'): + + for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]: + yield {"type": i[0], "extension": i[1], "node": archive_name} + + def _get_url(self, url): + """ + Returns normalized url. If schema is not given, would fall to + filesystem (``file:///``) schema. + """ + url = str(url) + if url != 'default' and not '://' in url: + url = ':///'.join(('file', url)) + return url + + @LazyProperty + def name(self): + return os.path.basename(self.path) + + @LazyProperty + def last_change(self): + """ + Returns last change made on this repository as datetime object + """ + return date_fromtimestamp(self._get_mtime(), makedate()[1]) + + def _get_mtime(self): + try: + return time.mktime(self.get_changeset().date.timetuple()) + except RepositoryError: + # fallback to filesystem + in_path = os.path.join(self.path, '.git', "index") + he_path = os.path.join(self.path, '.git', "HEAD") + if os.path.exists(in_path): + return os.stat(in_path).st_mtime + else: + return os.stat(he_path).st_mtime + + @LazyProperty + def description(self): + undefined_description = u'unknown' + description_path = os.path.join(self.path, '.git', 'description') + if os.path.isfile(description_path): + return safe_unicode(open(description_path).read()) + else: + return undefined_description + + @LazyProperty + def contact(self): + undefined_contact = u'Unknown' + return undefined_contact + + @property + def branches(self): + if not self.revisions: + return {} + refs = self._repo.refs.as_dict() + sortkey = lambda ctx: ctx[0] + _branches = [('/'.join(ref.split('/')[2:]), head) + for ref, head in refs.items() + if ref.startswith('refs/heads/') or + ref.startswith('refs/remotes/') and not ref.endswith('/HEAD')] + return OrderedDict(sorted(_branches, key=sortkey, reverse=False)) + + def _get_tags(self): + if not self.revisions: + return {} + sortkey = lambda ctx: ctx[0] + _tags = [('/'.join(ref.split('/')[2:]), head) for ref, head in + self._repo.get_refs().items() if ref.startswith('refs/tags/')] + return OrderedDict(sorted(_tags, key=sortkey, reverse=True)) + + @LazyProperty + def tags(self): + return self._get_tags() + + def tag(self, name, user, revision=None, message=None, date=None, + **kwargs): + """ + Creates and returns a tag for the given ``revision``. + + :param name: name for new tag + :param user: full username, i.e.: "Joe Doe " + :param revision: changeset id for which new tag would be created + :param message: message of the tag's commit + :param date: date of tag's commit + + :raises TagAlreadyExistError: if tag with same name already exists + """ + if name in self.tags: + raise TagAlreadyExistError("Tag %s already exists" % name) + changeset = self.get_changeset(revision) + message = message or "Added tag %s for commit %s" % (name, + changeset.raw_id) + self._repo.refs["refs/tags/%s" % name] = changeset._commit.id + + self.tags = self._get_tags() + return changeset + + def remove_tag(self, name, user, message=None, date=None): + """ + Removes tag with the given ``name``. + + :param name: name of the tag to be removed + :param user: full username, i.e.: "Joe Doe " + :param message: message of the tag's removal commit + :param date: date of tag's removal commit + + :raises TagDoesNotExistError: if tag with given name does not exists + """ + if name not in self.tags: + raise TagDoesNotExistError("Tag %s does not exist" % name) + tagpath = posixpath.join(self._repo.refs.path, 'refs', 'tags', name) + try: + os.remove(tagpath) + self.tags = self._get_tags() + except OSError, e: + raise RepositoryError(e.strerror) + + def get_changeset(self, revision=None): + """ + Returns ``GitChangeset`` object representing commit from git repository + at the given revision or head (most recent commit) if None given. + """ + if isinstance(revision, GitChangeset): + return revision + revision = self._get_revision(revision) + changeset = GitChangeset(repository=self, revision=revision) + return changeset + + def get_changesets(self, start=None, end=None, start_date=None, + end_date=None, branch_name=None, reverse=False): + """ + Returns iterator of ``GitChangeset`` objects from start to end (both + are inclusive), in ascending date order (unless ``reverse`` is set). + + :param start: changeset ID, as str; first returned changeset + :param end: changeset ID, as str; last returned changeset + :param start_date: if specified, changesets with commit date less than + ``start_date`` would be filtered out from returned set + :param end_date: if specified, changesets with commit date greater than + ``end_date`` would be filtered out from returned set + :param branch_name: if specified, changesets not reachable from given + branch would be filtered out from returned set + :param reverse: if ``True``, returned generator would be reversed + (meaning that returned changesets would have descending date order) + + :raise BranchDoesNotExistError: If given ``branch_name`` does not + exist. + :raise ChangesetDoesNotExistError: If changeset for given ``start`` or + ``end`` could not be found. + + """ + if branch_name and branch_name not in self.branches: + raise BranchDoesNotExistError("Branch '%s' not found" \ + % branch_name) + # %H at format means (full) commit hash, initial hashes are retrieved + # in ascending date order + cmd_template = 'log --date-order --reverse --pretty=format:"%H"' + cmd_params = {} + if start_date: + cmd_template += ' --since "$since"' + cmd_params['since'] = start_date.strftime('%m/%d/%y %H:%M:%S') + if end_date: + cmd_template += ' --until "$until"' + cmd_params['until'] = end_date.strftime('%m/%d/%y %H:%M:%S') + if branch_name: + cmd_template += ' $branch_name' + cmd_params['branch_name'] = branch_name + else: + cmd_template += ' --all' + + cmd = Template(cmd_template).safe_substitute(**cmd_params) + revs = self.run_git_command(cmd)[0].splitlines() + start_pos = 0 + end_pos = len(revs) + if start: + _start = self._get_revision(start) + try: + start_pos = revs.index(_start) + except ValueError: + pass + + if end is not None: + _end = self._get_revision(end) + try: + end_pos = revs.index(_end) + except ValueError: + pass + + if None not in [start, end] and start_pos > end_pos: + raise RepositoryError('start cannot be after end') + + if end_pos is not None: + end_pos += 1 + + revs = revs[start_pos:end_pos] + if reverse: + revs = reversed(revs) + for rev in revs: + yield self.get_changeset(rev) + + def get_diff(self, rev1, rev2, path=None, ignore_whitespace=False, + context=3): + """ + Returns (git like) *diff*, as plain text. Shows changes introduced by + ``rev2`` since ``rev1``. + + :param rev1: Entry point from which diff is shown. Can be + ``self.EMPTY_CHANGESET`` - in this case, patch showing all + the changes since empty state of the repository until ``rev2`` + :param rev2: Until which revision changes should be shown. + :param ignore_whitespace: If set to ``True``, would not show whitespace + changes. Defaults to ``False``. + :param context: How many lines before/after changed lines should be + shown. Defaults to ``3``. + """ + flags = ['-U%s' % context] + if ignore_whitespace: + flags.append('-w') + + if rev1 == self.EMPTY_CHANGESET: + rev2 = self.get_changeset(rev2).raw_id + cmd = ' '.join(['show'] + flags + [rev2]) + else: + rev1 = self.get_changeset(rev1).raw_id + rev2 = self.get_changeset(rev2).raw_id + cmd = ' '.join(['diff'] + flags + [rev1, rev2]) + + if path: + cmd += ' -- "%s"' % path + stdout, stderr = self.run_git_command(cmd) + # If we used 'show' command, strip first few lines (until actual diff + # starts) + if rev1 == self.EMPTY_CHANGESET: + lines = stdout.splitlines() + x = 0 + for line in lines: + if line.startswith('diff'): + break + x += 1 + # Append new line just like 'diff' command do + stdout = '\n'.join(lines[x:]) + '\n' + return stdout + + @LazyProperty + def in_memory_changeset(self): + """ + Returns ``GitInMemoryChangeset`` object for this repository. + """ + return GitInMemoryChangeset(self) + + def clone(self, url, update_after_clone=True, bare=False): + """ + Tries to clone changes from external location. + + :param update_after_clone: If set to ``False``, git won't checkout + working directory + :param bare: If set to ``True``, repository would be cloned into + *bare* git repository (no working directory at all). + """ + url = self._get_url(url) + cmd = ['clone'] + if bare: + cmd.append('--bare') + elif not update_after_clone: + cmd.append('--no-checkout') + cmd += ['--', '"%s"' % url, '"%s"' % self.path] + cmd = ' '.join(cmd) + # If error occurs run_git_command raises RepositoryError already + self.run_git_command(cmd) + + @LazyProperty + def workdir(self): + """ + Returns ``Workdir`` instance for this repository. + """ + return GitWorkdir(self) + + def get_config_value(self, section, name, config_file=None): + """ + Returns configuration value for a given [``section``] and ``name``. + + :param section: Section we want to retrieve value from + :param name: Name of configuration we want to retrieve + :param config_file: A path to file which should be used to retrieve + configuration from (might also be a list of file paths) + """ + if config_file is None: + config_file = [] + elif isinstance(config_file, basestring): + config_file = [config_file] + + def gen_configs(): + for path in config_file + self._config_files: + try: + yield ConfigFile.from_path(path) + except (IOError, OSError, ValueError): + continue + + for config in gen_configs(): + try: + return config.get(section, name) + except KeyError: + continue + return None + + def get_user_name(self, config_file=None): + """ + Returns user's name from global configuration file. + + :param config_file: A path to file which should be used to retrieve + configuration from (might also be a list of file paths) + """ + return self.get_config_value('user', 'name', config_file) + + def get_user_email(self, config_file=None): + """ + Returns user's email from global configuration file. + + :param config_file: A path to file which should be used to retrieve + configuration from (might also be a list of file paths) + """ + return self.get_config_value('user', 'email', config_file) diff --git a/rhodecode/lib/vcs/backends/git/workdir.py b/rhodecode/lib/vcs/backends/git/workdir.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/backends/git/workdir.py @@ -0,0 +1,31 @@ +import re +from rhodecode.lib.vcs.backends.base import BaseWorkdir +from rhodecode.lib.vcs.exceptions import RepositoryError +from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError + + +class GitWorkdir(BaseWorkdir): + + def get_branch(self): + headpath = self.repository._repo.refs.refpath('HEAD') + try: + content = open(headpath).read() + match = re.match(r'^ref: refs/heads/(?P.+)\n$', content) + if match: + return match.groupdict()['branch'] + else: + raise RepositoryError("Couldn't compute workdir's branch") + except IOError: + # Try naive way... + raise RepositoryError("Couldn't compute workdir's branch") + + def get_changeset(self): + return self.repository.get_changeset( + self.repository._repo.refs.as_dict().get('HEAD')) + + def checkout_branch(self, branch=None): + if branch is None: + branch = self.repository.DEFAULT_BRANCH_NAME + if branch not in self.repository.branches: + raise BranchDoesNotExistError + self.repository.run_git_command(['checkout', branch]) diff --git a/rhodecode/lib/vcs/backends/hg/__init__.py b/rhodecode/lib/vcs/backends/hg/__init__.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/backends/hg/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +""" + vcs.backends.hg + ~~~~~~~~~~~~~~~~ + + Mercurial backend implementation. + + :created_on: Apr 8, 2010 + :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak. +""" + +from .repository import MercurialRepository +from .changeset import MercurialChangeset +from .inmemory import MercurialInMemoryChangeset +from .workdir import MercurialWorkdir + + +__all__ = [ + 'MercurialRepository', 'MercurialChangeset', + 'MercurialInMemoryChangeset', 'MercurialWorkdir', +] diff --git a/rhodecode/lib/vcs/backends/hg/changeset.py b/rhodecode/lib/vcs/backends/hg/changeset.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/backends/hg/changeset.py @@ -0,0 +1,338 @@ +import os +import posixpath + +from rhodecode.lib.vcs.backends.base import BaseChangeset +from rhodecode.lib.vcs.conf import settings +from rhodecode.lib.vcs.exceptions import ChangesetDoesNotExistError, \ + ChangesetError, ImproperArchiveTypeError, NodeDoesNotExistError, VCSError +from rhodecode.lib.vcs.nodes import AddedFileNodesGenerator, ChangedFileNodesGenerator, \ + DirNode, FileNode, NodeKind, RemovedFileNodesGenerator, RootNode + +from rhodecode.lib.vcs.utils import safe_str, safe_unicode, date_fromtimestamp +from rhodecode.lib.vcs.utils.lazy import LazyProperty +from rhodecode.lib.vcs.utils.paths import get_dirs_for_path + +from ...utils.hgcompat import archival, hex + + +class MercurialChangeset(BaseChangeset): + """ + Represents state of the repository at the single revision. + """ + + def __init__(self, repository, revision): + self.repository = repository + self.raw_id = revision + self._ctx = repository._repo[revision] + self.revision = self._ctx._rev + self.nodes = {} + + @LazyProperty + def tags(self): + return map(safe_unicode, self._ctx.tags()) + + @LazyProperty + def branch(self): + return safe_unicode(self._ctx.branch()) + + @LazyProperty + def message(self): + return safe_unicode(self._ctx.description()) + + @LazyProperty + def author(self): + return safe_unicode(self._ctx.user()) + + @LazyProperty + def date(self): + return date_fromtimestamp(*self._ctx.date()) + + @LazyProperty + def status(self): + """ + Returns modified, added, removed, deleted files for current changeset + """ + return self.repository._repo.status(self._ctx.p1().node(), + self._ctx.node()) + + @LazyProperty + def _file_paths(self): + return list(self._ctx) + + @LazyProperty + def _dir_paths(self): + p = list(set(get_dirs_for_path(*self._file_paths))) + p.insert(0, '') + return p + + @LazyProperty + def _paths(self): + return self._dir_paths + self._file_paths + + @LazyProperty + def id(self): + if self.last: + return u'tip' + return self.short_id + + @LazyProperty + def short_id(self): + return self.raw_id[:12] + + @LazyProperty + def parents(self): + """ + Returns list of parents changesets. + """ + return [self.repository.get_changeset(parent.rev()) + for parent in self._ctx.parents() if parent.rev() >= 0] + + def next(self, branch=None): + + if branch and self.branch != branch: + raise VCSError('Branch option used on changeset not belonging ' + 'to that branch') + + def _next(changeset, branch): + try: + next_ = changeset.revision + 1 + next_rev = changeset.repository.revisions[next_] + except IndexError: + raise ChangesetDoesNotExistError + cs = changeset.repository.get_changeset(next_rev) + + if branch and branch != cs.branch: + return _next(cs, branch) + + return cs + + return _next(self, branch) + + def prev(self, branch=None): + if branch and self.branch != branch: + raise VCSError('Branch option used on changeset not belonging ' + 'to that branch') + + def _prev(changeset, branch): + try: + prev_ = changeset.revision - 1 + if prev_ < 0: + raise IndexError + prev_rev = changeset.repository.revisions[prev_] + except IndexError: + raise ChangesetDoesNotExistError + + cs = changeset.repository.get_changeset(prev_rev) + + if branch and branch != cs.branch: + return _prev(cs, branch) + + return cs + + return _prev(self, branch) + + def _fix_path(self, path): + """ + Paths are stored without trailing slash so we need to get rid off it if + needed. Also mercurial keeps filenodes as str so we need to decode + from unicode to str + """ + if path.endswith('/'): + path = path.rstrip('/') + + return safe_str(path) + + def _get_kind(self, path): + path = self._fix_path(path) + if path in self._file_paths: + return NodeKind.FILE + elif path in self._dir_paths: + return NodeKind.DIR + else: + raise ChangesetError("Node does not exist at the given path %r" + % (path)) + + def _get_filectx(self, path): + path = self._fix_path(path) + if self._get_kind(path) != NodeKind.FILE: + raise ChangesetError("File does not exist for revision %r at " + " %r" % (self.revision, path)) + return self._ctx.filectx(path) + + def get_file_mode(self, path): + """ + Returns stat mode of the file at the given ``path``. + """ + fctx = self._get_filectx(path) + if 'x' in fctx.flags(): + return 0100755 + else: + return 0100644 + + def get_file_content(self, path): + """ + Returns content of the file at given ``path``. + """ + fctx = self._get_filectx(path) + return fctx.data() + + def get_file_size(self, path): + """ + Returns size of the file at given ``path``. + """ + fctx = self._get_filectx(path) + return fctx.size() + + def get_file_changeset(self, path): + """ + Returns last commit of the file at the given ``path``. + """ + fctx = self._get_filectx(path) + changeset = self.repository.get_changeset(fctx.linkrev()) + return changeset + + def get_file_history(self, path): + """ + Returns history of file as reversed list of ``Changeset`` objects for + which file at given ``path`` has been modified. + """ + fctx = self._get_filectx(path) + nodes = [fctx.filectx(x).node() for x in fctx.filelog()] + changesets = [self.repository.get_changeset(hex(node)) + for node in reversed(nodes)] + return changesets + + def get_file_annotate(self, path): + """ + Returns a list of three element tuples with lineno,changeset and line + """ + fctx = self._get_filectx(path) + annotate = [] + for i, annotate_data in enumerate(fctx.annotate()): + ln_no = i + 1 + annotate.append((ln_no, self.repository\ + .get_changeset(hex(annotate_data[0].node())), + annotate_data[1],)) + + return annotate + + def fill_archive(self, stream=None, kind='tgz', prefix=None, + subrepos=False): + """ + Fills up given stream. + + :param stream: file like object. + :param kind: one of following: ``zip``, ``tgz`` or ``tbz2``. + Default: ``tgz``. + :param prefix: name of root directory in archive. + Default is repository name and changeset's raw_id joined with dash + (``repo-tip.``). + :param subrepos: include subrepos in this archive. + + :raise ImproperArchiveTypeError: If given kind is wrong. + :raise VcsError: If given stream is None + """ + + allowed_kinds = settings.ARCHIVE_SPECS.keys() + if kind not in allowed_kinds: + raise ImproperArchiveTypeError('Archive kind not supported use one' + 'of %s', allowed_kinds) + + if stream is None: + raise VCSError('You need to pass in a valid stream for filling' + ' with archival data') + + if prefix is None: + prefix = '%s-%s' % (self.repository.name, self.short_id) + elif prefix.startswith('/'): + raise VCSError("Prefix cannot start with leading slash") + elif prefix.strip() == '': + raise VCSError("Prefix cannot be empty") + + archival.archive(self.repository._repo, stream, self.raw_id, + kind, prefix=prefix, subrepos=subrepos) + + #stream.close() + + if stream.closed and hasattr(stream, 'name'): + stream = open(stream.name, 'rb') + elif hasattr(stream, 'mode') and 'r' not in stream.mode: + stream = open(stream.name, 'rb') + else: + stream.seek(0) + + def get_nodes(self, path): + """ + Returns combined ``DirNode`` and ``FileNode`` objects list representing + state of changeset at the given ``path``. If node at the given ``path`` + is not instance of ``DirNode``, ChangesetError would be raised. + """ + + if self._get_kind(path) != NodeKind.DIR: + raise ChangesetError("Directory does not exist for revision %r at " + " %r" % (self.revision, path)) + path = self._fix_path(path) + filenodes = [FileNode(f, changeset=self) for f in self._file_paths + if os.path.dirname(f) == path] + dirs = path == '' and '' or [d for d in self._dir_paths + if d and posixpath.dirname(d) == path] + dirnodes = [DirNode(d, changeset=self) for d in dirs + if os.path.dirname(d) == path] + nodes = dirnodes + filenodes + # cache nodes + for node in nodes: + self.nodes[node.path] = node + nodes.sort() + return nodes + + def get_node(self, path): + """ + Returns ``Node`` object from the given ``path``. If there is no node at + the given ``path``, ``ChangesetError`` would be raised. + """ + + path = self._fix_path(path) + + if not path in self.nodes: + if path in self._file_paths: + node = FileNode(path, changeset=self) + elif path in self._dir_paths or path in self._dir_paths: + if path == '': + node = RootNode(changeset=self) + else: + node = DirNode(path, changeset=self) + else: + raise NodeDoesNotExistError("There is no file nor directory " + "at the given path: %r at revision %r" + % (path, self.short_id)) + # cache node + self.nodes[path] = node + return self.nodes[path] + + @LazyProperty + def affected_files(self): + """ + Get's a fast accessible file changes for given changeset + """ + return self._ctx.files() + + @property + def added(self): + """ + Returns list of added ``FileNode`` objects. + """ + return AddedFileNodesGenerator([n for n in self.status[1]], self) + + @property + def changed(self): + """ + Returns list of modified ``FileNode`` objects. + """ + return ChangedFileNodesGenerator([n for n in self.status[0]], self) + + @property + def removed(self): + """ + Returns list of removed ``FileNode`` objects. + """ + return RemovedFileNodesGenerator([n for n in self.status[2]], self) diff --git a/rhodecode/lib/vcs/backends/hg/inmemory.py b/rhodecode/lib/vcs/backends/hg/inmemory.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/backends/hg/inmemory.py @@ -0,0 +1,110 @@ +import datetime +import errno + +from rhodecode.lib.vcs.backends.base import BaseInMemoryChangeset +from rhodecode.lib.vcs.exceptions import RepositoryError + +from ...utils.hgcompat import memfilectx, memctx, hex + + +class MercurialInMemoryChangeset(BaseInMemoryChangeset): + + def commit(self, message, author, parents=None, branch=None, date=None, + **kwargs): + """ + Performs in-memory commit (doesn't check workdir in any way) and + returns newly created ``Changeset``. Updates repository's + ``revisions``. + + :param message: message of the commit + :param author: full username, i.e. "Joe Doe " + :param parents: single parent or sequence of parents from which commit + would be derieved + :param date: ``datetime.datetime`` instance. Defaults to + ``datetime.datetime.now()``. + :param branch: branch name, as string. If none given, default backend's + branch would be used. + + :raises ``CommitError``: if any error occurs while committing + """ + self.check_integrity(parents) + + from .repository import MercurialRepository + if not isinstance(message, str) or not isinstance(author, str): + raise RepositoryError('Given message and author needs to be ' + 'an instance') + + if branch is None: + branch = MercurialRepository.DEFAULT_BRANCH_NAME + kwargs['branch'] = branch + + def filectxfn(_repo, memctx, path): + """ + Marks given path as added/changed/removed in a given _repo. This is + for internal mercurial commit function. + """ + + # check if this path is removed + if path in (node.path for node in self.removed): + # Raising exception is a way to mark node for removal + raise IOError(errno.ENOENT, '%s is deleted' % path) + + # check if this path is added + for node in self.added: + if node.path == path: + return memfilectx(path=node.path, + data=(node.content.encode('utf8') + if not node.is_binary else node.content), + islink=False, + isexec=node.is_executable, + copied=False) + + # or changed + for node in self.changed: + if node.path == path: + return memfilectx(path=node.path, + data=(node.content.encode('utf8') + if not node.is_binary else node.content), + islink=False, + isexec=node.is_executable, + copied=False) + + raise RepositoryError("Given path haven't been marked as added," + "changed or removed (%s)" % path) + + parents = [None, None] + for i, parent in enumerate(self.parents): + if parent is not None: + parents[i] = parent._ctx.node() + + if date and isinstance(date, datetime.datetime): + date = date.ctime() + + commit_ctx = memctx(repo=self.repository._repo, + parents=parents, + text='', + files=self.get_paths(), + filectxfn=filectxfn, + user=author, + date=date, + extra=kwargs) + + # injecting given _repo params + commit_ctx._text = message + commit_ctx._user = author + commit_ctx._date = date + + # TODO: Catch exceptions! + n = self.repository._repo.commitctx(commit_ctx) + # Returns mercurial node + self._commit_ctx = commit_ctx # For reference + # Update vcs repository object & recreate mercurial _repo + # new_ctx = self.repository._repo[node] + # new_tip = self.repository.get_changeset(new_ctx.hex()) + new_id = hex(n) + self.repository.revisions.append(new_id) + self._repo = self.repository._get_repo(create=False) + self.repository.branches = self.repository._get_branches() + tip = self.repository.get_changeset() + self.reset() + return tip diff --git a/rhodecode/lib/vcs/backends/hg/repository.py b/rhodecode/lib/vcs/backends/hg/repository.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/backends/hg/repository.py @@ -0,0 +1,521 @@ +import os +import time +import datetime +import urllib +import urllib2 + +from rhodecode.lib.vcs.backends.base import BaseRepository +from .workdir import MercurialWorkdir +from .changeset import MercurialChangeset +from .inmemory import MercurialInMemoryChangeset + +from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError, \ + ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, \ + VCSError, TagAlreadyExistError, TagDoesNotExistError +from rhodecode.lib.vcs.utils import author_email, author_name, date_fromtimestamp, \ + makedate, safe_unicode +from rhodecode.lib.vcs.utils.lazy import LazyProperty +from rhodecode.lib.vcs.utils.ordered_dict import OrderedDict +from rhodecode.lib.vcs.utils.paths import abspath + +from ...utils.hgcompat import ui, nullid, match, patch, diffopts, clone, \ + get_contact, pull, localrepository, RepoLookupError, Abort, RepoError, hex + + +class MercurialRepository(BaseRepository): + """ + Mercurial repository backend + """ + DEFAULT_BRANCH_NAME = 'default' + scm = 'hg' + + def __init__(self, repo_path, create=False, baseui=None, src_url=None, + update_after_clone=False): + """ + Raises RepositoryError if repository could not be find at the given + ``repo_path``. + + :param repo_path: local path of the repository + :param create=False: if set to True, would try to create repository if + it does not exist rather than raising exception + :param baseui=None: user data + :param src_url=None: would try to clone repository from given location + :param update_after_clone=False: sets update of working copy after + making a clone + """ + + if not isinstance(repo_path, str): + raise VCSError('Mercurial backend requires repository path to ' + 'be instance of got %s instead' % + type(repo_path)) + + self.path = abspath(repo_path) + self.baseui = baseui or ui.ui() + # We've set path and ui, now we can set _repo itself + self._repo = self._get_repo(create, src_url, update_after_clone) + + @property + def _empty(self): + """ + Checks if repository is empty without any changesets + """ + # TODO: Following raises errors when using InMemoryChangeset... + # return len(self._repo.changelog) == 0 + return len(self.revisions) == 0 + + @LazyProperty + def revisions(self): + """ + Returns list of revisions' ids, in ascending order. Being lazy + attribute allows external tools to inject shas from cache. + """ + return self._get_all_revisions() + + @LazyProperty + def name(self): + return os.path.basename(self.path) + + @LazyProperty + def branches(self): + return self._get_branches() + + def _get_branches(self, closed=False): + """ + Get's branches for this repository + Returns only not closed branches by default + + :param closed: return also closed branches for mercurial + """ + + if self._empty: + return {} + + def _branchtags(localrepo): + """ + Patched version of mercurial branchtags to not return the closed + branches + + :param localrepo: locarepository instance + """ + + bt = {} + bt_closed = {} + for bn, heads in localrepo.branchmap().iteritems(): + tip = heads[-1] + if 'close' in localrepo.changelog.read(tip)[5]: + bt_closed[bn] = tip + else: + bt[bn] = tip + + if closed: + bt.update(bt_closed) + return bt + + sortkey = lambda ctx: ctx[0] # sort by name + _branches = [(safe_unicode(n), hex(h),) for n, h in + _branchtags(self._repo).items()] + + return OrderedDict(sorted(_branches, key=sortkey, reverse=False)) + + @LazyProperty + def tags(self): + """ + Get's tags for this repository + """ + return self._get_tags() + + def _get_tags(self): + if self._empty: + return {} + + sortkey = lambda ctx: ctx[0] # sort by name + _tags = [(safe_unicode(n), hex(h),) for n, h in + self._repo.tags().items()] + + return OrderedDict(sorted(_tags, key=sortkey, reverse=True)) + + def tag(self, name, user, revision=None, message=None, date=None, + **kwargs): + """ + Creates and returns a tag for the given ``revision``. + + :param name: name for new tag + :param user: full username, i.e.: "Joe Doe " + :param revision: changeset id for which new tag would be created + :param message: message of the tag's commit + :param date: date of tag's commit + + :raises TagAlreadyExistError: if tag with same name already exists + """ + if name in self.tags: + raise TagAlreadyExistError("Tag %s already exists" % name) + changeset = self.get_changeset(revision) + local = kwargs.setdefault('local', False) + + if message is None: + message = "Added tag %s for changeset %s" % (name, + changeset.short_id) + + if date is None: + date = datetime.datetime.now().ctime() + + try: + self._repo.tag(name, changeset._ctx.node(), message, local, user, + date) + except Abort, e: + raise RepositoryError(e.message) + + # Reinitialize tags + self.tags = self._get_tags() + tag_id = self.tags[name] + + return self.get_changeset(revision=tag_id) + + def remove_tag(self, name, user, message=None, date=None): + """ + Removes tag with the given ``name``. + + :param name: name of the tag to be removed + :param user: full username, i.e.: "Joe Doe " + :param message: message of the tag's removal commit + :param date: date of tag's removal commit + + :raises TagDoesNotExistError: if tag with given name does not exists + """ + if name not in self.tags: + raise TagDoesNotExistError("Tag %s does not exist" % name) + if message is None: + message = "Removed tag %s" % name + if date is None: + date = datetime.datetime.now().ctime() + local = False + + try: + self._repo.tag(name, nullid, message, local, user, date) + self.tags = self._get_tags() + except Abort, e: + raise RepositoryError(e.message) + + @LazyProperty + def bookmarks(self): + """ + Get's bookmarks for this repository + """ + return self._get_bookmarks() + + def _get_bookmarks(self): + if self._empty: + return {} + + sortkey = lambda ctx: ctx[0] # sort by name + _bookmarks = [(safe_unicode(n), hex(h),) for n, h in + self._repo._bookmarks.items()] + return OrderedDict(sorted(_bookmarks, key=sortkey, reverse=True)) + + def _get_all_revisions(self): + + return map(lambda x: hex(x[7]), self._repo.changelog.index)[:-1] + + def get_diff(self, rev1, rev2, path='', ignore_whitespace=False, + context=3): + """ + Returns (git like) *diff*, as plain text. Shows changes introduced by + ``rev2`` since ``rev1``. + + :param rev1: Entry point from which diff is shown. Can be + ``self.EMPTY_CHANGESET`` - in this case, patch showing all + the changes since empty state of the repository until ``rev2`` + :param rev2: Until which revision changes should be shown. + :param ignore_whitespace: If set to ``True``, would not show whitespace + changes. Defaults to ``False``. + :param context: How many lines before/after changed lines should be + shown. Defaults to ``3``. + """ + # Check if given revisions are present at repository (may raise + # ChangesetDoesNotExistError) + if rev1 != self.EMPTY_CHANGESET: + self.get_changeset(rev1) + self.get_changeset(rev2) + + file_filter = match(self.path, '', [path]) + return ''.join(patch.diff(self._repo, rev1, rev2, match=file_filter, + opts=diffopts(git=True, + ignorews=ignore_whitespace, + context=context))) + + def _check_url(self, url): + """ + Function will check given url and try to verify if it's a valid + link. Sometimes it may happened that mercurial will issue basic + auth request that can cause whole API to hang when used from python + or other external calls. + + On failures it'll raise urllib2.HTTPError, return code 200 if url + is valid or True if it's a local path + """ + + from mercurial.util import url as Url + + # those authnadlers are patched for python 2.6.5 bug an + # infinit looping when given invalid resources + from mercurial.url import httpbasicauthhandler, httpdigestauthhandler + + # check first if it's not an local url + if os.path.isdir(url) or url.startswith('file:'): + return True + + handlers = [] + test_uri, authinfo = Url(url).authinfo() + + if authinfo: + #create a password manager + passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm() + passmgr.add_password(*authinfo) + + handlers.extend((httpbasicauthhandler(passmgr), + httpdigestauthhandler(passmgr))) + + o = urllib2.build_opener(*handlers) + o.addheaders = [('Content-Type', 'application/mercurial-0.1'), + ('Accept', 'application/mercurial-0.1')] + + q = {"cmd": 'between'} + q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)}) + qs = '?%s' % urllib.urlencode(q) + cu = "%s%s" % (test_uri, qs) + req = urllib2.Request(cu, None, {}) + + try: + resp = o.open(req) + return resp.code == 200 + except Exception, e: + # means it cannot be cloned + raise urllib2.URLError(e) + + def _get_repo(self, create, src_url=None, update_after_clone=False): + """ + Function will check for mercurial repository in given path and return + a localrepo object. If there is no repository in that path it will + raise an exception unless ``create`` parameter is set to True - in + that case repository would be created and returned. + If ``src_url`` is given, would try to clone repository from the + location at given clone_point. Additionally it'll make update to + working copy accordingly to ``update_after_clone`` flag + """ + try: + if src_url: + url = str(self._get_url(src_url)) + opts = {} + if not update_after_clone: + opts.update({'noupdate': True}) + try: + self._check_url(url) + clone(self.baseui, url, self.path, **opts) +# except urllib2.URLError: +# raise Abort("Got HTTP 404 error") + except Exception: + raise + # Don't try to create if we've already cloned repo + create = False + return localrepository(self.baseui, self.path, create=create) + except (Abort, RepoError), err: + if create: + msg = "Cannot create repository at %s. Original error was %s"\ + % (self.path, err) + else: + msg = "Not valid repository at %s. Original error was %s"\ + % (self.path, err) + raise RepositoryError(msg) + + @LazyProperty + def in_memory_changeset(self): + return MercurialInMemoryChangeset(self) + + @LazyProperty + def description(self): + undefined_description = u'unknown' + return safe_unicode(self._repo.ui.config('web', 'description', + undefined_description, untrusted=True)) + + @LazyProperty + def contact(self): + undefined_contact = u'Unknown' + return safe_unicode(get_contact(self._repo.ui.config) + or undefined_contact) + + @LazyProperty + def last_change(self): + """ + Returns last change made on this repository as datetime object + """ + return date_fromtimestamp(self._get_mtime(), makedate()[1]) + + def _get_mtime(self): + try: + return time.mktime(self.get_changeset().date.timetuple()) + except RepositoryError: + #fallback to filesystem + cl_path = os.path.join(self.path, '.hg', "00changelog.i") + st_path = os.path.join(self.path, '.hg', "store") + if os.path.exists(cl_path): + return os.stat(cl_path).st_mtime + else: + return os.stat(st_path).st_mtime + + def _get_hidden(self): + return self._repo.ui.configbool("web", "hidden", untrusted=True) + + def _get_revision(self, revision): + """ + Get's an ID revision given as str. This will always return a fill + 40 char revision number + + :param revision: str or int or None + """ + + if self._empty: + raise EmptyRepositoryError("There are no changesets yet") + + if revision in [-1, 'tip', None]: + revision = 'tip' + + try: + revision = hex(self._repo.lookup(revision)) + except (IndexError, ValueError, RepoLookupError, TypeError): + raise ChangesetDoesNotExistError("Revision %r does not " + "exist for this repository %s" \ + % (revision, self)) + return revision + + def _get_archives(self, archive_name='tip'): + allowed = self.baseui.configlist("web", "allow_archive", + untrusted=True) + for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]: + if i[0] in allowed or self._repo.ui.configbool("web", + "allow" + i[0], + untrusted=True): + yield {"type": i[0], "extension": i[1], "node": archive_name} + + def _get_url(self, url): + """ + Returns normalized url. If schema is not given, would fall + to filesystem + (``file:///``) schema. + """ + url = str(url) + if url != 'default' and not '://' in url: + url = "file:" + urllib.pathname2url(url) + return url + + def get_changeset(self, revision=None): + """ + Returns ``MercurialChangeset`` object representing repository's + changeset at the given ``revision``. + """ + revision = self._get_revision(revision) + changeset = MercurialChangeset(repository=self, revision=revision) + return changeset + + def get_changesets(self, start=None, end=None, start_date=None, + end_date=None, branch_name=None, reverse=False): + """ + Returns iterator of ``MercurialChangeset`` objects from start to end + (both are inclusive) + + :param start: None, str, int or mercurial lookup format + :param end: None, str, int or mercurial lookup format + :param start_date: + :param end_date: + :param branch_name: + :param reversed: return changesets in reversed order + """ + + start_raw_id = self._get_revision(start) + start_pos = self.revisions.index(start_raw_id) if start else None + end_raw_id = self._get_revision(end) + end_pos = self.revisions.index(end_raw_id) if end else None + + if None not in [start, end] and start_pos > end_pos: + raise RepositoryError("start revision '%s' cannot be " + "after end revision '%s'" % (start, end)) + + if branch_name and branch_name not in self.branches.keys(): + raise BranchDoesNotExistError('Such branch %s does not exists for' + ' this repository' % branch_name) + if end_pos is not None: + end_pos += 1 + + slice_ = reversed(self.revisions[start_pos:end_pos]) if reverse else \ + self.revisions[start_pos:end_pos] + + for id_ in slice_: + cs = self.get_changeset(id_) + if branch_name and cs.branch != branch_name: + continue + if start_date and cs.date < start_date: + continue + if end_date and cs.date > end_date: + continue + + yield cs + + def pull(self, url): + """ + Tries to pull changes from external location. + """ + url = self._get_url(url) + try: + pull(self.baseui, self._repo, url) + except Abort, err: + # Propagate error but with vcs's type + raise RepositoryError(str(err)) + + @LazyProperty + def workdir(self): + """ + Returns ``Workdir`` instance for this repository. + """ + return MercurialWorkdir(self) + + def get_config_value(self, section, name, config_file=None): + """ + Returns configuration value for a given [``section``] and ``name``. + + :param section: Section we want to retrieve value from + :param name: Name of configuration we want to retrieve + :param config_file: A path to file which should be used to retrieve + configuration from (might also be a list of file paths) + """ + if config_file is None: + config_file = [] + elif isinstance(config_file, basestring): + config_file = [config_file] + + config = self._repo.ui + for path in config_file: + config.readconfig(path) + return config.config(section, name) + + def get_user_name(self, config_file=None): + """ + Returns user's name from global configuration file. + + :param config_file: A path to file which should be used to retrieve + configuration from (might also be a list of file paths) + """ + username = self.get_config_value('ui', 'username') + if username: + return author_name(username) + return None + + def get_user_email(self, config_file=None): + """ + Returns user's email from global configuration file. + + :param config_file: A path to file which should be used to retrieve + configuration from (might also be a list of file paths) + """ + username = self.get_config_value('ui', 'username') + if username: + return author_email(username) + return None diff --git a/rhodecode/lib/vcs/backends/hg/workdir.py b/rhodecode/lib/vcs/backends/hg/workdir.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/backends/hg/workdir.py @@ -0,0 +1,21 @@ +from rhodecode.lib.vcs.backends.base import BaseWorkdir +from rhodecode.lib.vcs.exceptions import BranchDoesNotExistError + +from ...utils.hgcompat import hg_merge + + +class MercurialWorkdir(BaseWorkdir): + + def get_branch(self): + return self.repository._repo.dirstate.branch() + + def get_changeset(self): + return self.repository.get_changeset() + + def checkout_branch(self, branch=None): + if branch is None: + branch = self.repository.DEFAULT_BRANCH_NAME + if branch not in self.repository.branches: + raise BranchDoesNotExistError + + hg_merge.update(self.repository._repo, branch, False, False, None) diff --git a/rhodecode/lib/vcs/conf/__init__.py b/rhodecode/lib/vcs/conf/__init__.py new file mode 100644 diff --git a/rhodecode/lib/vcs/conf/settings.py b/rhodecode/lib/vcs/conf/settings.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/conf/settings.py @@ -0,0 +1,33 @@ +import os +import tempfile +from rhodecode.lib.vcs.utils.paths import get_user_home + +abspath = lambda * p: os.path.abspath(os.path.join(*p)) + +VCSRC_PATH = os.environ.get('VCSRC_PATH') + +if not VCSRC_PATH: + HOME_ = get_user_home() + if not HOME_: + HOME_ = tempfile.gettempdir() + +VCSRC_PATH = VCSRC_PATH or abspath(HOME_, '.vcsrc') +if os.path.isdir(VCSRC_PATH): + VCSRC_PATH = os.path.join(VCSRC_PATH, '__init__.py') + +BACKENDS = { + 'hg': 'vcs.backends.hg.MercurialRepository', + 'git': 'vcs.backends.git.GitRepository', +} + +ARCHIVE_SPECS = { + 'tar': ('application/x-tar', '.tar'), + 'tbz2': ('application/x-bzip2', '.tar.bz2'), + 'tgz': ('application/x-gzip', '.tar.gz'), + 'zip': ('application/zip', '.zip'), +} + +BACKENDS = { + 'hg': 'rhodecode.lib.vcs.backends.hg.MercurialRepository', + 'git': 'rhodecode.lib.vcs.backends.git.GitRepository', +} diff --git a/rhodecode/lib/vcs/exceptions.py b/rhodecode/lib/vcs/exceptions.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/exceptions.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +""" + vcs.exceptions + ~~~~~~~~~~~~~~ + + Custom exceptions module + + :created_on: Apr 8, 2010 + :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak. +""" + + +class VCSError(Exception): + pass + + +class RepositoryError(VCSError): + pass + + +class EmptyRepositoryError(RepositoryError): + pass + + +class TagAlreadyExistError(RepositoryError): + pass + + +class TagDoesNotExistError(RepositoryError): + pass + + +class BranchAlreadyExistError(RepositoryError): + pass + + +class BranchDoesNotExistError(RepositoryError): + pass + + +class ChangesetError(RepositoryError): + pass + + +class ChangesetDoesNotExistError(ChangesetError): + pass + + +class CommitError(RepositoryError): + pass + + +class NothingChangedError(CommitError): + pass + + +class NodeError(VCSError): + pass + + +class RemovedFileNodeError(NodeError): + pass + + +class NodeAlreadyExistsError(CommitError): + pass + + +class NodeAlreadyChangedError(CommitError): + pass + + +class NodeDoesNotExistError(CommitError): + pass + + +class NodeNotChangedError(CommitError): + pass + + +class NodeAlreadyAddedError(CommitError): + pass + + +class NodeAlreadyRemovedError(CommitError): + pass + + +class ImproperArchiveTypeError(VCSError): + pass + +class CommandError(VCSError): + pass diff --git a/rhodecode/lib/vcs/nodes.py b/rhodecode/lib/vcs/nodes.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/nodes.py @@ -0,0 +1,551 @@ +# -*- coding: utf-8 -*- +""" + vcs.nodes + ~~~~~~~~~ + + Module holding everything related to vcs nodes. + + :created_on: Apr 8, 2010 + :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak. +""" +import stat +import posixpath +import mimetypes + +from rhodecode.lib.vcs.utils.lazy import LazyProperty +from rhodecode.lib.vcs.utils import safe_unicode +from rhodecode.lib.vcs.exceptions import NodeError +from rhodecode.lib.vcs.exceptions import RemovedFileNodeError + +from pygments import lexers + + +class NodeKind: + DIR = 1 + FILE = 2 + + +class NodeState: + ADDED = u'added' + CHANGED = u'changed' + NOT_CHANGED = u'not changed' + REMOVED = u'removed' + + +class NodeGeneratorBase(object): + """ + Base class for removed added and changed filenodes, it's a lazy generator + class that will create filenodes only on iteration or call + + The len method doesn't need to create filenodes at all + """ + + def __init__(self, current_paths, cs): + self.cs = cs + self.current_paths = current_paths + + def __call__(self): + return [n for n in self] + + def __getslice__(self, i, j): + for p in self.current_paths[i:j]: + yield self.cs.get_node(p) + + def __len__(self): + return len(self.current_paths) + + def __iter__(self): + for p in self.current_paths: + yield self.cs.get_node(p) + + +class AddedFileNodesGenerator(NodeGeneratorBase): + """ + Class holding Added files for current changeset + """ + pass + + +class ChangedFileNodesGenerator(NodeGeneratorBase): + """ + Class holding Changed files for current changeset + """ + pass + + +class RemovedFileNodesGenerator(NodeGeneratorBase): + """ + Class holding removed files for current changeset + """ + def __iter__(self): + for p in self.current_paths: + yield RemovedFileNode(path=p) + + def __getslice__(self, i, j): + for p in self.current_paths[i:j]: + yield RemovedFileNode(path=p) + + +class Node(object): + """ + Simplest class representing file or directory on repository. SCM backends + should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node`` + directly. + + Node's ``path`` cannot start with slash as we operate on *relative* paths + only. Moreover, every single node is identified by the ``path`` attribute, + so it cannot end with slash, too. Otherwise, path could lead to mistakes. + """ + + def __init__(self, path, kind): + if path.startswith('/'): + raise NodeError("Cannot initialize Node objects with slash at " + "the beginning as only relative paths are supported") + self.path = path.rstrip('/') + if path == '' and kind != NodeKind.DIR: + raise NodeError("Only DirNode and its subclasses may be " + "initialized with empty path") + self.kind = kind + #self.dirs, self.files = [], [] + if self.is_root() and not self.is_dir(): + raise NodeError("Root node cannot be FILE kind") + + @LazyProperty + def parent(self): + parent_path = self.get_parent_path() + if parent_path: + if self.changeset: + return self.changeset.get_node(parent_path) + return DirNode(parent_path) + return None + + @LazyProperty + def name(self): + """ + Returns name of the node so if its path + then only last part is returned. + """ + return safe_unicode(self.path.rstrip('/').split('/')[-1]) + + def _get_kind(self): + return self._kind + + def _set_kind(self, kind): + if hasattr(self, '_kind'): + raise NodeError("Cannot change node's kind") + else: + self._kind = kind + # Post setter check (path's trailing slash) + if self.path.endswith('/'): + raise NodeError("Node's path cannot end with slash") + + kind = property(_get_kind, _set_kind) + + def __cmp__(self, other): + """ + Comparator using name of the node, needed for quick list sorting. + """ + kind_cmp = cmp(self.kind, other.kind) + if kind_cmp: + return kind_cmp + return cmp(self.name, other.name) + + def __eq__(self, other): + for attr in ['name', 'path', 'kind']: + if getattr(self, attr) != getattr(other, attr): + return False + if self.is_file(): + if self.content != other.content: + return False + else: + # For DirNode's check without entering each dir + self_nodes_paths = list(sorted(n.path for n in self.nodes)) + other_nodes_paths = list(sorted(n.path for n in self.nodes)) + if self_nodes_paths != other_nodes_paths: + return False + return True + + def __nq__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return '<%s %r>' % (self.__class__.__name__, self.path) + + def __str__(self): + return self.__repr__() + + def __unicode__(self): + return self.name + + def get_parent_path(self): + """ + Returns node's parent path or empty string if node is root. + """ + if self.is_root(): + return '' + return posixpath.dirname(self.path.rstrip('/')) + '/' + + def is_file(self): + """ + Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False`` + otherwise. + """ + return self.kind == NodeKind.FILE + + def is_dir(self): + """ + Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False`` + otherwise. + """ + return self.kind == NodeKind.DIR + + def is_root(self): + """ + Returns ``True`` if node is a root node and ``False`` otherwise. + """ + return self.kind == NodeKind.DIR and self.path == '' + + @LazyProperty + def added(self): + return self.state is NodeState.ADDED + + @LazyProperty + def changed(self): + return self.state is NodeState.CHANGED + + @LazyProperty + def not_changed(self): + return self.state is NodeState.NOT_CHANGED + + @LazyProperty + def removed(self): + return self.state is NodeState.REMOVED + + +class FileNode(Node): + """ + Class representing file nodes. + + :attribute: path: path to the node, relative to repostiory's root + :attribute: content: if given arbitrary sets content of the file + :attribute: changeset: if given, first time content is accessed, callback + :attribute: mode: octal stat mode for a node. Default is 0100644. + """ + + def __init__(self, path, content=None, changeset=None, mode=None): + """ + Only one of ``content`` and ``changeset`` may be given. Passing both + would raise ``NodeError`` exception. + + :param path: relative path to the node + :param content: content may be passed to constructor + :param changeset: if given, will use it to lazily fetch content + :param mode: octal representation of ST_MODE (i.e. 0100644) + """ + + if content and changeset: + raise NodeError("Cannot use both content and changeset") + super(FileNode, self).__init__(path, kind=NodeKind.FILE) + self.changeset = changeset + self._content = content + self._mode = mode or 0100644 + + @LazyProperty + def mode(self): + """ + Returns lazily mode of the FileNode. If ``changeset`` is not set, would + use value given at initialization or 0100644 (default). + """ + if self.changeset: + mode = self.changeset.get_file_mode(self.path) + else: + mode = self._mode + return mode + + @property + def content(self): + """ + Returns lazily content of the FileNode. If possible, would try to + decode content from UTF-8. + """ + if self.changeset: + content = self.changeset.get_file_content(self.path) + else: + content = self._content + + if bool(content and '\0' in content): + return content + return safe_unicode(content) + + @LazyProperty + def size(self): + if self.changeset: + return self.changeset.get_file_size(self.path) + raise NodeError("Cannot retrieve size of the file without related " + "changeset attribute") + + @LazyProperty + def message(self): + if self.changeset: + return self.last_changeset.message + raise NodeError("Cannot retrieve message of the file without related " + "changeset attribute") + + @LazyProperty + def last_changeset(self): + if self.changeset: + return self.changeset.get_file_changeset(self.path) + raise NodeError("Cannot retrieve last changeset of the file without " + "related changeset attribute") + + def get_mimetype(self): + """ + Mimetype is calculated based on the file's content. If ``_mimetype`` + attribute is available, it will be returned (backends which store + mimetypes or can easily recognize them, should set this private + attribute to indicate that type should *NOT* be calculated). + """ + if hasattr(self, '_mimetype'): + if (isinstance(self._mimetype,(tuple,list,)) and + len(self._mimetype) == 2): + return self._mimetype + else: + raise NodeError('given _mimetype attribute must be an 2 ' + 'element list or tuple') + + mtype,encoding = mimetypes.guess_type(self.name) + + if mtype is None: + if self.is_binary: + mtype = 'application/octet-stream' + encoding = None + else: + mtype = 'text/plain' + encoding = None + return mtype,encoding + + @LazyProperty + def mimetype(self): + """ + Wrapper around full mimetype info. It returns only type of fetched + mimetype without the encoding part. use get_mimetype function to fetch + full set of (type,encoding) + """ + return self.get_mimetype()[0] + + @LazyProperty + def mimetype_main(self): + return self.mimetype.split('/')[0] + + @LazyProperty + def lexer(self): + """ + Returns pygment's lexer class. Would try to guess lexer taking file's + content, name and mimetype. + """ + try: + lexer = lexers.guess_lexer_for_filename(self.name, self.content) + except lexers.ClassNotFound: + lexer = lexers.TextLexer() + # returns first alias + return lexer + + @LazyProperty + def lexer_alias(self): + """ + Returns first alias of the lexer guessed for this file. + """ + return self.lexer.aliases[0] + + @LazyProperty + def history(self): + """ + Returns a list of changeset for this file in which the file was changed + """ + if self.changeset is None: + raise NodeError('Unable to get changeset for this FileNode') + return self.changeset.get_file_history(self.path) + + @LazyProperty + def annotate(self): + """ + Returns a list of three element tuples with lineno,changeset and line + """ + if self.changeset is None: + raise NodeError('Unable to get changeset for this FileNode') + return self.changeset.get_file_annotate(self.path) + + @LazyProperty + def state(self): + if not self.changeset: + raise NodeError("Cannot check state of the node if it's not " + "linked with changeset") + elif self.path in (node.path for node in self.changeset.added): + return NodeState.ADDED + elif self.path in (node.path for node in self.changeset.changed): + return NodeState.CHANGED + else: + return NodeState.NOT_CHANGED + + @property + def is_binary(self): + """ + Returns True if file has binary content. + """ + bin = '\0' in self.content + return bin + + @LazyProperty + def extension(self): + """Returns filenode extension""" + return self.name.split('.')[-1] + + def is_executable(self): + """ + Returns ``True`` if file has executable flag turned on. + """ + return bool(self.mode & stat.S_IXUSR) + + +class RemovedFileNode(FileNode): + """ + Dummy FileNode class - trying to access any public attribute except path, + name, kind or state (or methods/attributes checking those two) would raise + RemovedFileNodeError. + """ + ALLOWED_ATTRIBUTES = ['name', 'path', 'state', 'is_root', 'is_file', + 'is_dir', 'kind', 'added', 'changed', 'not_changed', 'removed'] + + def __init__(self, path): + """ + :param path: relative path to the node + """ + super(RemovedFileNode, self).__init__(path=path) + + def __getattribute__(self, attr): + if attr.startswith('_') or attr in RemovedFileNode.ALLOWED_ATTRIBUTES: + return super(RemovedFileNode, self).__getattribute__(attr) + raise RemovedFileNodeError("Cannot access attribute %s on " + "RemovedFileNode" % attr) + + @LazyProperty + def state(self): + return NodeState.REMOVED + + +class DirNode(Node): + """ + DirNode stores list of files and directories within this node. + Nodes may be used standalone but within repository context they + lazily fetch data within same repositorty's changeset. + """ + + def __init__(self, path, nodes=(), changeset=None): + """ + Only one of ``nodes`` and ``changeset`` may be given. Passing both + would raise ``NodeError`` exception. + + :param path: relative path to the node + :param nodes: content may be passed to constructor + :param changeset: if given, will use it to lazily fetch content + :param size: always 0 for ``DirNode`` + """ + if nodes and changeset: + raise NodeError("Cannot use both nodes and changeset") + super(DirNode, self).__init__(path, NodeKind.DIR) + self.changeset = changeset + self._nodes = nodes + + @LazyProperty + def content(self): + raise NodeError("%s represents a dir and has no ``content`` attribute" + % self) + + @LazyProperty + def nodes(self): + if self.changeset: + nodes = self.changeset.get_nodes(self.path) + else: + nodes = self._nodes + self._nodes_dict = dict((node.path, node) for node in nodes) + return sorted(nodes) + + @LazyProperty + def files(self): + return sorted((node for node in self.nodes if node.is_file())) + + @LazyProperty + def dirs(self): + return sorted((node for node in self.nodes if node.is_dir())) + + def __iter__(self): + for node in self.nodes: + yield node + + def get_node(self, path): + """ + Returns node from within this particular ``DirNode``, so it is now + allowed to fetch, i.e. node located at 'docs/api/index.rst' from node + 'docs'. In order to access deeper nodes one must fetch nodes between + them first - this would work:: + + docs = root.get_node('docs') + docs.get_node('api').get_node('index.rst') + + :param: path - relative to the current node + + .. note:: + To access lazily (as in example above) node have to be initialized + with related changeset object - without it node is out of + context and may know nothing about anything else than nearest + (located at same level) nodes. + """ + try: + path = path.rstrip('/') + if path == '': + raise NodeError("Cannot retrieve node without path") + self.nodes # access nodes first in order to set _nodes_dict + paths = path.split('/') + if len(paths) == 1: + if not self.is_root(): + path = '/'.join((self.path, paths[0])) + else: + path = paths[0] + return self._nodes_dict[path] + elif len(paths) > 1: + if self.changeset is None: + raise NodeError("Cannot access deeper " + "nodes without changeset") + else: + path1, path2 = paths[0], '/'.join(paths[1:]) + return self.get_node(path1).get_node(path2) + else: + raise KeyError + except KeyError: + raise NodeError("Node does not exist at %s" % path) + + @LazyProperty + def state(self): + raise NodeError("Cannot access state of DirNode") + + @LazyProperty + def size(self): + size = 0 + for root, dirs, files in self.changeset.walk(self.path): + for f in files: + size += f.size + + return size + + +class RootNode(DirNode): + """ + DirNode being the root node of the repository. + """ + + def __init__(self, nodes=(), changeset=None): + super(RootNode, self).__init__(path='', nodes=nodes, + changeset=changeset) + + def __repr__(self): + return '<%s>' % self.__class__.__name__ diff --git a/rhodecode/lib/vcs/utils/__init__.py b/rhodecode/lib/vcs/utils/__init__.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/__init__.py @@ -0,0 +1,133 @@ +""" +This module provides some useful tools for ``vcs`` like annotate/diff html +output. It also includes some internal helpers. +""" +import sys +import time +import datetime + + +def makedate(): + lt = time.localtime() + if lt[8] == 1 and time.daylight: + tz = time.altzone + else: + tz = time.timezone + return time.mktime(lt), tz + + +def date_fromtimestamp(unixts, tzoffset=0): + """ + Makes a local datetime object out of unix timestamp + + :param unixts: + :param tzoffset: + """ + + return datetime.datetime.fromtimestamp(float(unixts)) + + +def safe_unicode(str_, from_encoding='utf8'): + """ + safe unicode function. Does few trick to turn str_ into unicode + + In case of UnicodeDecode error we try to return it with encoding detected + by chardet library if it fails fallback to unicode with errors replaced + + :param str_: string to decode + :rtype: unicode + :returns: unicode object + """ + if isinstance(str_, unicode): + return str_ + + try: + return unicode(str_) + except UnicodeDecodeError: + pass + + try: + return unicode(str_, from_encoding) + except UnicodeDecodeError: + pass + + try: + import chardet + encoding = chardet.detect(str_)['encoding'] + if encoding is None: + raise Exception() + return str_.decode(encoding) + except (ImportError, UnicodeDecodeError, Exception): + return unicode(str_, from_encoding, 'replace') + + +def safe_str(unicode_, to_encoding='utf8'): + """ + safe str function. Does few trick to turn unicode_ into string + + In case of UnicodeEncodeError we try to return it with encoding detected + by chardet library if it fails fallback to string with errors replaced + + :param unicode_: unicode to encode + :rtype: str + :returns: str object + """ + + if isinstance(unicode_, str): + return unicode_ + + try: + return unicode_.encode(to_encoding) + except UnicodeEncodeError: + pass + + try: + import chardet + encoding = chardet.detect(unicode_)['encoding'] + print encoding + if encoding is None: + raise UnicodeEncodeError() + + return unicode_.encode(encoding) + except (ImportError, UnicodeEncodeError): + return unicode_.encode(to_encoding, 'replace') + + return safe_str + + +def author_email(author): + """ + returns email address of given author. + If any of <,> sign are found, it fallbacks to regex findall() + and returns first found result or empty string + + Regex taken from http://www.regular-expressions.info/email.html + """ + import re + r = author.find('>') + l = author.find('<') + + if l == -1 or r == -1: + # fallback to regex match of email out of a string + email_re = re.compile(r"""[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!""" + r"""#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z""" + r"""0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]""" + r"""*[a-z0-9])?""", re.IGNORECASE) + m = re.findall(email_re, author) + return m[0] if m else '' + + return author[l + 1:r].strip() + + +def author_name(author): + """ + get name of author, or else username. + It'll try to find an email in the author string and just cut it off + to get the username + """ + + if not '@' in author: + return author + else: + return author.replace(author_email(author), '').replace('<', '')\ + .replace('>', '').strip() diff --git a/rhodecode/lib/vcs/utils/annotate.py b/rhodecode/lib/vcs/utils/annotate.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/annotate.py @@ -0,0 +1,177 @@ +from rhodecode.lib.vcs.exceptions import VCSError +from rhodecode.lib.vcs.nodes import FileNode +from pygments.formatters import HtmlFormatter +from pygments import highlight + +import StringIO + + +def annotate_highlight(filenode, annotate_from_changeset_func=None, + order=None, headers=None, **options): + """ + Returns html portion containing annotated table with 3 columns: line + numbers, changeset information and pygmentized line of code. + + :param filenode: FileNode object + :param annotate_from_changeset_func: function taking changeset and + returning single annotate cell; needs break line at the end + :param order: ordered sequence of ``ls`` (line numbers column), + ``annotate`` (annotate column), ``code`` (code column); Default is + ``['ls', 'annotate', 'code']`` + :param headers: dictionary with headers (keys are whats in ``order`` + parameter) + """ + options['linenos'] = True + formatter = AnnotateHtmlFormatter(filenode=filenode, order=order, + headers=headers, + annotate_from_changeset_func=annotate_from_changeset_func, **options) + lexer = filenode.lexer + highlighted = highlight(filenode.content, lexer, formatter) + return highlighted + + +class AnnotateHtmlFormatter(HtmlFormatter): + + def __init__(self, filenode, annotate_from_changeset_func=None, + order=None, **options): + """ + If ``annotate_from_changeset_func`` is passed it should be a function + which returns string from the given changeset. For example, we may pass + following function as ``annotate_from_changeset_func``:: + + def changeset_to_anchor(changeset): + return '%s\n' %\ + (changeset.id, changeset.id) + + :param annotate_from_changeset_func: see above + :param order: (default: ``['ls', 'annotate', 'code']``); order of + columns; + :param options: standard pygment's HtmlFormatter options, there is + extra option tough, ``headers``. For instance we can pass:: + + formatter = AnnotateHtmlFormatter(filenode, headers={ + 'ls': '#', + 'annotate': 'Annotate', + 'code': 'Code', + }) + + """ + super(AnnotateHtmlFormatter, self).__init__(**options) + self.annotate_from_changeset_func = annotate_from_changeset_func + self.order = order or ('ls', 'annotate', 'code') + headers = options.pop('headers', None) + if headers and not ('ls' in headers and 'annotate' in headers and + 'code' in headers): + raise ValueError("If headers option dict is specified it must " + "all 'ls', 'annotate' and 'code' keys") + self.headers = headers + if isinstance(filenode, FileNode): + self.filenode = filenode + else: + raise VCSError("This formatter expect FileNode parameter, not %r" + % type(filenode)) + + def annotate_from_changeset(self, changeset): + """ + Returns full html line for single changeset per annotated line. + """ + if self.annotate_from_changeset_func: + return self.annotate_from_changeset_func(changeset) + else: + return ''.join((changeset.id, '\n')) + + def _wrap_tablelinenos(self, inner): + dummyoutfile = StringIO.StringIO() + lncount = 0 + for t, line in inner: + if t: + lncount += 1 + dummyoutfile.write(line) + + fl = self.linenostart + mw = len(str(lncount + fl - 1)) + sp = self.linenospecial + st = self.linenostep + la = self.lineanchors + aln = self.anchorlinenos + if sp: + lines = [] + + for i in range(fl, fl + lncount): + if i % st == 0: + if i % sp == 0: + if aln: + lines.append('' + '%*d' % + (la, i, mw, i)) + else: + lines.append('' + '%*d' % (mw, i)) + else: + if aln: + lines.append('' + '%*d' % (la, i, mw, i)) + else: + lines.append('%*d' % (mw, i)) + else: + lines.append('') + ls = '\n'.join(lines) + else: + lines = [] + for i in range(fl, fl + lncount): + if i % st == 0: + if aln: + lines.append('%*d' \ + % (la, i, mw, i)) + else: + lines.append('%*d' % (mw, i)) + else: + lines.append('') + ls = '\n'.join(lines) + + annotate_changesets = [tup[1] for tup in self.filenode.annotate] + # If pygments cropped last lines break we need do that too + ln_cs = len(annotate_changesets) + ln_ = len(ls.splitlines()) + if ln_cs > ln_: + annotate_changesets = annotate_changesets[:ln_ - ln_cs] + annotate = ''.join((self.annotate_from_changeset(changeset) + for changeset in annotate_changesets)) + # in case you wonder about the seemingly redundant
here: + # since the content in the other cell also is wrapped in a div, + # some browsers in some configurations seem to mess up the formatting. + ''' + yield 0, ('' % self.cssclass + + '' + + '
' +
+                  ls + '
') + yield 0, dummyoutfile.getvalue() + yield 0, '
' + + ''' + headers_row = [] + if self.headers: + headers_row = [''] + for key in self.order: + td = ''.join(('', self.headers[key], '')) + headers_row.append(td) + headers_row.append('') + + body_row_start = [''] + for key in self.order: + if key == 'ls': + body_row_start.append( + '
' +
+                    ls + '
') + elif key == 'annotate': + body_row_start.append( + '
' +
+                    annotate + '
') + elif key == 'code': + body_row_start.append('') + yield 0, ('' % self.cssclass + + ''.join(headers_row) + + ''.join(body_row_start) + ) + yield 0, dummyoutfile.getvalue() + yield 0, '
' diff --git a/rhodecode/lib/vcs/utils/archivers.py b/rhodecode/lib/vcs/utils/archivers.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/archivers.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" + vcs.utils.archivers + ~~~~~~~~~~~~~~~~~~~ + + set of archiver functions for creating archives from repository content + + :created_on: Jan 21, 2011 + :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak. +""" + + +class BaseArchiver(object): + + def __init__(self): + self.archive_file = self._get_archive_file() + + def addfile(self): + """ + Adds a file to archive container + """ + pass + + def close(self): + """ + Closes and finalizes operation of archive container object + """ + self.archive_file.close() + + def _get_archive_file(self): + """ + Returns container for specific archive + """ + raise NotImplementedError() + + +class TarArchiver(BaseArchiver): + pass + + +class Tbz2Archiver(BaseArchiver): + pass + + +class TgzArchiver(BaseArchiver): + pass + + +class ZipArchiver(BaseArchiver): + pass + + +def get_archiver(self, kind): + """ + Returns instance of archiver class specific to given kind + + :param kind: archive kind + """ + + archivers = { + 'tar': TarArchiver, + 'tbz2': Tbz2Archiver, + 'tgz': TgzArchiver, + 'zip': ZipArchiver, + } + + return archivers[kind]() diff --git a/rhodecode/lib/vcs/utils/baseui_config.py b/rhodecode/lib/vcs/utils/baseui_config.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/baseui_config.py @@ -0,0 +1,47 @@ +from mercurial import ui, config + + +def make_ui(self, path='hgwebdir.config'): + """ + A funcion that will read python rc files and make an ui from read options + + :param path: path to mercurial config file + """ + #propagated from mercurial documentation + sections = [ + 'alias', + 'auth', + 'decode/encode', + 'defaults', + 'diff', + 'email', + 'extensions', + 'format', + 'merge-patterns', + 'merge-tools', + 'hooks', + 'http_proxy', + 'smtp', + 'patch', + 'paths', + 'profiling', + 'server', + 'trusted', + 'ui', + 'web', + ] + + repos = path + baseui = ui.ui() + cfg = config.config() + cfg.read(repos) + self.paths = cfg.items('paths') + self.base_path = self.paths[0][1].replace('*', '') + self.check_repo_dir(self.paths) + self.set_statics(cfg) + + for section in sections: + for k, v in cfg.items(section): + baseui.setconfig(section, k, v) + + return baseui diff --git a/rhodecode/lib/vcs/utils/compat.py b/rhodecode/lib/vcs/utils/compat.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/compat.py @@ -0,0 +1,13 @@ +""" +Various utilities to work with Python < 2.7. + +Those utilities may be deleted once ``vcs`` stops support for older Python +versions. +""" +import sys + + +if sys.version_info >= (2, 7): + unittest = __import__('unittest') +else: + unittest = __import__('unittest2') diff --git a/rhodecode/lib/vcs/utils/diffs.py b/rhodecode/lib/vcs/utils/diffs.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/diffs.py @@ -0,0 +1,460 @@ +# -*- coding: utf-8 -*- +# original copyright: 2007-2008 by Armin Ronacher +# licensed under the BSD license. + +import re +import difflib +import logging + +from difflib import unified_diff +from itertools import tee, imap + +from mercurial.match import match + +from rhodecode.lib.vcs.exceptions import VCSError +from rhodecode.lib.vcs.nodes import FileNode, NodeError + + +def get_udiff(filenode_old, filenode_new,show_whitespace=True): + """ + Returns unified diff between given ``filenode_old`` and ``filenode_new``. + """ + try: + filenode_old_date = filenode_old.last_changeset.date + except NodeError: + filenode_old_date = None + + try: + filenode_new_date = filenode_new.last_changeset.date + except NodeError: + filenode_new_date = None + + for filenode in (filenode_old, filenode_new): + if not isinstance(filenode, FileNode): + raise VCSError("Given object should be FileNode object, not %s" + % filenode.__class__) + + if filenode_old_date and filenode_new_date: + if not filenode_old_date < filenode_new_date: + logging.debug("Generating udiff for filenodes with not increasing " + "dates") + + vcs_udiff = unified_diff(filenode_old.content.splitlines(True), + filenode_new.content.splitlines(True), + filenode_old.name, + filenode_new.name, + filenode_old_date, + filenode_old_date) + return vcs_udiff + + +def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True): + """ + Returns git style diff between given ``filenode_old`` and ``filenode_new``. + + :param ignore_whitespace: ignore whitespaces in diff + """ + + for filenode in (filenode_old, filenode_new): + if not isinstance(filenode, FileNode): + raise VCSError("Given object should be FileNode object, not %s" + % filenode.__class__) + + old_raw_id = getattr(filenode_old.changeset, 'raw_id', '0' * 40) + new_raw_id = getattr(filenode_new.changeset, 'raw_id', '0' * 40) + + repo = filenode_new.changeset.repository + vcs_gitdiff = repo._get_diff(old_raw_id, new_raw_id, filenode_new.path, + ignore_whitespace) + + return vcs_gitdiff + + +class DiffProcessor(object): + """ + Give it a unified diff and it returns a list of the files that were + mentioned in the diff together with a dict of meta information that + can be used to render it in a HTML template. + """ + _chunk_re = re.compile(r'@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)') + + def __init__(self, diff, differ='diff', format='udiff'): + """ + :param diff: a text in diff format or generator + :param format: format of diff passed, `udiff` or `gitdiff` + """ + if isinstance(diff, basestring): + diff = [diff] + + self.__udiff = diff + self.__format = format + self.adds = 0 + self.removes = 0 + + if isinstance(self.__udiff, basestring): + self.lines = iter(self.__udiff.splitlines(1)) + + elif self.__format == 'gitdiff': + udiff_copy = self.copy_iterator() + self.lines = imap(self.escaper, self._parse_gitdiff(udiff_copy)) + else: + udiff_copy = self.copy_iterator() + self.lines = imap(self.escaper, udiff_copy) + + # Select a differ. + if differ == 'difflib': + self.differ = self._highlight_line_difflib + else: + self.differ = self._highlight_line_udiff + + def escaper(self, string): + return string.replace('<', '<').replace('>', '>') + + def copy_iterator(self): + """ + make a fresh copy of generator, we should not iterate thru + an original as it's needed for repeating operations on + this instance of DiffProcessor + """ + self.__udiff, iterator_copy = tee(self.__udiff) + return iterator_copy + + def _extract_rev(self, line1, line2): + """ + Extract the filename and revision hint from a line. + """ + + try: + if line1.startswith('--- ') and line2.startswith('+++ '): + l1 = line1[4:].split(None, 1) + old_filename = l1[0].lstrip('a/') if len(l1) >= 1 else None + old_rev = l1[1] if len(l1) == 2 else 'old' + + l2 = line2[4:].split(None, 1) + new_filename = l2[0].lstrip('b/') if len(l1) >= 1 else None + new_rev = l2[1] if len(l2) == 2 else 'new' + + filename = old_filename if (old_filename != + 'dev/null') else new_filename + + return filename, new_rev, old_rev + except (ValueError, IndexError): + pass + + return None, None, None + + def _parse_gitdiff(self, diffiterator): + def line_decoder(l): + if l.startswith('+') and not l.startswith('+++'): + self.adds += 1 + elif l.startswith('-') and not l.startswith('---'): + self.removes += 1 + return l.decode('utf8', 'replace') + + output = list(diffiterator) + size = len(output) + + if size == 2: + l = [] + l.extend([output[0]]) + l.extend(output[1].splitlines(1)) + return map(line_decoder, l) + elif size == 1: + return map(line_decoder, output[0].splitlines(1)) + elif size == 0: + return [] + + raise Exception('wrong size of diff %s' % size) + + def _highlight_line_difflib(self, line, next): + """ + Highlight inline changes in both lines. + """ + + if line['action'] == 'del': + old, new = line, next + else: + old, new = next, line + + oldwords = re.split(r'(\W)', old['line']) + newwords = re.split(r'(\W)', new['line']) + + sequence = difflib.SequenceMatcher(None, oldwords, newwords) + + oldfragments, newfragments = [], [] + for tag, i1, i2, j1, j2 in sequence.get_opcodes(): + oldfrag = ''.join(oldwords[i1:i2]) + newfrag = ''.join(newwords[j1:j2]) + if tag != 'equal': + if oldfrag: + oldfrag = '%s' % oldfrag + if newfrag: + newfrag = '%s' % newfrag + oldfragments.append(oldfrag) + newfragments.append(newfrag) + + old['line'] = "".join(oldfragments) + new['line'] = "".join(newfragments) + + def _highlight_line_udiff(self, line, next): + """ + Highlight inline changes in both lines. + """ + start = 0 + limit = min(len(line['line']), len(next['line'])) + while start < limit and line['line'][start] == next['line'][start]: + start += 1 + end = -1 + limit -= start + while -end <= limit and line['line'][end] == next['line'][end]: + end -= 1 + end += 1 + if start or end: + def do(l): + last = end + len(l['line']) + if l['action'] == 'add': + tag = 'ins' + else: + tag = 'del' + l['line'] = '%s<%s>%s%s' % ( + l['line'][:start], + tag, + l['line'][start:last], + tag, + l['line'][last:] + ) + do(line) + do(next) + + def _parse_udiff(self): + """ + Parse the diff an return data for the template. + """ + lineiter = self.lines + files = [] + try: + line = lineiter.next() + # skip first context + skipfirst = True + while 1: + # continue until we found the old file + if not line.startswith('--- '): + line = lineiter.next() + continue + + chunks = [] + filename, old_rev, new_rev = \ + self._extract_rev(line, lineiter.next()) + files.append({ + 'filename': filename, + 'old_revision': old_rev, + 'new_revision': new_rev, + 'chunks': chunks + }) + + line = lineiter.next() + while line: + match = self._chunk_re.match(line) + if not match: + break + + lines = [] + chunks.append(lines) + + old_line, old_end, new_line, new_end = \ + [int(x or 1) for x in match.groups()[:-1]] + old_line -= 1 + new_line -= 1 + context = len(match.groups()) == 5 + old_end += old_line + new_end += new_line + + if context: + if not skipfirst: + lines.append({ + 'old_lineno': '...', + 'new_lineno': '...', + 'action': 'context', + 'line': line, + }) + else: + skipfirst = False + + line = lineiter.next() + while old_line < old_end or new_line < new_end: + if line: + command, line = line[0], line[1:] + else: + command = ' ' + affects_old = affects_new = False + + # ignore those if we don't expect them + if command in '#@': + continue + elif command == '+': + affects_new = True + action = 'add' + elif command == '-': + affects_old = True + action = 'del' + else: + affects_old = affects_new = True + action = 'unmod' + + old_line += affects_old + new_line += affects_new + lines.append({ + 'old_lineno': affects_old and old_line or '', + 'new_lineno': affects_new and new_line or '', + 'action': action, + 'line': line + }) + line = lineiter.next() + + except StopIteration: + pass + + # highlight inline changes + for file in files: + for chunk in chunks: + lineiter = iter(chunk) + #first = True + try: + while 1: + line = lineiter.next() + if line['action'] != 'unmod': + nextline = lineiter.next() + if nextline['action'] == 'unmod' or \ + nextline['action'] == line['action']: + continue + self.differ(line, nextline) + except StopIteration: + pass + + return files + + def prepare(self): + """ + Prepare the passed udiff for HTML rendering. It'l return a list + of dicts + """ + return self._parse_udiff() + + def _safe_id(self, idstring): + """Make a string safe for including in an id attribute. + + The HTML spec says that id attributes 'must begin with + a letter ([A-Za-z]) and may be followed by any number + of letters, digits ([0-9]), hyphens ("-"), underscores + ("_"), colons (":"), and periods (".")'. These regexps + are slightly over-zealous, in that they remove colons + and periods unnecessarily. + + Whitespace is transformed into underscores, and then + anything which is not a hyphen or a character that + matches \w (alphanumerics and underscore) is removed. + + """ + # Transform all whitespace to underscore + idstring = re.sub(r'\s', "_", '%s' % idstring) + # Remove everything that is not a hyphen or a member of \w + idstring = re.sub(r'(?!-)\W', "", idstring).lower() + return idstring + + def raw_diff(self): + """ + Returns raw string as udiff + """ + udiff_copy = self.copy_iterator() + if self.__format == 'gitdiff': + udiff_copy = self._parse_gitdiff(udiff_copy) + return u''.join(udiff_copy) + + def as_html(self, table_class='code-difftable', line_class='line', + new_lineno_class='lineno old', old_lineno_class='lineno new', + code_class='code'): + """ + Return udiff as html table with customized css classes + """ + def _link_to_if(condition, label, url): + """ + Generates a link if condition is meet or just the label if not. + """ + + if condition: + return '''%(label)s''' % {'url': url, + 'label': label} + else: + return label + diff_lines = self.prepare() + _html_empty = True + _html = [] + _html.append('''\n''' \ + % {'table_class': table_class}) + for diff in diff_lines: + for line in diff['chunks']: + _html_empty = False + for change in line: + _html.append('''\n''' \ + % {'line_class': line_class, + 'action': change['action']}) + anchor_old_id = '' + anchor_new_id = '' + anchor_old = "%(filename)s_o%(oldline_no)s" % \ + {'filename': self._safe_id(diff['filename']), + 'oldline_no': change['old_lineno']} + anchor_new = "%(filename)s_n%(oldline_no)s" % \ + {'filename': self._safe_id(diff['filename']), + 'oldline_no': change['new_lineno']} + cond_old = change['old_lineno'] != '...' and \ + change['old_lineno'] + cond_new = change['new_lineno'] != '...' and \ + change['new_lineno'] + if cond_old: + anchor_old_id = 'id="%s"' % anchor_old + if cond_new: + anchor_new_id = 'id="%s"' % anchor_new + ########################################################### + # OLD LINE NUMBER + ########################################################### + _html.append('''\t\n''') + ########################################################### + # NEW LINE NUMBER + ########################################################### + + _html.append('''\t\n''') + ########################################################### + # CODE + ########################################################### + _html.append('''\t''') + _html.append('''\n\n''') + _html.append('''
''' \ + % {'a_id': anchor_old_id, + 'old_lineno_cls': old_lineno_class}) + + _html.append('''
%(link)s
''' \ + % {'link': + _link_to_if(cond_old, change['old_lineno'], '#%s' \ + % anchor_old)}) + _html.append('''		''' \ + % {'a_id': anchor_new_id, + 'new_lineno_cls': new_lineno_class}) + + _html.append('''
%(link)s
''' \ + % {'link': + _link_to_if(cond_new, change['new_lineno'], '#%s' \ + % anchor_new)}) + _html.append('''		''' \ + % {'code_class': code_class}) + _html.append('''\n\t\t
%(code)s
\n''' \ + % {'code': change['line']}) + _html.append('''\t
''') + if _html_empty: + return None + return ''.join(_html) + + def stat(self): + """ + Returns tuple of adde,and removed lines for this instance + """ + return self.adds, self.removes diff --git a/rhodecode/lib/vcs/utils/fakemod.py b/rhodecode/lib/vcs/utils/fakemod.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/fakemod.py @@ -0,0 +1,13 @@ +import imp + + +def create_module(name, path): + """ + Returns module created *on the fly*. Returned module would have name same + as given ``name`` and would contain code read from file at the given + ``path`` (it may also be a zip or package containing *__main__* module). + """ + module = imp.new_module(name) + module.__file__ = path + execfile(path, module.__dict__) + return module diff --git a/rhodecode/lib/vcs/utils/filesize.py b/rhodecode/lib/vcs/utils/filesize.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/filesize.py @@ -0,0 +1,28 @@ +def filesizeformat(bytes, sep=' '): + """ + Formats the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB, + 102 B, 2.3 GB etc). + + Grabbed from Django (http://www.djangoproject.com), slightly modified. + + :param bytes: size in bytes (as integer) + :param sep: string separator between number and abbreviation + """ + try: + bytes = float(bytes) + except (TypeError, ValueError, UnicodeDecodeError): + return '0%sB' % sep + + if bytes < 1024: + size = bytes + template = '%.0f%sB' + elif bytes < 1024 * 1024: + size = bytes / 1024 + template = '%.0f%sKB' + elif bytes < 1024 * 1024 * 1024: + size = bytes / 1024 / 1024 + template = '%.1f%sMB' + else: + size = bytes / 1024 / 1024 / 1024 + template = '%.2f%sGB' + return template % (size, sep) diff --git a/rhodecode/lib/vcs/utils/helpers.py b/rhodecode/lib/vcs/utils/helpers.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/helpers.py @@ -0,0 +1,252 @@ +""" +Utitlites aimed to help achieve mostly basic tasks. +""" +from __future__ import division + +import re +import time +import datetime +import os.path +from subprocess import Popen, PIPE +from rhodecode.lib.vcs.exceptions import VCSError +from rhodecode.lib.vcs.exceptions import RepositoryError +from rhodecode.lib.vcs.utils.paths import abspath + +ALIASES = ['hg', 'git'] + + +def get_scm(path, search_recursively=False, explicit_alias=None): + """ + Returns one of alias from ``ALIASES`` (in order of precedence same as + shortcuts given in ``ALIASES``) and top working dir path for the given + argument. If no scm-specific directory is found or more than one scm is + found at that directory, ``VCSError`` is raised. + + :param search_recursively: if set to ``True``, this function would try to + move up to parent directory every time no scm is recognized for the + currently checked path. Default: ``False``. + :param explicit_alias: can be one of available backend aliases, when given + it will return given explicit alias in repositories under more than one + version control, if explicit_alias is different than found it will raise + VCSError + """ + if not os.path.isdir(path): + raise VCSError("Given path %s is not a directory" % path) + + def get_scms(path): + return [(scm, path) for scm in get_scms_for_path(path)] + + found_scms = get_scms(path) + while not found_scms and search_recursively: + newpath = abspath(path, '..') + if newpath == path: + break + path = newpath + found_scms = get_scms(path) + + if len(found_scms) > 1: + for scm in found_scms: + if scm[0] == explicit_alias: + return scm + raise VCSError('More than one [%s] scm found at given path %s' + % (','.join((x[0] for x in found_scms)), path)) + + if len(found_scms) is 0: + raise VCSError('No scm found at given path %s' % path) + + return found_scms[0] + + +def get_scms_for_path(path): + """ + Returns all scm's found at the given path. If no scm is recognized + - empty list is returned. + + :param path: path to directory which should be checked. May be callable. + + :raises VCSError: if given ``path`` is not a directory + """ + from rhodecode.lib.vcs.backends import get_backend + if hasattr(path, '__call__'): + path = path() + if not os.path.isdir(path): + raise VCSError("Given path %r is not a directory" % path) + + result = [] + for key in ALIASES: + dirname = os.path.join(path, '.' + key) + if os.path.isdir(dirname): + result.append(key) + continue + # We still need to check if it's not bare repository as + # bare repos don't have working directories + try: + get_backend(key)(path) + result.append(key) + continue + except RepositoryError: + # Wrong backend + pass + except VCSError: + # No backend at all + pass + return result + + +def get_repo_paths(path): + """ + Returns path's subdirectories which seems to be a repository. + """ + repo_paths = [] + dirnames = (os.path.abspath(dirname) for dirname in os.listdir(path)) + for dirname in dirnames: + try: + get_scm(dirname) + repo_paths.append(dirname) + except VCSError: + pass + return repo_paths + + +def run_command(cmd, *args): + """ + Runs command on the system with given ``args``. + """ + command = ' '.join((cmd, args)) + p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + return p.retcode, stdout, stderr + + +def get_highlighted_code(name, code, type='terminal'): + """ + If pygments are available on the system + then returned output is colored. Otherwise + unchanged content is returned. + """ + import logging + try: + import pygments + pygments + except ImportError: + return code + from pygments import highlight + from pygments.lexers import guess_lexer_for_filename, ClassNotFound + from pygments.formatters import TerminalFormatter + + try: + lexer = guess_lexer_for_filename(name, code) + formatter = TerminalFormatter() + content = highlight(code, lexer, formatter) + except ClassNotFound: + logging.debug("Couldn't guess Lexer, will not use pygments.") + content = code + return content + +def parse_changesets(text): + """ + Returns dictionary with *start*, *main* and *end* ids. + + Examples:: + + >>> parse_changesets('aaabbb') + {'start': None, 'main': 'aaabbb', 'end': None} + >>> parse_changesets('aaabbb..cccddd') + {'start': 'aaabbb', 'main': None, 'end': 'cccddd'} + + """ + text = text.strip() + CID_RE = r'[a-zA-Z0-9]+' + if not '..' in text: + m = re.match(r'^(?P%s)$' % CID_RE, text) + if m: + return { + 'start': None, + 'main': text, + 'end': None, + } + else: + RE = r'^(?P%s)?\.{2,3}(?P%s)?$' % (CID_RE, CID_RE) + m = re.match(RE, text) + if m: + result = m.groupdict() + result['main'] = None + return result + raise ValueError("IDs not recognized") + +def parse_datetime(text): + """ + Parses given text and returns ``datetime.datetime`` instance or raises + ``ValueError``. + + :param text: string of desired date/datetime or something more verbose, + like *yesterday*, *2weeks 3days*, etc. + """ + + text = text.strip().lower() + + INPUT_FORMATS = ( + '%Y-%m-%d %H:%M:%S', + '%Y-%m-%d %H:%M', + '%Y-%m-%d', + '%m/%d/%Y %H:%M:%S', + '%m/%d/%Y %H:%M', + '%m/%d/%Y', + '%m/%d/%y %H:%M:%S', + '%m/%d/%y %H:%M', + '%m/%d/%y', + ) + for format in INPUT_FORMATS: + try: + return datetime.datetime(*time.strptime(text, format)[:6]) + except ValueError: + pass + + # Try descriptive texts + if text == 'tomorrow': + future = datetime.datetime.now() + datetime.timedelta(days=1) + args = future.timetuple()[:3] + (23, 59, 59) + return datetime.datetime(*args) + elif text == 'today': + return datetime.datetime(*datetime.datetime.today().timetuple()[:3]) + elif text == 'now': + return datetime.datetime.now() + elif text == 'yesterday': + past = datetime.datetime.now() - datetime.timedelta(days=1) + return datetime.datetime(*past.timetuple()[:3]) + else: + days = 0 + matched = re.match( + r'^((?P\d+) ?w(eeks?)?)? ?((?P\d+) ?d(ays?)?)?$', text) + if matched: + groupdict = matched.groupdict() + if groupdict['days']: + days += int(matched.groupdict()['days']) + if groupdict['weeks']: + days += int(matched.groupdict()['weeks']) * 7 + past = datetime.datetime.now() - datetime.timedelta(days=days) + return datetime.datetime(*past.timetuple()[:3]) + + raise ValueError('Wrong date: "%s"' % text) + + +def get_dict_for_attrs(obj, attrs): + """ + Returns dictionary for each attribute from given ``obj``. + """ + data = {} + for attr in attrs: + data[attr] = getattr(obj, attr) + return data + + +def get_total_seconds(timedelta): + """ + Backported for Python 2.5. + + See http://docs.python.org/library/datetime.html. + """ + return ((timedelta.microseconds + ( + timedelta.seconds + + timedelta.days * 24 * 60 * 60 + ) * 10**6) / 10**6) diff --git a/rhodecode/lib/vcs/utils/hgcompat.py b/rhodecode/lib/vcs/utils/hgcompat.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/hgcompat.py @@ -0,0 +1,12 @@ +"""Mercurial libs compatibility + +""" +from mercurial import archival, merge as hg_merge, patch, ui +from mercurial.commands import clone, nullid, pull +from mercurial.context import memctx, memfilectx +from mercurial.error import RepoError, RepoLookupError, Abort +from mercurial.hgweb.common import get_contact +from mercurial.localrepo import localrepository +from mercurial.match import match +from mercurial.mdiff import diffopts +from mercurial.node import hex diff --git a/rhodecode/lib/vcs/utils/imports.py b/rhodecode/lib/vcs/utils/imports.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/imports.py @@ -0,0 +1,27 @@ +from rhodecode.lib.vcs.exceptions import VCSError + + +def import_class(class_path): + """ + Returns class from the given path. + + For example, in order to get class located at + ``vcs.backends.hg.MercurialRepository``: + + try: + hgrepo = import_class('vcs.backends.hg.MercurialRepository') + except VCSError: + # hadle error + """ + splitted = class_path.split('.') + mod_path = '.'.join(splitted[:-1]) + class_name = splitted[-1] + try: + class_mod = __import__(mod_path, {}, {}, [class_name]) + except ImportError, err: + msg = "There was problem while trying to import backend class. "\ + "Original error was:\n%s" % err + raise VCSError(msg) + cls = getattr(class_mod, class_name) + + return cls diff --git a/rhodecode/lib/vcs/utils/lazy.py b/rhodecode/lib/vcs/utils/lazy.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/lazy.py @@ -0,0 +1,27 @@ +class LazyProperty(object): + """ + Decorator for easier creation of ``property`` from potentially expensive to + calculate attribute of the class. + + Usage:: + + class Foo(object): + @LazyProperty + def bar(self): + print 'Calculating self._bar' + return 42 + + Taken from http://blog.pythonisito.com/2008/08/lazy-descriptors.html and + used widely. + """ + + def __init__(self, func): + self._func = func + self.__name__ = func.__name__ + self.__doc__ = func.__doc__ + + def __get__(self, obj, klass=None): + if obj is None: + return None + result = obj.__dict__[self.__name__] = self._func(obj) + return result diff --git a/rhodecode/lib/vcs/utils/lockfiles.py b/rhodecode/lib/vcs/utils/lockfiles.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/lockfiles.py @@ -0,0 +1,72 @@ +import os + + +class LockFile(object): + """Provides methods to obtain, check for, and release a file based lock which + should be used to handle concurrent access to the same file. + + As we are a utility class to be derived from, we only use protected methods. + + Locks will automatically be released on destruction""" + __slots__ = ("_file_path", "_owns_lock") + + def __init__(self, file_path): + self._file_path = file_path + self._owns_lock = False + + def __del__(self): + self._release_lock() + + def _lock_file_path(self): + """:return: Path to lockfile""" + return "%s.lock" % (self._file_path) + + def _has_lock(self): + """:return: True if we have a lock and if the lockfile still exists + :raise AssertionError: if our lock-file does not exist""" + if not self._owns_lock: + return False + + return True + + def _obtain_lock_or_raise(self): + """Create a lock file as flag for other instances, mark our instance as lock-holder + + :raise IOError: if a lock was already present or a lock file could not be written""" + if self._has_lock(): + return + lock_file = self._lock_file_path() + if os.path.isfile(lock_file): + raise IOError("Lock for file %r did already exist, delete %r in case the lock is illegal" % (self._file_path, lock_file)) + + try: + fd = os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0) + os.close(fd) + except OSError,e: + raise IOError(str(e)) + + self._owns_lock = True + + def _obtain_lock(self): + """The default implementation will raise if a lock cannot be obtained. + Subclasses may override this method to provide a different implementation""" + return self._obtain_lock_or_raise() + + def _release_lock(self): + """Release our lock if we have one""" + if not self._has_lock(): + return + + # if someone removed our file beforhand, lets just flag this issue + # instead of failing, to make it more usable. + lfp = self._lock_file_path() + try: + # on bloody windows, the file needs write permissions to be removable. + # Why ... + if os.name == 'nt': + os.chmod(lfp, 0777) + # END handle win32 + os.remove(lfp) + except OSError: + pass + self._owns_lock = False diff --git a/rhodecode/lib/vcs/utils/ordered_dict.py b/rhodecode/lib/vcs/utils/ordered_dict.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/ordered_dict.py @@ -0,0 +1,102 @@ +"""Ordered dict implementation""" +from UserDict import DictMixin + + +class OrderedDict(dict, DictMixin): + + def __init__(self, *args, **kwds): + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__end + except AttributeError: + self.clear() + self.update(*args, **kwds) + + def clear(self): + self.__end = end = [] + end += [None, end, end] # sentinel node for doubly linked list + self.__map = {} # key --> [key, prev, next] + dict.clear(self) + + def __setitem__(self, key, value): + if key not in self: + end = self.__end + curr = end[1] + curr[2] = end[1] = self.__map[key] = [key, curr, end] + dict.__setitem__(self, key, value) + + def __delitem__(self, key): + dict.__delitem__(self, key) + key, prev, next = self.__map.pop(key) + prev[2] = next + next[1] = prev + + def __iter__(self): + end = self.__end + curr = end[2] + while curr is not end: + yield curr[0] + curr = curr[2] + + def __reversed__(self): + end = self.__end + curr = end[1] + while curr is not end: + yield curr[0] + curr = curr[1] + + def popitem(self, last=True): + if not self: + raise KeyError('dictionary is empty') + if last: + key = reversed(self).next() + else: + key = iter(self).next() + value = self.pop(key) + return key, value + + def __reduce__(self): + items = [[k, self[k]] for k in self] + tmp = self.__map, self.__end + del self.__map, self.__end + inst_dict = vars(self).copy() + self.__map, self.__end = tmp + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def keys(self): + return list(self) + + setdefault = DictMixin.setdefault + update = DictMixin.update + pop = DictMixin.pop + values = DictMixin.values + items = DictMixin.items + iterkeys = DictMixin.iterkeys + itervalues = DictMixin.itervalues + iteritems = DictMixin.iteritems + + def __repr__(self): + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + + def copy(self): + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + if isinstance(other, OrderedDict): + return len(self) == len(other) and self.items() == other.items() + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other diff --git a/rhodecode/lib/vcs/utils/paths.py b/rhodecode/lib/vcs/utils/paths.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/paths.py @@ -0,0 +1,36 @@ +import os + +abspath = lambda * p: os.path.abspath(os.path.join(*p)) + + +def get_dirs_for_path(*paths): + """ + Returns list of directories, including intermediate. + """ + for path in paths: + head = path + while head: + head, tail = os.path.split(head) + if head: + yield head + else: + # We don't need to yield empty path + break + + +def get_dir_size(path): + root_path = path + size = 0 + for path, dirs, files in os.walk(root_path): + for f in files: + try: + size += os.path.getsize(os.path.join(path, f)) + except OSError: + pass + return size + +def get_user_home(): + """ + Returns home path of the user. + """ + return os.getenv('HOME', os.getenv('USERPROFILE')) diff --git a/rhodecode/lib/vcs/utils/progressbar.py b/rhodecode/lib/vcs/utils/progressbar.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/progressbar.py @@ -0,0 +1,419 @@ +# encoding: UTF-8 +import sys +import datetime +from string import Template +from rhodecode.lib.vcs.utils.filesize import filesizeformat +from rhodecode.lib.vcs.utils.helpers import get_total_seconds + + +class ProgressBarError(Exception): + pass + +class AlreadyFinishedError(ProgressBarError): + pass + + +class ProgressBar(object): + + default_elements = ['percentage', 'bar', 'steps'] + + def __init__(self, steps=100, stream=None, elements=None): + self.step = 0 + self.steps = steps + self.stream = stream or sys.stderr + self.bar_char = '=' + self.width = 50 + self.separator = ' | ' + self.elements = elements or self.default_elements + self.started = None + self.finished = False + self.steps_label = 'Step' + self.time_label = 'Time' + self.eta_label = 'ETA' + self.speed_label = 'Speed' + self.transfer_label = 'Transfer' + + def __str__(self): + return self.get_line() + + def __iter__(self): + start = self.step + end = self.steps + 1 + for x in xrange(start, end): + self.render(x) + yield x + + def get_separator(self): + return self.separator + + def get_bar_char(self): + return self.bar_char + + def get_bar(self): + char = self.get_bar_char() + perc = self.get_percentage() + length = int(self.width * perc / 100) + bar = char * length + bar = bar.ljust(self.width) + return bar + + def get_elements(self): + return self.elements + + def get_template(self): + separator = self.get_separator() + elements = self.get_elements() + return Template(separator.join((('$%s' % e) for e in elements))) + + def get_total_time(self, current_time=None): + if current_time is None: + current_time = datetime.datetime.now() + if not self.started: + return datetime.timedelta() + return current_time - self.started + + def get_rendered_total_time(self): + delta = self.get_total_time() + if not delta: + ttime = '-' + else: + ttime = str(delta) + return '%s %s' % (self.time_label, ttime) + + def get_eta(self, current_time=None): + if current_time is None: + current_time = datetime.datetime.now() + if self.step == 0: + return datetime.timedelta() + total_seconds = get_total_seconds(self.get_total_time()) + eta_seconds = total_seconds * self.steps / self.step - total_seconds + return datetime.timedelta(seconds=int(eta_seconds)) + + def get_rendered_eta(self): + eta = self.get_eta() + if not eta: + eta = '--:--:--' + else: + eta = str(eta).rjust(8) + return '%s: %s' % (self.eta_label, eta) + + def get_percentage(self): + return float(self.step) / self.steps * 100 + + def get_rendered_percentage(self): + perc = self.get_percentage() + return ('%s%%' % (int(perc))).rjust(5) + + def get_rendered_steps(self): + return '%s: %s/%s' % (self.steps_label, self.step, self.steps) + + def get_rendered_speed(self, step=None, total_seconds=None): + if step is None: + step = self.step + if total_seconds is None: + total_seconds = get_total_seconds(self.get_total_time()) + if step <= 0 or total_seconds <= 0: + speed = '-' + else: + speed = filesizeformat(float(step) / total_seconds) + return '%s: %s/s' % (self.speed_label, speed) + + def get_rendered_transfer(self, step=None, steps=None): + if step is None: + step = self.step + if steps is None: + steps = self.steps + + if steps <= 0: + return '%s: -' % self.transfer_label + total = filesizeformat(float(steps)) + if step <= 0: + transferred = '-' + else: + transferred = filesizeformat(float(step)) + return '%s: %s / %s' % (self.transfer_label, transferred, total) + + def get_context(self): + return { + 'percentage': self.get_rendered_percentage(), + 'bar': self.get_bar(), + 'steps': self.get_rendered_steps(), + 'time': self.get_rendered_total_time(), + 'eta': self.get_rendered_eta(), + 'speed': self.get_rendered_speed(), + 'transfer': self.get_rendered_transfer(), + } + + def get_line(self): + template = self.get_template() + context = self.get_context() + return template.safe_substitute(**context) + + def write(self, data): + self.stream.write(data) + + def render(self, step): + if not self.started: + self.started = datetime.datetime.now() + if self.finished: + raise AlreadyFinishedError + self.step = step + self.write('\r%s' % self) + if step == self.steps: + self.finished = True + if step == self.steps: + self.write('\n') + + +""" +termcolors.py + +Grabbed from Django (http://www.djangoproject.com) +""" + +color_names = ('black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white') +foreground = dict([(color_names[x], '3%s' % x) for x in range(8)]) +background = dict([(color_names[x], '4%s' % x) for x in range(8)]) + +RESET = '0' +opt_dict = {'bold': '1', 'underscore': '4', 'blink': '5', 'reverse': '7', 'conceal': '8'} + +def colorize(text='', opts=(), **kwargs): + """ + Returns your text, enclosed in ANSI graphics codes. + + Depends on the keyword arguments 'fg' and 'bg', and the contents of + the opts tuple/list. + + Returns the RESET code if no parameters are given. + + Valid colors: + 'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white' + + Valid options: + 'bold' + 'underscore' + 'blink' + 'reverse' + 'conceal' + 'noreset' - string will not be auto-terminated with the RESET code + + Examples: + colorize('hello', fg='red', bg='blue', opts=('blink',)) + colorize() + colorize('goodbye', opts=('underscore',)) + print colorize('first line', fg='red', opts=('noreset',)) + print 'this should be red too' + print colorize('and so should this') + print 'this should not be red' + """ + code_list = [] + if text == '' and len(opts) == 1 and opts[0] == 'reset': + return '\x1b[%sm' % RESET + for k, v in kwargs.iteritems(): + if k == 'fg': + code_list.append(foreground[v]) + elif k == 'bg': + code_list.append(background[v]) + for o in opts: + if o in opt_dict: + code_list.append(opt_dict[o]) + if 'noreset' not in opts: + text = text + '\x1b[%sm' % RESET + return ('\x1b[%sm' % ';'.join(code_list)) + text + +def make_style(opts=(), **kwargs): + """ + Returns a function with default parameters for colorize() + + Example: + bold_red = make_style(opts=('bold',), fg='red') + print bold_red('hello') + KEYWORD = make_style(fg='yellow') + COMMENT = make_style(fg='blue', opts=('bold',)) + """ + return lambda text: colorize(text, opts, **kwargs) + +NOCOLOR_PALETTE = 'nocolor' +DARK_PALETTE = 'dark' +LIGHT_PALETTE = 'light' + +PALETTES = { + NOCOLOR_PALETTE: { + 'ERROR': {}, + 'NOTICE': {}, + 'SQL_FIELD': {}, + 'SQL_COLTYPE': {}, + 'SQL_KEYWORD': {}, + 'SQL_TABLE': {}, + 'HTTP_INFO': {}, + 'HTTP_SUCCESS': {}, + 'HTTP_REDIRECT': {}, + 'HTTP_NOT_MODIFIED': {}, + 'HTTP_BAD_REQUEST': {}, + 'HTTP_NOT_FOUND': {}, + 'HTTP_SERVER_ERROR': {}, + }, + DARK_PALETTE: { + 'ERROR': { 'fg': 'red', 'opts': ('bold',) }, + 'NOTICE': { 'fg': 'red' }, + 'SQL_FIELD': { 'fg': 'green', 'opts': ('bold',) }, + 'SQL_COLTYPE': { 'fg': 'green' }, + 'SQL_KEYWORD': { 'fg': 'yellow' }, + 'SQL_TABLE': { 'opts': ('bold',) }, + 'HTTP_INFO': { 'opts': ('bold',) }, + 'HTTP_SUCCESS': { }, + 'HTTP_REDIRECT': { 'fg': 'green' }, + 'HTTP_NOT_MODIFIED': { 'fg': 'cyan' }, + 'HTTP_BAD_REQUEST': { 'fg': 'red', 'opts': ('bold',) }, + 'HTTP_NOT_FOUND': { 'fg': 'yellow' }, + 'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) }, + }, + LIGHT_PALETTE: { + 'ERROR': { 'fg': 'red', 'opts': ('bold',) }, + 'NOTICE': { 'fg': 'red' }, + 'SQL_FIELD': { 'fg': 'green', 'opts': ('bold',) }, + 'SQL_COLTYPE': { 'fg': 'green' }, + 'SQL_KEYWORD': { 'fg': 'blue' }, + 'SQL_TABLE': { 'opts': ('bold',) }, + 'HTTP_INFO': { 'opts': ('bold',) }, + 'HTTP_SUCCESS': { }, + 'HTTP_REDIRECT': { 'fg': 'green', 'opts': ('bold',) }, + 'HTTP_NOT_MODIFIED': { 'fg': 'green' }, + 'HTTP_BAD_REQUEST': { 'fg': 'red', 'opts': ('bold',) }, + 'HTTP_NOT_FOUND': { 'fg': 'red' }, + 'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) }, + } +} +DEFAULT_PALETTE = DARK_PALETTE + +# ---------------------------- # +# --- End of termcolors.py --- # +# ---------------------------- # + + +class ColoredProgressBar(ProgressBar): + + BAR_COLORS = ( + (10, 'red'), + (30, 'magenta'), + (50, 'yellow'), + (99, 'green'), + (100, 'blue'), + ) + + def get_line(self): + line = super(ColoredProgressBar, self).get_line() + perc = self.get_percentage() + if perc > 100: + color = 'blue' + for max_perc, color in self.BAR_COLORS: + if perc <= max_perc: + break + return colorize(line, fg=color) + + +class AnimatedProgressBar(ProgressBar): + + def get_bar_char(self): + chars = '-/|\\' + if self.step >= self.steps: + return '=' + return chars[self.step % len(chars)] + + +class BarOnlyProgressBar(ProgressBar): + + default_elements = ['bar', 'steps'] + + def get_bar(self): + bar = super(BarOnlyProgressBar, self).get_bar() + perc = self.get_percentage() + perc_text = '%s%%' % int(perc) + text = (' %s%% ' % (perc_text)).center(self.width, '=') + L = text.find(' ') + R = text.rfind(' ') + bar = ' '.join((bar[:L], perc_text, bar[R:])) + return bar + + +class AnimatedColoredProgressBar(AnimatedProgressBar, + ColoredProgressBar): + pass + + +class BarOnlyColoredProgressBar(ColoredProgressBar, + BarOnlyProgressBar): + pass + + + +def main(): + import time + + print "Standard progress bar..." + bar = ProgressBar(30) + for x in xrange(1, 31): + bar.render(x) + time.sleep(0.02) + bar.stream.write('\n') + print + + print "Empty bar..." + bar = ProgressBar(50) + bar.render(0) + print + print + + print "Colored bar..." + bar = ColoredProgressBar(20) + for x in bar: + time.sleep(0.01) + print + + print "Animated char bar..." + bar = AnimatedProgressBar(20) + for x in bar: + time.sleep(0.01) + print + + print "Animated + colored char bar..." + bar = AnimatedColoredProgressBar(20) + for x in bar: + time.sleep(0.01) + print + + print "Bar only ..." + bar = BarOnlyProgressBar(20) + for x in bar: + time.sleep(0.01) + print + + print "Colored, longer bar-only, eta, total time ..." + bar = BarOnlyColoredProgressBar(40) + bar.width = 60 + bar.elements += ['time', 'eta'] + for x in bar: + time.sleep(0.01) + print + print + + print "File transfer bar, breaks after 2 seconds ..." + total_bytes = 1024 * 1024 * 2 + bar = ProgressBar(total_bytes) + bar.width = 50 + bar.elements.remove('steps') + bar.elements += ['transfer', 'time', 'eta', 'speed'] + for x in xrange(0, bar.steps, 1024): + bar.render(x) + time.sleep(0.01) + now = datetime.datetime.now() + if now - bar.started >= datetime.timedelta(seconds=2): + break + print + print + + + +if __name__ == '__main__': + main() diff --git a/rhodecode/lib/vcs/utils/termcolors.py b/rhodecode/lib/vcs/utils/termcolors.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/vcs/utils/termcolors.py @@ -0,0 +1,200 @@ +""" +termcolors.py + +Grabbed from Django (http://www.djangoproject.com) +""" + +color_names = ('black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white') +foreground = dict([(color_names[x], '3%s' % x) for x in range(8)]) +background = dict([(color_names[x], '4%s' % x) for x in range(8)]) + +RESET = '0' +opt_dict = {'bold': '1', 'underscore': '4', 'blink': '5', 'reverse': '7', 'conceal': '8'} + +def colorize(text='', opts=(), **kwargs): + """ + Returns your text, enclosed in ANSI graphics codes. + + Depends on the keyword arguments 'fg' and 'bg', and the contents of + the opts tuple/list. + + Returns the RESET code if no parameters are given. + + Valid colors: + 'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white' + + Valid options: + 'bold' + 'underscore' + 'blink' + 'reverse' + 'conceal' + 'noreset' - string will not be auto-terminated with the RESET code + + Examples: + colorize('hello', fg='red', bg='blue', opts=('blink',)) + colorize() + colorize('goodbye', opts=('underscore',)) + print colorize('first line', fg='red', opts=('noreset',)) + print 'this should be red too' + print colorize('and so should this') + print 'this should not be red' + """ + code_list = [] + if text == '' and len(opts) == 1 and opts[0] == 'reset': + return '\x1b[%sm' % RESET + for k, v in kwargs.iteritems(): + if k == 'fg': + code_list.append(foreground[v]) + elif k == 'bg': + code_list.append(background[v]) + for o in opts: + if o in opt_dict: + code_list.append(opt_dict[o]) + if 'noreset' not in opts: + text = text + '\x1b[%sm' % RESET + return ('\x1b[%sm' % ';'.join(code_list)) + text + +def make_style(opts=(), **kwargs): + """ + Returns a function with default parameters for colorize() + + Example: + bold_red = make_style(opts=('bold',), fg='red') + print bold_red('hello') + KEYWORD = make_style(fg='yellow') + COMMENT = make_style(fg='blue', opts=('bold',)) + """ + return lambda text: colorize(text, opts, **kwargs) + +NOCOLOR_PALETTE = 'nocolor' +DARK_PALETTE = 'dark' +LIGHT_PALETTE = 'light' + +PALETTES = { + NOCOLOR_PALETTE: { + 'ERROR': {}, + 'NOTICE': {}, + 'SQL_FIELD': {}, + 'SQL_COLTYPE': {}, + 'SQL_KEYWORD': {}, + 'SQL_TABLE': {}, + 'HTTP_INFO': {}, + 'HTTP_SUCCESS': {}, + 'HTTP_REDIRECT': {}, + 'HTTP_NOT_MODIFIED': {}, + 'HTTP_BAD_REQUEST': {}, + 'HTTP_NOT_FOUND': {}, + 'HTTP_SERVER_ERROR': {}, + }, + DARK_PALETTE: { + 'ERROR': { 'fg': 'red', 'opts': ('bold',) }, + 'NOTICE': { 'fg': 'red' }, + 'SQL_FIELD': { 'fg': 'green', 'opts': ('bold',) }, + 'SQL_COLTYPE': { 'fg': 'green' }, + 'SQL_KEYWORD': { 'fg': 'yellow' }, + 'SQL_TABLE': { 'opts': ('bold',) }, + 'HTTP_INFO': { 'opts': ('bold',) }, + 'HTTP_SUCCESS': { }, + 'HTTP_REDIRECT': { 'fg': 'green' }, + 'HTTP_NOT_MODIFIED': { 'fg': 'cyan' }, + 'HTTP_BAD_REQUEST': { 'fg': 'red', 'opts': ('bold',) }, + 'HTTP_NOT_FOUND': { 'fg': 'yellow' }, + 'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) }, + }, + LIGHT_PALETTE: { + 'ERROR': { 'fg': 'red', 'opts': ('bold',) }, + 'NOTICE': { 'fg': 'red' }, + 'SQL_FIELD': { 'fg': 'green', 'opts': ('bold',) }, + 'SQL_COLTYPE': { 'fg': 'green' }, + 'SQL_KEYWORD': { 'fg': 'blue' }, + 'SQL_TABLE': { 'opts': ('bold',) }, + 'HTTP_INFO': { 'opts': ('bold',) }, + 'HTTP_SUCCESS': { }, + 'HTTP_REDIRECT': { 'fg': 'green', 'opts': ('bold',) }, + 'HTTP_NOT_MODIFIED': { 'fg': 'green' }, + 'HTTP_BAD_REQUEST': { 'fg': 'red', 'opts': ('bold',) }, + 'HTTP_NOT_FOUND': { 'fg': 'red' }, + 'HTTP_SERVER_ERROR': { 'fg': 'magenta', 'opts': ('bold',) }, + } +} +DEFAULT_PALETTE = DARK_PALETTE + +def parse_color_setting(config_string): + """Parse a DJANGO_COLORS environment variable to produce the system palette + + The general form of a pallete definition is: + + "palette;role=fg;role=fg/bg;role=fg,option,option;role=fg/bg,option,option" + + where: + palette is a named palette; one of 'light', 'dark', or 'nocolor'. + role is a named style used by Django + fg is a background color. + bg is a background color. + option is a display options. + + Specifying a named palette is the same as manually specifying the individual + definitions for each role. Any individual definitions following the pallete + definition will augment the base palette definition. + + Valid roles: + 'error', 'notice', 'sql_field', 'sql_coltype', 'sql_keyword', 'sql_table', + 'http_info', 'http_success', 'http_redirect', 'http_bad_request', + 'http_not_found', 'http_server_error' + + Valid colors: + 'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white' + + Valid options: + 'bold', 'underscore', 'blink', 'reverse', 'conceal' + + """ + if not config_string: + return PALETTES[DEFAULT_PALETTE] + + # Split the color configuration into parts + parts = config_string.lower().split(';') + palette = PALETTES[NOCOLOR_PALETTE].copy() + for part in parts: + if part in PALETTES: + # A default palette has been specified + palette.update(PALETTES[part]) + elif '=' in part: + # Process a palette defining string + definition = {} + + # Break the definition into the role, + # plus the list of specific instructions. + # The role must be in upper case + role, instructions = part.split('=') + role = role.upper() + + styles = instructions.split(',') + styles.reverse() + + # The first instruction can contain a slash + # to break apart fg/bg. + colors = styles.pop().split('/') + colors.reverse() + fg = colors.pop() + if fg in color_names: + definition['fg'] = fg + if colors and colors[-1] in color_names: + definition['bg'] = colors[-1] + + # All remaining instructions are options + opts = tuple(s for s in styles if s in opt_dict.keys()) + if opts: + definition['opts'] = opts + + # The nocolor palette has all available roles. + # Use that palette as the basis for determining + # if the role is valid. + if role in PALETTES[NOCOLOR_PALETTE] and definition: + palette[role] = definition + + # If there are no colors specified, return the empty palette. + if palette == PALETTES[NOCOLOR_PALETTE]: + return None + return palette diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -34,10 +34,10 @@ from sqlalchemy.ext.hybrid import hybrid from sqlalchemy.orm import relationship, joinedload, class_mapper, validates from beaker.cache import cache_region, region_invalidate -from vcs import get_backend -from vcs.utils.helpers import get_scm -from vcs.exceptions import VCSError -from vcs.utils.lazy import LazyProperty +from rhodecode.lib.vcs import get_backend +from rhodecode.lib.vcs.utils.helpers import get_scm +from rhodecode.lib.vcs.exceptions import VCSError +from rhodecode.lib.vcs.utils.lazy import LazyProperty from rhodecode.lib import str2bool, safe_str, get_changeset_safe, safe_unicode from rhodecode.lib.compat import json diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -28,7 +28,7 @@ import logging import traceback from datetime import datetime -from vcs.backends import get_backend +from rhodecode.lib.vcs.backends import get_backend from rhodecode.lib import LazyProperty from rhodecode.lib import safe_str, safe_unicode diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -28,10 +28,10 @@ import traceback import logging import cStringIO -from vcs import get_backend -from vcs.exceptions import RepositoryError -from vcs.utils.lazy import LazyProperty -from vcs.nodes import FileNode +from rhodecode.lib.vcs import get_backend +from rhodecode.lib.vcs.exceptions import RepositoryError +from rhodecode.lib.vcs.utils.lazy import LazyProperty +from rhodecode.lib.vcs.nodes import FileNode from rhodecode import BACKENDS from rhodecode.lib import helpers as h @@ -359,9 +359,9 @@ class ScmModel(BaseModel): content, f_path): if repo.alias == 'hg': - from vcs.backends.hg import MercurialInMemoryChangeset as IMC + from rhodecode.lib.vcs.backends.hg import MercurialInMemoryChangeset as IMC elif repo.alias == 'git': - from vcs.backends.git import GitInMemoryChangeset as IMC + from rhodecode.lib.vcs.backends.git import GitInMemoryChangeset as IMC # decoding here will force that we have proper encoded values # in any other case this will throw exceptions and deny commit @@ -385,9 +385,9 @@ class ScmModel(BaseModel): def create_node(self, repo, repo_name, cs, user, author, message, content, f_path): if repo.alias == 'hg': - from vcs.backends.hg import MercurialInMemoryChangeset as IMC + from rhodecode.lib.vcs.backends.hg import MercurialInMemoryChangeset as IMC elif repo.alias == 'git': - from vcs.backends.git import GitInMemoryChangeset as IMC + from rhodecode.lib.vcs.backends.git import GitInMemoryChangeset as IMC # decoding here will force that we have proper encoded values # in any other case this will throw exceptions and deny commit diff --git a/rhodecode/tests/functional/test_admin_repos.py b/rhodecode/tests/functional/test_admin_repos.py --- a/rhodecode/tests/functional/test_admin_repos.py +++ b/rhodecode/tests/functional/test_admin_repos.py @@ -1,18 +1,16 @@ # -*- coding: utf-8 -*- import os -import vcs +from rhodecode.lib import vcs from rhodecode.model.db import Repository from rhodecode.tests import * class TestAdminReposController(TestController): - def __make_repo(self): pass - def test_index(self): self.log_user() response = self.app.get(url('repos')) diff --git a/rhodecode/tests/rhodecode_crawler.py b/rhodecode/tests/rhodecode_crawler.py --- a/rhodecode/tests/rhodecode_crawler.py +++ b/rhodecode/tests/rhodecode_crawler.py @@ -31,11 +31,10 @@ import cookielib import urllib import urllib2 -import vcs import time from os.path import join as jn - +from rhodecode.lib import vcs BASE_URI = 'http://127.0.0.1:5000/%s' PROJECT = 'CPython' @@ -52,7 +51,6 @@ o.addheaders = [ urllib2.install_opener(o) - def test_changelog_walk(pages=100): total_time = 0 for i in range(1, pages): @@ -67,7 +65,6 @@ def test_changelog_walk(pages=100): total_time += e print 'visited %s size:%s req:%s ms' % (full_uri, size, e) - print 'total_time', total_time print 'average on req', total_time / float(pages) @@ -103,7 +100,7 @@ def test_files_walk(limit=100): repo = vcs.get_repo(jn(PROJECT_PATH, PROJECT)) from rhodecode.lib.compat import OrderedSet - from vcs.exceptions import RepositoryError + from rhodecode.lib.vcs.exceptions import RepositoryError paths_ = OrderedSet(['']) try: @@ -141,7 +138,6 @@ def test_files_walk(limit=100): print 'average on req', total_time / float(cnt) - test_changelog_walk(40) time.sleep(2) test_changeset_walk(limit=100) diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -7,10 +7,7 @@ from rhodecode import requirements if __py_version__ < (2, 5): raise Exception('RhodeCode requires python 2.5 or later') - dependency_links = [ - "https://secure.rhodecode.org/vcs/archive/default.zip#egg=vcs-0.2.3.dev", - "https://bitbucket.org/marcinkuzminski/vcs/get/default.zip#egg=vcs-0.2.3.dev", ] classifiers = [