integrations: add integration support

* added db migration for integration table, version 55
* added integrations views/templates/models
* added slack integration
* added h.url.current() support for pyramid only contexts
* added issues extractor for commit messages
* renamed repo-deleted/repo-created to repo-delete/repo-create
* removed marshmallow
* added serialization for events to static dicts of data
* added lazy_ugettext for translation of start time string

rhodecode/integrations/__init__.py

@@ -0,0 +1,52 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+from rhodecode.integrations.registry import IntegrationTypeRegistry
+from rhodecode.integrations.types import slack
+
+log = logging.getLogger(__name__)
+
+
+# TODO: dan: This is currently global until we figure out what to do about
+# VCS's not having a pyramid context - move it to pyramid app configuration
+# includeme level later to allow per instance integration setup
+integration_type_registry = IntegrationTypeRegistry()
+integration_type_registry.register_integration_type(slack.SlackIntegrationType)
+
+def integrations_event_handler(event):
+    """
+    Takes an event and passes it to all enabled integrations
+    """
+    from rhodecode.model.integration import IntegrationModel
+
+    integration_model = IntegrationModel()
+    integrations = integration_model.get_for_event(event)
+    for integration in integrations:
+        try:
+            integration_model.send_event(integration, event)
+        except Exception:
+            log.exception(
+                'failure occured when sending event %s to integration %s' % (
+                    event, integration))
+
+
+def includeme(config):
+    config.include('rhodecode.integrations.routes')

rhodecode/integrations/registry.py

@@ -0,0 +1,37 b''
+# -*- coding: utf-8 -*-
+# Copyright (C) 2012-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+
+log = logging.getLogger()
+
+
+class IntegrationTypeRegistry(dict):
+    """
+    Registry Class to hold IntegrationTypes
+    """
+    def register_integration_type(self, IntegrationType):
+        key = IntegrationType.key
+        if key in self:
+            log.warning(
+                'Overriding existing integration type %s (%s) with %s' % (
+                    self[key], key, IntegrationType))
+
+        self[key] = IntegrationType
+

rhodecode/integrations/routes.py

@@ -0,0 +1,133 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+
+from rhodecode.model.db import Repository, Integration
+from rhodecode.config.routing import (
+    ADMIN_PREFIX, add_route_requirements, URL_NAME_REQUIREMENTS)
+from rhodecode.integrations import integration_type_registry
+
+log = logging.getLogger(__name__)
+
+
+def includeme(config):
+    config.add_route('global_integrations_home',
+                     ADMIN_PREFIX + '/integrations')
+    config.add_route('global_integrations_list',
+                     ADMIN_PREFIX + '/integrations/{integration}')
+    for route_name in ['global_integrations_home', 'global_integrations_list']:
+        config.add_view('rhodecode.integrations.views.GlobalIntegrationsView',
+                        attr='index',
+                        renderer='rhodecode:templates/admin/integrations/list.html',
+                        request_method='GET',
+                        route_name=route_name)
+
+    config.add_route('global_integrations_create',
+                     ADMIN_PREFIX + '/integrations/{integration}/new',
+                     custom_predicates=(valid_integration,))
+    config.add_route('global_integrations_edit',
+                     ADMIN_PREFIX + '/integrations/{integration}/{integration_id}',
+                     custom_predicates=(valid_integration,))
+    for route_name in ['global_integrations_create', 'global_integrations_edit']:
+        config.add_view('rhodecode.integrations.views.GlobalIntegrationsView',
+                        attr='settings_get',
+                        renderer='rhodecode:templates/admin/integrations/edit.html',
+                        request_method='GET',
+                        route_name=route_name)
+        config.add_view('rhodecode.integrations.views.GlobalIntegrationsView',
+                        attr='settings_post',
+                        renderer='rhodecode:templates/admin/integrations/edit.html',
+                        request_method='POST',
+                        route_name=route_name)
+
+    config.add_route('repo_integrations_home',
+                     add_route_requirements(
+                        '{repo_name}/settings/integrations',
+                        URL_NAME_REQUIREMENTS
+                     ),
+                     custom_predicates=(valid_repo,))
+    config.add_route('repo_integrations_list',
+                     add_route_requirements(
+                        '{repo_name}/settings/integrations/{integration}',
+                        URL_NAME_REQUIREMENTS
+                     ),
+                     custom_predicates=(valid_repo, valid_integration))
+    for route_name in ['repo_integrations_home', 'repo_integrations_list']:
+        config.add_view('rhodecode.integrations.views.RepoIntegrationsView',
+                        attr='index',
+                        request_method='GET',
+                        route_name=route_name)
+
+    config.add_route('repo_integrations_create',
+                     add_route_requirements(
+                        '{repo_name}/settings/integrations/{integration}/new',
+                        URL_NAME_REQUIREMENTS
+                     ),
+                     custom_predicates=(valid_repo, valid_integration))
+    config.add_route('repo_integrations_edit',
+                     add_route_requirements(
+                        '{repo_name}/settings/integrations/{integration}/{integration_id}',
+                        URL_NAME_REQUIREMENTS
+                     ),
+                     custom_predicates=(valid_repo, valid_integration))
+    for route_name in ['repo_integrations_edit', 'repo_integrations_create']:
+        config.add_view('rhodecode.integrations.views.RepoIntegrationsView',
+                        attr='settings_get',
+                        renderer='rhodecode:templates/admin/integrations/edit.html',
+                        request_method='GET',
+                        route_name=route_name)
+        config.add_view('rhodecode.integrations.views.RepoIntegrationsView',
+                        attr='settings_post',
+                        renderer='rhodecode:templates/admin/integrations/edit.html',
+                        request_method='POST',
+                        route_name=route_name)
+
+
+def valid_repo(info, request):
+    repo = Repository.get_by_repo_name(info['match']['repo_name'])
+    if repo:
+        return True
+
+
+def valid_integration(info, request):
+    integration_type = info['match']['integration']
+    integration_id = info['match'].get('integration_id')
+    repo_name = info['match'].get('repo_name')
+
+    if integration_type not in integration_type_registry:
+        return False
+
+    repo = None
+    if repo_name:
+        repo = Repository.get_by_repo_name(info['match']['repo_name'])
+        if not repo:
+            return False
+
+    if integration_id:
+        integration = Integration.get(integration_id)
+        if not integration:
+            return False
+        if integration.integration_type != integration_type:
+            return False
+        if repo and repo.repo_id != integration.repo_id:
+            return False
+
+    return True

rhodecode/integrations/schema.py

@@ -0,0 +1,48 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import colander
+
+from rhodecode.translation import lazy_ugettext
+
+
+class IntegrationSettingsSchemaBase(colander.MappingSchema):
+    """
+    This base schema is intended for use in integrations.
+    It adds a few default settings (e.g., "enabled"), so that integration
+    authors don't have to maintain a bunch of boilerplate.
+    """
+    enabled = colander.SchemaNode(
+        colander.Bool(),
+        default=True,
+        description=lazy_ugettext('Enable or disable this integration.'),
+        missing=False,
+        title=lazy_ugettext('Enabled'),
+        widget='bool',
+    )
+
+    name = colander.SchemaNode(
+        colander.String(),
+        description=lazy_ugettext('Short name for this integration.'),
+        missing=colander.required,
+        title=lazy_ugettext('Integration name'),
+        widget='string',
+    )
+

rhodecode/integrations/types/__init__.py

@@ -0,0 +1,19 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/

rhodecode/integrations/types/base.py

@@ -0,0 +1,43 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+from rhodecode.integrations.schema import IntegrationSettingsSchemaBase
+
+
+class IntegrationTypeBase(object):
+    """ Base class for IntegrationType plugins """
+
+    def __init__(self, settings):
+        """
+        :param settings: dict of settings to be used for the integration
+        """
+        self.settings = settings
+
+
+    @classmethod
+    def settings_schema(cls):
+        """
+        A colander schema of settings for the integration type
+
+        Subclasses can return their own schema but should always
+        inherit from IntegrationSettingsSchemaBase
+        """
+        return IntegrationSettingsSchemaBase()
+

rhodecode/integrations/types/slack.py

@@ -0,0 +1,199 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+from __future__ import unicode_literals
+
+import re
+import logging
+import requests
+import colander
+from celery.task import task
+from mako.template import Template
+
+from rhodecode import events
+from rhodecode.translation import lazy_ugettext
+from rhodecode.lib import helpers as h
+from rhodecode.lib.celerylib import run_task
+from rhodecode.lib.colander_utils import strip_whitespace
+from rhodecode.integrations.types.base import IntegrationTypeBase
+from rhodecode.integrations.schema import IntegrationSettingsSchemaBase
+
+log = logging.getLogger()
+
+
+class SlackSettingsSchema(IntegrationSettingsSchemaBase):
+    service = colander.SchemaNode(
+        colander.String(),
+        title=lazy_ugettext('Slack service URL'),
+        description=h.literal(lazy_ugettext(
+            'This can be setup at the '
+            '<a href="https://my.slack.com/services/new/incoming-webhook/">'
+            'slack app manager</a>')),
+        default='',
+        placeholder='https://hooks.slack.com/services/...',
+        preparer=strip_whitespace,
+        validator=colander.url,
+        widget='string'
+    )
+    username = colander.SchemaNode(
+        colander.String(),
+        title=lazy_ugettext('Username'),
+        description=lazy_ugettext('Username to show notifications coming from.'),
+        missing='Rhodecode',
+        preparer=strip_whitespace,
+        widget='string',
+        placeholder='Rhodecode'
+    )
+    channel = colander.SchemaNode(
+        colander.String(),
+        title=lazy_ugettext('Channel'),
+        description=lazy_ugettext('Channel to send notifications to.'),
+        missing='',
+        preparer=strip_whitespace,
+        widget='string',
+        placeholder='#general'
+    )
+    icon_emoji = colander.SchemaNode(
+        colander.String(),
+        title=lazy_ugettext('Emoji'),
+        description=lazy_ugettext('Emoji to use eg. :studio_microphone:'),
+        missing='',
+        preparer=strip_whitespace,
+        widget='string',
+        placeholder=':studio_microphone:'
+    )
+
+
+repo_push_template = Template(r'''
+*${data['actor']['username']}* pushed to \
+%if data['push']['branches']:
+${len(data['push']['branches']) > 1 and 'branches' or 'branch'} \
+${', '.join('<%s|%s>' % (branch['url'], branch['name']) for branch in data['push']['branches'])} \
+%else:
+unknown branch \
+%endif
+in <${data['repo']['url']}|${data['repo']['repo_name']}>
+>>>
+%for commit in data['push']['commits']:
+<${commit['url']}|${commit['short_id']}> - ${commit['message_html']|html_to_slack_links}
+%endfor
+''')
+
+
+class SlackIntegrationType(IntegrationTypeBase):
+    key = 'slack'
+    display_name = lazy_ugettext('Slack')
+    SettingsSchema = SlackSettingsSchema
+    valid_events = [
+        events.PullRequestCloseEvent,
+        events.PullRequestMergeEvent,
+        events.PullRequestUpdateEvent,
+        events.PullRequestReviewEvent,
+        events.PullRequestCreateEvent,
+        events.RepoPushEvent,
+        events.RepoCreateEvent,
+    ]
+
+    def send_event(self, event):
+        if event.__class__ not in self.valid_events:
+            log.debug('event not valid: %r' % event)
+            return
+
+        if event.name not in self.settings['events']:
+            log.debug('event ignored: %r' % event)
+            return
+
+        data = event.as_dict()
+
+        text = '*%s* caused a *%s* event' % (
+            data['actor']['username'], event.name)
+
+        if isinstance(event, events.PullRequestEvent):
+            text = self.format_pull_request_event(event, data)
+        elif isinstance(event, events.RepoPushEvent):
+            text = self.format_repo_push_event(data)
+        elif isinstance(event, events.RepoCreateEvent):
+            text = self.format_repo_create_event(data)
+        else:
+            log.error('unhandled event type: %r' % event)
+
+        run_task(post_text_to_slack, self.settings, text)
+
+    @classmethod
+    def settings_schema(cls):
+        schema = SlackSettingsSchema()
+        schema.add(colander.SchemaNode(
+            colander.Set(),
+            widget='checkbox_list',
+            choices=sorted([e.name for e in cls.valid_events]),
+            description="Events activated for this integration",
+            default=[e.name for e in cls.valid_events],
+            name='events'
+        ))
+        return schema
+
+    def format_pull_request_event(self, event, data):
+        action = {
+            events.PullRequestCloseEvent: 'closed',
+            events.PullRequestMergeEvent: 'merged',
+            events.PullRequestUpdateEvent: 'updated',
+            events.PullRequestReviewEvent: 'reviewed',
+            events.PullRequestCreateEvent: 'created',
+        }.get(event.__class__, '<unknown action>')
+
+        return ('Pull request <{url}|#{number}> ({title}) '
+                '{action} by {user}').format(
+            user=data['actor']['username'],
+            number=data['pullrequest']['pull_request_id'],
+            url=data['pullrequest']['url'],
+            title=data['pullrequest']['title'],
+            action=action
+        )
+
+    def format_repo_push_event(self, data):
+        result = repo_push_template.render(
+            data=data,
+            html_to_slack_links=html_to_slack_links,
+        )
+        return result
+
+    def format_repo_create_msg(self, data):
+        return '<{}|{}> ({}) repository created by *{}*'.format(
+            data['repo']['url'],
+            data['repo']['repo_name'],
+            data['repo']['repo_type'],
+            data['actor']['username'],
+        )
+
+
+def html_to_slack_links(message):
+    return re.compile(r'<a .*?href=["\'](.+?)".*?>(.+?)</a>').sub(
+        r'<\1|\2>', message)
+
+
+@task(ignore_result=True)
+def post_text_to_slack(settings, text):
+    resp = requests.post(settings['service'], json={
+        "channel": settings.get('channel', ''),
+        "username": settings.get('username', 'Rhodecode'),
+        "text": text,
+        "icon_emoji": settings.get('icon_emoji', ':studio_microphone:')
+    })
+    resp.raise_for_status() # raise exception on a failed request

rhodecode/integrations/views.py

@@ -0,0 +1,257 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import colander
+import logging
+import pylons
+
+from pyramid.httpexceptions import HTTPFound, HTTPForbidden
+from pyramid.renderers import render
+from pyramid.response import Response
+
+from rhodecode.lib import auth
+from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
+from rhodecode.model.db import Repository, Session, Integration
+from rhodecode.model.scm import ScmModel
+from rhodecode.model.integration import IntegrationModel
+from rhodecode.admin.navigation import navigation_list
+from rhodecode.translation import _
+from rhodecode.integrations import integration_type_registry
+
+log = logging.getLogger(__name__)
+
+
+class IntegrationSettingsViewBase(object):
+    """ Base Integration settings view used by both repo / global settings """
+
+    def __init__(self, context, request):
+        self.context = context
+        self.request = request
+        self._load_general_context()
+
+        if not self.perm_check(request.user):
+            raise HTTPForbidden()
+
+    def _load_general_context(self):
+        """
+        This avoids boilerplate for repo/global+list/edit+views/templates
+        by doing all possible contexts at the same time however it should
+        be split up into separate functions once more "contexts" exist
+        """
+
+        self.IntegrationType = None
+        self.repo = None
+        self.integration = None
+        self.integrations = {}
+
+        request = self.request
+
+        if 'repo_name' in request.matchdict:  # we're in a repo context
+            repo_name = request.matchdict['repo_name']
+            self.repo = Repository.get_by_repo_name(repo_name)
+
+        if 'integration' in request.matchdict:  # we're in integration context
+            integration_type = request.matchdict['integration']
+            self.IntegrationType = integration_type_registry[integration_type]
+
+        if 'integration_id' in request.matchdict:  # single integration context
+            integration_id = request.matchdict['integration_id']
+            self.integration = Integration.get(integration_id)
+        else:                                      # list integrations context
+            for integration in IntegrationModel().get_integrations(self.repo):
+                self.integrations.setdefault(integration.integration_type, []
+                    ).append(integration)
+
+        self.settings = self.integration and self.integration.settings or {}
+
+    def _template_c_context(self):
+        # TODO: dan: this is a stopgap in order to inherit from current pylons
+        # based admin/repo settings templates - this should be removed entirely
+        # after port to pyramid
+
+        c = pylons.tmpl_context
+        c.active = 'integrations'
+        c.rhodecode_user = self.request.user
+        c.repo = self.repo
+        c.repo_name = self.repo and self.repo.repo_name or None
+        if self.repo:
+            c.repo_info = self.repo
+            c.rhodecode_db_repo = self.repo
+            c.repository_pull_requests = ScmModel().get_pull_requests(self.repo)
+        else:
+            c.navlist = navigation_list(self.request)
+
+        return c
+
+    def _form_schema(self):
+        return self.IntegrationType.settings_schema()
+
+    def settings_get(self, defaults=None, errors=None):
+        """
+        View that displays the plugin settings as a form.
+        """
+        defaults = defaults or {}
+        errors = errors or {}
+
+        schema = self._form_schema()
+
+        if not defaults:
+            if self.integration:
+                defaults['enabled'] = self.integration.enabled
+                defaults['name'] = self.integration.name
+            else:
+                if self.repo:
+                    scope = self.repo.repo_name
+                else:
+                    scope = _('Global')
+
+                defaults['name'] = '{} {} integration'.format(scope,
+                    self.IntegrationType.display_name)
+                defaults['enabled'] = True
+
+        for node in schema:
+            setting = self.settings.get(node.name)
+            if setting is not None:
+                defaults.setdefault(node.name, setting)
+            else:
+                if node.default:
+                    defaults.setdefault(node.name, node.default)
+
+        template_context = {
+            'defaults': defaults,
+            'errors': errors,
+            'schema': schema,
+            'current_IntegrationType': self.IntegrationType,
+            'integration': self.integration,
+            'settings': self.settings,
+            'resource': self.context,
+            'c': self._template_c_context(),
+        }
+
+        return template_context
+
+    @auth.CSRFRequired()
+    def settings_post(self):
+        """
+        View that validates and stores the plugin settings.
+        """
+        if self.request.params.get('delete'):
+            Session().delete(self.integration)
+            Session().commit()
+            self.request.session.flash(
+                _('Integration {integration_name} deleted successfully.').format(
+                    integration_name=self.integration.name),
+                queue='success')
+            if self.repo:
+                redirect_to = self.request.route_url(
+                    'repo_integrations_home', repo_name=self.repo.repo_name)
+            else:
+                redirect_to = self.request.route_url('global_integrations_home')
+            raise HTTPFound(redirect_to)
+
+        schema = self._form_schema()
+
+        params = {}
+        for node in schema.children:
+            if type(node.typ) in (colander.Set, colander.List):
+                val = self.request.params.getall(node.name)
+            else:
+                val = self.request.params.get(node.name)
+            if val:
+                params[node.name] = val
+
+        try:
+            valid_data = schema.deserialize(params)
+        except colander.Invalid, e:
+            # Display error message and display form again.
+            self.request.session.flash(
+                _('Errors exist when saving plugin settings. '
+                  'Please check the form inputs.'),
+                queue='error')
+            return self.settings_get(errors=e.asdict(), defaults=params)
+
+        if not self.integration:
+            self.integration = Integration(
+                integration_type=self.IntegrationType.key)
+            if self.repo:
+                self.integration.repo = self.repo
+            Session.add(self.integration)
+
+        self.integration.enabled = valid_data.pop('enabled', False)
+        self.integration.name = valid_data.pop('name')
+        self.integration.settings = valid_data
+
+        Session.commit()
+
+        # Display success message and redirect.
+        self.request.session.flash(
+            _('Integration {integration_name} updated successfully.').format(
+                integration_name=self.IntegrationType.display_name,
+                queue='success'))
+        if self.repo:
+            redirect_to = self.request.route_url(
+                'repo_integrations_edit', repo_name=self.repo.repo_name,
+                integration=self.integration.integration_type,
+                integration_id=self.integration.integration_id)
+        else:
+            redirect_to = self.request.route_url(
+                'global_integrations_edit',
+                integration=self.integration.integration_type,
+                integration_id=self.integration.integration_id)
+
+        return HTTPFound(redirect_to)
+
+    def index(self):
+        current_integrations = self.integrations
+        if self.IntegrationType:
+            current_integrations = {
+                self.IntegrationType.key: self.integrations.get(
+                    self.IntegrationType.key, [])
+            }
+
+        template_context = {
+            'current_IntegrationType': self.IntegrationType,
+            'current_integrations': current_integrations,
+            'current_integration': 'none',
+            'available_integrations': integration_type_registry,
+            'c': self._template_c_context()
+        }
+
+        if self.repo:
+            html = render('rhodecode:templates/admin/integrations/list.html',
+                          template_context,
+                          request=self.request)
+        else:
+            html = render('rhodecode:templates/admin/integrations/list.html',
+                          template_context,
+                          request=self.request)
+
+        return Response(html)
+
+
+class GlobalIntegrationsView(IntegrationSettingsViewBase):
+    def perm_check(self, user):
+        return auth.HasPermissionAll('hg.admin').check_permissions(user=user)
+
+
+class RepoIntegrationsView(IntegrationSettingsViewBase):
+    def perm_check(self, user):
+        return auth.HasRepoPermissionAll('repository.admin'
+            )(repo_name=self.repo.repo_name, user=user)

rhodecode/lib/dbmigrate/versions/055_version_4_3_0.py

@@ -0,0 +1,27 b''
+# -*- coding: utf-8 -*-
+
+import logging
+import sqlalchemy as sa
+
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+
+from rhodecode.lib.dbmigrate.versions import _reset_base
+
+log = logging.getLogger(__name__)
+
+
+def upgrade(migrate_engine):
+    """
+    Upgrade operations go here.
+    Don't create your own engine; bind migrate_engine to your metadata
+    """
+    _reset_base(migrate_engine)
+    from rhodecode.lib.dbmigrate.schema import db_4_3_0_0
+
+    integrations_table = db_4_3_0_0.Integration.__table__
+    integrations_table.create()
+
+
+def downgrade(migrate_engine):
+    pass

pkgs/python-packages.nix

@@ -1,1654 +1,1641 b''
 {
   Babel = super.buildPythonPackage {
     name = "Babel-1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pytz];
     src = fetchurl {
       url = "https://pypi.python.org/packages/33/27/e3978243a03a76398c384c83f7ca879bc6e8f1511233a621fcada135606e/Babel-1.3.tar.gz";
       md5 = "5264ceb02717843cbc9ffce8e6e06bdb";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   Beaker = super.buildPythonPackage {
     name = "Beaker-1.7.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/97/8e/409d2e7c009b8aa803dc9e6f239f1db7c3cdf578249087a404e7c27a505d/Beaker-1.7.0.tar.gz";
       md5 = "386be3f7fe427358881eee4622b428b3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   CProfileV = super.buildPythonPackage {
     name = "CProfileV-1.0.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [bottle];
     src = fetchurl {
       url = "https://pypi.python.org/packages/eb/df/983a0b6cfd3ac94abf023f5011cb04f33613ace196e33f53c86cf91850d5/CProfileV-1.0.6.tar.gz";
       md5 = "08c7c242b6e64237bc53c5d13537e03d";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   Fabric = super.buildPythonPackage {
     name = "Fabric-1.10.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [paramiko];
     src = fetchurl {
       url = "https://pypi.python.org/packages/e3/5f/b6ebdb5241d5ec9eab582a5c8a01255c1107da396f849e538801d2fe64a5/Fabric-1.10.0.tar.gz";
       md5 = "2cb96473387f0e7aa035210892352f4a";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   FormEncode = super.buildPythonPackage {
     name = "FormEncode-1.2.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/8e/59/0174271a6f004512e0201188593e6d319db139d14cb7490e488bbb078015/FormEncode-1.2.4.tar.gz";
       md5 = "6bc17fb9aed8aea198975e888e2077f4";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   Jinja2 = super.buildPythonPackage {
     name = "Jinja2-2.7.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [MarkupSafe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b0/73/eab0bca302d6d6a0b5c402f47ad1760dc9cb2dd14bbc1873ad48db258e4d/Jinja2-2.7.3.tar.gz";
       md5 = "b9dffd2f3b43d673802fe857c8445b1a";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   Mako = super.buildPythonPackage {
     name = "Mako-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [MarkupSafe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/8e/a4/aa56533ecaa5f22ca92428f74e074d0c9337282933c722391902c8f9e0f8/Mako-1.0.1.tar.gz";
       md5 = "9f0aafd177b039ef67b90ea350497a54";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   Markdown = super.buildPythonPackage {
     name = "Markdown-2.6.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/62/8b/83658b5f6c220d5fcde9f9852d46ea54765d734cfbc5a9f4c05bfc36db4d/Markdown-2.6.2.tar.gz";
       md5 = "256d19afcc564dc4ce4c229bb762f7ae";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   MarkupSafe = super.buildPythonPackage {
     name = "MarkupSafe-0.23";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c0/41/bae1254e0396c0cc8cf1751cb7d9afc90a602353695af5952530482c963f/MarkupSafe-0.23.tar.gz";
       md5 = "f5ab3deee4c37cd6a922fb81e730da6e";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   MySQL-python = super.buildPythonPackage {
     name = "MySQL-python-1.2.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/a5/e9/51b544da85a36a68debe7a7091f068d802fc515a3a202652828c73453cad/MySQL-python-1.2.5.zip";
       md5 = "654f75b302db6ed8dc5a898c625e030c";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 ];
     };
   };
   Paste = super.buildPythonPackage {
     name = "Paste-2.0.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six];
     src = fetchurl {
       url = "https://pypi.python.org/packages/d5/8d/0f8ac40687b97ff3e07ebd1369be20bdb3f93864d2dc3c2ff542edb4ce50/Paste-2.0.2.tar.gz";
       md5 = "4bfc8a7eaf858f6309d2ac0f40fc951c";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   PasteDeploy = super.buildPythonPackage {
     name = "PasteDeploy-1.5.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/0f/90/8e20cdae206c543ea10793cbf4136eb9a8b3f417e04e40a29d72d9922cbd/PasteDeploy-1.5.2.tar.gz";
       md5 = "352b7205c78c8de4987578d19431af3b";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   PasteScript = super.buildPythonPackage {
     name = "PasteScript-1.7.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [Paste PasteDeploy];
     src = fetchurl {
       url = "https://pypi.python.org/packages/a5/05/fc60efa7c2f17a1dbaeccb2a903a1e90902d92b9d00eebabe3095829d806/PasteScript-1.7.5.tar.gz";
       md5 = "4c72d78dcb6bb993f30536842c16af4d";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   Pygments = super.buildPythonPackage {
     name = "Pygments-2.1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b8/67/ab177979be1c81bc99c8d0592ef22d547e70bb4c6815c383286ed5dec504/Pygments-2.1.3.tar.gz";
       md5 = "ed3fba2467c8afcda4d317e4ef2c6150";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   Pylons = super.buildPythonPackage {
     name = "Pylons-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [Routes WebHelpers Beaker Paste PasteDeploy PasteScript FormEncode simplejson decorator nose Mako WebError WebTest Tempita MarkupSafe WebOb];
     src = fetchurl {
       url = "https://pypi.python.org/packages/a2/69/b835a6bad00acbfeed3f33c6e44fa3f936efc998c795bfb15c61a79ecf62/Pylons-1.0.1.tar.gz";
       md5 = "6cb880d75fa81213192142b07a6e4915";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   Pyro4 = super.buildPythonPackage {
     name = "Pyro4-4.41";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [serpent];
     src = fetchurl {
       url = "https://pypi.python.org/packages/56/2b/89b566b4bf3e7f8ba790db2d1223852f8cb454c52cab7693dd41f608ca2a/Pyro4-4.41.tar.gz";
       md5 = "ed69e9bfafa9c06c049a87cb0c4c2b6c";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   Routes = super.buildPythonPackage {
     name = "Routes-1.13";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [repoze.lru];
     src = fetchurl {
       url = "https://pypi.python.org/packages/88/d3/259c3b3cde8837eb9441ab5f574a660e8a4acea8f54a078441d4d2acac1c/Routes-1.13.tar.gz";
       md5 = "d527b0ab7dd9172b1275a41f97448783";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   SQLAlchemy = super.buildPythonPackage {
     name = "SQLAlchemy-0.9.9";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/28/f7/1bbfd0d8597e8c358d5e15a166a486ad82fc5579b4e67b6ef7c05b1d182b/SQLAlchemy-0.9.9.tar.gz";
       md5 = "8a10a9bd13ed3336ef7333ac2cc679ff";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   Sphinx = super.buildPythonPackage {
     name = "Sphinx-1.2.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [Pygments docutils Jinja2];
     src = fetchurl {
       url = "https://pypi.python.org/packages/0a/50/34017e6efcd372893a416aba14b84a1a149fc7074537b0e9cb6ca7b7abe9/Sphinx-1.2.2.tar.gz";
       md5 = "3dc73ccaa8d0bfb2d62fb671b1f7e8a4";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   Tempita = super.buildPythonPackage {
     name = "Tempita-0.5.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/56/c8/8ed6eee83dbddf7b0fc64dd5d4454bc05e6ccaafff47991f73f2894d9ff4/Tempita-0.5.2.tar.gz";
       md5 = "4c2f17bb9d481821c41b6fbee904cea1";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   URLObject = super.buildPythonPackage {
     name = "URLObject-2.4.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/cb/b6/e25e58500f9caef85d664bec71ec67c116897bfebf8622c32cb75d1ca199/URLObject-2.4.0.tar.gz";
       md5 = "2ed819738a9f0a3051f31dc9924e3065";
     };
     meta = {
       license = [  ];
     };
   };
   WebError = super.buildPythonPackage {
     name = "WebError-0.10.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [WebOb Tempita Pygments Paste];
     src = fetchurl {
       url = "https://pypi.python.org/packages/35/76/e7e5c2ce7e9c7f31b54c1ff295a495886d1279a002557d74dd8957346a79/WebError-0.10.3.tar.gz";
       md5 = "84b9990b0baae6fd440b1e60cdd06f9a";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   WebHelpers = super.buildPythonPackage {
     name = "WebHelpers-1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [MarkupSafe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/ee/68/4d07672821d514184357f1552f2dad923324f597e722de3b016ca4f7844f/WebHelpers-1.3.tar.gz";
       md5 = "32749ffadfc40fea51075a7def32588b";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   WebHelpers2 = super.buildPythonPackage {
     name = "WebHelpers2-2.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [MarkupSafe six];
     src = fetchurl {
       url = "https://pypi.python.org/packages/ff/30/56342c6ea522439e3662427c8d7b5e5b390dff4ff2dc92d8afcb8ab68b75/WebHelpers2-2.0.tar.gz";
       md5 = "0f6b68d70c12ee0aed48c00b24da13d3";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   WebOb = super.buildPythonPackage {
     name = "WebOb-1.3.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/16/78/adfc0380b8a0d75b2d543fa7085ba98a573b1ae486d9def88d172b81b9fa/WebOb-1.3.1.tar.gz";
       md5 = "20918251c5726956ba8fef22d1556177";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   WebTest = super.buildPythonPackage {
     name = "WebTest-1.4.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [WebOb];
     src = fetchurl {
       url = "https://pypi.python.org/packages/51/3d/84fd0f628df10b30c7db87895f56d0158e5411206b721ca903cb51bfd948/WebTest-1.4.3.zip";
       md5 = "631ce728bed92c681a4020a36adbc353";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   Whoosh = super.buildPythonPackage {
     name = "Whoosh-2.7.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/1c/dc/2f0231ff3875ded36df8c1ab851451e51a237dc0e5a86d3d96036158da94/Whoosh-2.7.0.zip";
       md5 = "7abfd970f16fadc7311960f3fa0bc7a9";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.bsd2 ];
     };
   };
   alembic = super.buildPythonPackage {
     name = "alembic-0.8.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [SQLAlchemy Mako python-editor];
     src = fetchurl {
       url = "https://pypi.python.org/packages/ca/7e/299b4499b5c75e5a38c5845145ad24755bebfb8eec07a2e1c366b7181eeb/alembic-0.8.4.tar.gz";
       md5 = "5f95d8ee62b443f9b37eb5bee76c582d";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   amqplib = super.buildPythonPackage {
     name = "amqplib-1.0.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz";
       md5 = "5c92f17fbedd99b2b4a836d4352d1e2f";
     };
     meta = {
       license = [ { fullName = "LGPL"; } { fullName = "GNU Library or Lesser General Public License (LGPL)"; } ];
     };
   };
   anyjson = super.buildPythonPackage {
     name = "anyjson-0.3.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c3/4d/d4089e1a3dd25b46bebdb55a992b0797cff657b4477bc32ce28038fdecbc/anyjson-0.3.3.tar.gz";
       md5 = "2ea28d6ec311aeeebaf993cb3008b27c";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   appenlight-client = super.buildPythonPackage {
     name = "appenlight-client-0.6.14";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [WebOb requests];
     src = fetchurl {
       url = "https://pypi.python.org/packages/4d/e0/23fee3ebada8143f707e65c06bcb82992040ee64ea8355e044ed55ebf0c1/appenlight_client-0.6.14.tar.gz";
       md5 = "578c69b09f4356d898fff1199b98a95c";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "DFSG approved"; } ];
     };
   };
   authomatic = super.buildPythonPackage {
     name = "authomatic-0.1.0.post1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/08/1a/8a930461e604c2d5a7a871e1ac59fa82ccf994c32e807230c8d2fb07815a/Authomatic-0.1.0.post1.tar.gz";
       md5 = "be3f3ce08747d776aae6d6cc8dcb49a9";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   backport-ipaddress = super.buildPythonPackage {
     name = "backport-ipaddress-0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/d3/30/54c6dab05a4dec44db25ff309f1fbb6b7a8bde3f2bade38bb9da67bbab8f/backport_ipaddress-0.1.tar.gz";
       md5 = "9c1f45f4361f71b124d7293a60006c05";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   bottle = super.buildPythonPackage {
     name = "bottle-0.12.8";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/52/df/e4a408f3a7af396d186d4ecd3b389dd764f0f943b4fa8d257bfe7b49d343/bottle-0.12.8.tar.gz";
       md5 = "13132c0a8f607bf860810a6ee9064c5b";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   bumpversion = super.buildPythonPackage {
     name = "bumpversion-0.5.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/14/41/8c9da3549f8e00c84f0432c3a8cf8ed6898374714676aab91501d48760db/bumpversion-0.5.3.tar.gz";
       md5 = "c66a3492eafcf5ad4b024be9fca29820";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   celery = super.buildPythonPackage {
     name = "celery-2.2.10";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [python-dateutil anyjson kombu pyparsing];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b1/64/860fd50e45844c83442e7953effcddeff66b2851d90b2d784f7201c111b8/celery-2.2.10.tar.gz";
       md5 = "898bc87e54f278055b561316ba73e222";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   click = super.buildPythonPackage {
     name = "click-5.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b7/34/a496632c4fb6c1ee76efedf77bb8d28b29363d839953d95095b12defe791/click-5.1.tar.gz";
       md5 = "9c5323008cccfe232a8b161fc8196d41";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   colander = super.buildPythonPackage {
     name = "colander-1.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [translationstring iso8601];
     src = fetchurl {
       url = "https://pypi.python.org/packages/14/23/c9ceba07a6a1dc0eefbb215fc0dc64aabc2b22ee756bc0f0c13278fa0887/colander-1.2.tar.gz";
       md5 = "83db21b07936a0726e588dae1914b9ed";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   configobj = super.buildPythonPackage {
     name = "configobj-5.0.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six];
     src = fetchurl {
       url = "https://pypi.python.org/packages/64/61/079eb60459c44929e684fa7d9e2fdca403f67d64dd9dbac27296be2e0fab/configobj-5.0.6.tar.gz";
       md5 = "e472a3a1c2a67bb0ec9b5d54c13a47d6";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   cov-core = super.buildPythonPackage {
     name = "cov-core-1.15.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [coverage];
     src = fetchurl {
       url = "https://pypi.python.org/packages/4b/87/13e75a47b4ba1be06f29f6d807ca99638bedc6b57fa491cd3de891ca2923/cov-core-1.15.0.tar.gz";
       md5 = "f519d4cb4c4e52856afb14af52919fe6";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   coverage = super.buildPythonPackage {
     name = "coverage-3.7.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/09/4f/89b06c7fdc09687bca507dc411c342556ef9c5a3b26756137a4878ff19bf/coverage-3.7.1.tar.gz";
       md5 = "c47b36ceb17eaff3ecfab3bcd347d0df";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   cssselect = super.buildPythonPackage {
     name = "cssselect-0.9.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/aa/e5/9ee1460d485b94a6d55732eb7ad5b6c084caf73dd6f9cb0bb7d2a78fafe8/cssselect-0.9.1.tar.gz";
       md5 = "c74f45966277dc7a0f768b9b0f3522ac";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   decorator = super.buildPythonPackage {
     name = "decorator-3.4.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/35/3a/42566eb7a2cbac774399871af04e11d7ae3fc2579e7dae85213b8d1d1c57/decorator-3.4.2.tar.gz";
       md5 = "9e0536870d2b83ae27d58dbf22582f4d";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   docutils = super.buildPythonPackage {
     name = "docutils-0.12";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/37/38/ceda70135b9144d84884ae2fc5886c6baac4edea39550f28bcd144c1234d/docutils-0.12.tar.gz";
       md5 = "4622263b62c5c771c03502afa3157768";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.publicDomain pkgs.lib.licenses.gpl1 { fullName = "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)"; } pkgs.lib.licenses.psfl ];
     };
   };
   dogpile.cache = super.buildPythonPackage {
     name = "dogpile.cache-0.6.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [dogpile.core];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f6/a0/6f2142c58c6588d17c734265b103ae1cd0741e1681dd9483a63f22033375/dogpile.cache-0.6.1.tar.gz";
       md5 = "35d7fb30f22bbd0685763d894dd079a9";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   dogpile.core = super.buildPythonPackage {
     name = "dogpile.core-0.4.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/0e/77/e72abc04c22aedf874301861e5c1e761231c288b5de369c18be8f4b5c9bb/dogpile.core-0.4.1.tar.gz";
       md5 = "01cb19f52bba3e95c9b560f39341f045";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   dulwich = super.buildPythonPackage {
     name = "dulwich-0.12.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/6f/04/fbe561b6d45c0ec758330d5b7f5ba4b6cb4f1ca1ab49859d2fc16320da75/dulwich-0.12.0.tar.gz";
       md5 = "f3a8a12bd9f9dd8c233e18f3d49436fa";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl2Plus ];
     };
   };
   ecdsa = super.buildPythonPackage {
     name = "ecdsa-0.11";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/6c/3f/92fe5dcdcaa7bd117be21e5520c9a54375112b66ec000d209e9e9519fad1/ecdsa-0.11.tar.gz";
       md5 = "8ef586fe4dbb156697d756900cb41d7c";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   elasticsearch = super.buildPythonPackage {
     name = "elasticsearch-2.3.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [urllib3];
     src = fetchurl {
       url = "https://pypi.python.org/packages/10/35/5fd52c5f0b0ee405ed4b5195e8bce44c5e041787680dc7b94b8071cac600/elasticsearch-2.3.0.tar.gz";
       md5 = "2550f3b51629cf1ef9636608af92c340";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   elasticsearch-dsl = super.buildPythonPackage {
     name = "elasticsearch-dsl-2.0.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six python-dateutil elasticsearch];
     src = fetchurl {
       url = "https://pypi.python.org/packages/4e/5d/e788ae8dbe2ff4d13426db0a027533386a5c276c77a2654dc0e2007ce04a/elasticsearch-dsl-2.0.0.tar.gz";
       md5 = "4cdfec81bb35383dd3b7d02d7dc5ee68";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   flake8 = super.buildPythonPackage {
     name = "flake8-2.4.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pyflakes pep8 mccabe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/8f/b5/9a73c66c7dba273bac8758398f060c008a25f3e84531063b42503b5d0a95/flake8-2.4.1.tar.gz";
       md5 = "ed45d3db81a3b7c88bd63c6e37ca1d65";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   future = super.buildPythonPackage {
     name = "future-0.14.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/83/80/8ef3a11a15f8eaafafa0937b20c1b3f73527e69ab6b3fa1cf94a5a96aabb/future-0.14.3.tar.gz";
       md5 = "e94079b0bd1fc054929e8769fc0f6083";
     };
     meta = {
       license = [ { fullName = "OSI Approved"; } pkgs.lib.licenses.mit ];
     };
   };
   futures = super.buildPythonPackage {
     name = "futures-3.0.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f8/e7/fc0fcbeb9193ba2d4de00b065e7fd5aecd0679e93ce95a07322b2b1434f4/futures-3.0.2.tar.gz";
       md5 = "42aaf1e4de48d6e871d77dc1f9d96d5a";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   gnureadline = super.buildPythonPackage {
     name = "gnureadline-6.3.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/3a/ee/2c3f568b0a74974791ac590ec742ef6133e2fbd287a074ba72a53fa5e97c/gnureadline-6.3.3.tar.gz";
       md5 = "c4af83c9a3fbeac8f2da9b5a7c60e51c";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 ];
     };
   };
   gprof2dot = super.buildPythonPackage {
     name = "gprof2dot-2015.12.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b9/34/7bf93c1952d40fa5c95ad963f4d8344b61ef58558632402eca18e6c14127/gprof2dot-2015.12.1.tar.gz";
       md5 = "e23bf4e2f94db032750c193384b4165b";
     };
     meta = {
       license = [ { fullName = "LGPL"; } ];
     };
   };
   gunicorn = super.buildPythonPackage {
     name = "gunicorn-19.6.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/84/ce/7ea5396efad1cef682bbc4068e72a0276341d9d9d0f501da609fab9fcb80/gunicorn-19.6.0.tar.gz";
       md5 = "338e5e8a83ea0f0625f768dba4597530";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   infrae.cache = super.buildPythonPackage {
     name = "infrae.cache-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [Beaker repoze.lru];
     src = fetchurl {
       url = "https://pypi.python.org/packages/bb/f0/e7d5e984cf6592fd2807dc7bc44a93f9d18e04e6a61f87fdfb2622422d74/infrae.cache-1.0.1.tar.gz";
       md5 = "b09076a766747e6ed2a755cc62088e32";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
   invoke = super.buildPythonPackage {
     name = "invoke-0.13.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/47/bf/d07ef52fa1ac645468858bbac7cb95b246a972a045e821493d17d89c81be/invoke-0.13.0.tar.gz";
       md5 = "c0d1ed4bfb34eaab551662d8cfee6540";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   ipdb = super.buildPythonPackage {
     name = "ipdb-0.8";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [ipython];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f0/25/d7dd430ced6cd8dc242a933c8682b5dbf32eb4011d82f87e34209e5ec845/ipdb-0.8.zip";
       md5 = "96dca0712efa01aa5eaf6b22071dd3ed";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 ];
     };
   };
   ipython = super.buildPythonPackage {
     name = "ipython-3.1.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/06/91/120c0835254c120af89f066afaabf81289bc2726c1fc3ca0555df6882f58/ipython-3.1.0.tar.gz";
       md5 = "a749d90c16068687b0ec45a27e72ef8f";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   iso8601 = super.buildPythonPackage {
     name = "iso8601-0.1.11";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c0/75/c9209ee4d1b5975eb8c2cba4428bde6b61bd55664a98290dd015cdb18e98/iso8601-0.1.11.tar.gz";
       md5 = "b06d11cd14a64096f907086044f0fe38";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   itsdangerous = super.buildPythonPackage {
     name = "itsdangerous-0.24";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/dc/b4/a60bcdba945c00f6d608d8975131ab3f25b22f2bcfe1dab221165194b2d4/itsdangerous-0.24.tar.gz";
       md5 = "a3d55aa79369aef5345c036a8a26307f";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   kombu = super.buildPythonPackage {
     name = "kombu-1.5.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [anyjson amqplib];
     src = fetchurl {
       url = "https://pypi.python.org/packages/19/53/74bf2a624644b45f0850a638752514fc10a8e1cbd738f10804951a6df3f5/kombu-1.5.1.tar.gz";
       md5 = "50662f3c7e9395b3d0721fb75d100b63";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   lxml = super.buildPythonPackage {
     name = "lxml-3.4.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/63/c7/4f2a2a4ad6c6fa99b14be6b3c1cece9142e2d915aa7c43c908677afc8fa4/lxml-3.4.4.tar.gz";
       md5 = "a9a65972afc173ec7a39c585f4eea69c";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
-  marshmallow = super.buildPythonPackage {
-    name = "marshmallow-2.8.0";
-    buildInputs = with self; [];
-    doCheck = false;
-    propagatedBuildInputs = with self; [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/4f/64/9393d77847d86981c84b88bbea627d30ff71b5ab1402636b366f73737817/marshmallow-2.8.0.tar.gz";
-      md5 = "204513fc123a3d9bdd7b63b9747f02e6";
-    };
-    meta = {
-      license = [ pkgs.lib.licenses.mit ];
-    };
-  };
   mccabe = super.buildPythonPackage {
     name = "mccabe-0.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c9/2e/75231479e11a906b64ac43bad9d0bb534d00080b18bdca8db9da46e1faf7/mccabe-0.3.tar.gz";
       md5 = "81640948ff226f8c12b3277059489157";
     };
     meta = {
       license = [ { fullName = "Expat license"; } pkgs.lib.licenses.mit ];
     };
   };
   meld3 = super.buildPythonPackage {
     name = "meld3-1.0.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/45/a0/317c6422b26c12fe0161e936fc35f36552069ba8e6f7ecbd99bbffe32a5f/meld3-1.0.2.tar.gz";
       md5 = "3ccc78cd79cffd63a751ad7684c02c91";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   mock = super.buildPythonPackage {
     name = "mock-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/15/45/30273ee91feb60dabb8fbb2da7868520525f02cf910279b3047182feed80/mock-1.0.1.zip";
       md5 = "869f08d003c289a97c1a6610faf5e913";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   msgpack-python = super.buildPythonPackage {
     name = "msgpack-python-0.4.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/15/ce/ff2840885789ef8035f66cd506ea05bdb228340307d5e71a7b1e3f82224c/msgpack-python-0.4.6.tar.gz";
       md5 = "8b317669314cf1bc881716cccdaccb30";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   nose = super.buildPythonPackage {
     name = "nose-1.3.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/70/c7/469e68148d17a0d3db5ed49150242fd70a74a8147b8f3f8b87776e028d99/nose-1.3.6.tar.gz";
       md5 = "0ca546d81ca8309080fc80cb389e7a16";
     };
     meta = {
       license = [ { fullName = "GNU Library or Lesser General Public License (LGPL)"; } { fullName = "GNU LGPL"; } ];
     };
   };
   objgraph = super.buildPythonPackage {
     name = "objgraph-2.0.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/d7/33/ace750b59247496ed769b170586c5def7202683f3d98e737b75b767ff29e/objgraph-2.0.0.tar.gz";
       md5 = "25b0d5e5adc74aa63ead15699614159c";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   packaging = super.buildPythonPackage {
     name = "packaging-15.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/24/c4/185da1304f07047dc9e0c46c31db75c0351bd73458ac3efad7da3dbcfbe1/packaging-15.2.tar.gz";
       md5 = "c16093476f6ced42128bf610e5db3784";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   paramiko = super.buildPythonPackage {
     name = "paramiko-1.15.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pycrypto ecdsa];
     src = fetchurl {
       url = "https://pypi.python.org/packages/04/2b/a22d2a560c1951abbbf95a0628e245945565f70dc082d9e784666887222c/paramiko-1.15.1.tar.gz";
       md5 = "48c274c3f9b1282932567b21f6acf3b5";
     };
     meta = {
       license = [ { fullName = "LGPL"; } { fullName = "GNU Library or Lesser General Public License (LGPL)"; } ];
     };
   };
   pep8 = super.buildPythonPackage {
     name = "pep8-1.5.7";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/8b/de/259f5e735897ada1683489dd514b2a1c91aaa74e5e6b68f80acf128a6368/pep8-1.5.7.tar.gz";
       md5 = "f6adbdd69365ecca20513c709f9b7c93";
     };
     meta = {
       license = [ { fullName = "Expat license"; } pkgs.lib.licenses.mit ];
     };
   };
   psutil = super.buildPythonPackage {
     name = "psutil-2.2.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/df/47/ee54ef14dd40f8ce831a7581001a5096494dc99fe71586260ca6b531fe86/psutil-2.2.1.tar.gz";
       md5 = "1a2b58cd9e3a53528bb6148f0c4d5244";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   psycopg2 = super.buildPythonPackage {
     name = "psycopg2-2.6.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/86/fd/cc8315be63a41fe000cce20482a917e874cdc1151e62cb0141f5e55f711e/psycopg2-2.6.1.tar.gz";
       md5 = "842b44f8c95517ed5b792081a2370da1";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 { fullName = "GNU Library or Lesser General Public License (LGPL)"; } { fullName = "LGPL with exceptions or ZPL"; } ];
     };
   };
   py = super.buildPythonPackage {
     name = "py-1.4.29";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/2a/bc/a1a4a332ac10069b8e5e25136a35e08a03f01fd6ab03d819889d79a1fd65/py-1.4.29.tar.gz";
       md5 = "c28e0accba523a29b35a48bb703fb96c";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   py-bcrypt = super.buildPythonPackage {
     name = "py-bcrypt-0.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/68/b1/1c3068c5c4d2e35c48b38dcc865301ebfdf45f54507086ac65ced1fd3b3d/py-bcrypt-0.4.tar.gz";
       md5 = "dd8b367d6b716a2ea2e72392525f4e36";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   py-gfm = super.buildPythonPackage {
     name = "py-gfm-0.1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools Markdown];
     src = fetchurl {
       url = "https://pypi.python.org/packages/12/e4/6b3d8678da04f97d7490d8264d8de51c2dc9fb91209ccee9c515c95e14c5/py-gfm-0.1.3.tar.gz";
       md5 = "e588d9e69640a241b97e2c59c22527a6";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   pycrypto = super.buildPythonPackage {
     name = "pycrypto-2.6.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/60/db/645aa9af249f059cc3a368b118de33889219e0362141e75d4eaf6f80f163/pycrypto-2.6.1.tar.gz";
       md5 = "55a61a054aa66812daf5161a0d5d7eda";
     };
     meta = {
       license = [ pkgs.lib.licenses.publicDomain ];
     };
   };
   pycurl = super.buildPythonPackage {
     name = "pycurl-7.19.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/6c/48/13bad289ef6f4869b1d8fc11ae54de8cfb3cc4a2eb9f7419c506f763be46/pycurl-7.19.5.tar.gz";
       md5 = "47b4eac84118e2606658122104e62072";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit { fullName = "LGPL/MIT"; } { fullName = "GNU Library or Lesser General Public License (LGPL)"; } ];
     };
   };
   pyflakes = super.buildPythonPackage {
     name = "pyflakes-0.8.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/75/22/a90ec0252f4f87f3ffb6336504de71fe16a49d69c4538dae2f12b9360a38/pyflakes-0.8.1.tar.gz";
       md5 = "905fe91ad14b912807e8fdc2ac2e2c23";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pyparsing = super.buildPythonPackage {
     name = "pyparsing-1.5.7";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/2e/26/e8fb5b4256a5f5036be7ce115ef8db8d06bc537becfbdc46c6af008314ee/pyparsing-1.5.7.zip";
       md5 = "b86854857a368d6ccb4d5b6e76d0637f";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pyramid = super.buildPythonPackage {
     name = "pyramid-1.6.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools WebOb repoze.lru zope.interface zope.deprecation venusian translationstring PasteDeploy];
     src = fetchurl {
       url = "https://pypi.python.org/packages/30/b3/fcc4a2a4800cbf21989e00454b5828cf1f7fe35c63e0810b350e56d4c475/pyramid-1.6.1.tar.gz";
       md5 = "b18688ff3cc33efdbb098a35b45dd122";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   pyramid-beaker = super.buildPythonPackage {
     name = "pyramid-beaker-0.8";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pyramid Beaker];
     src = fetchurl {
       url = "https://pypi.python.org/packages/d9/6e/b85426e00fd3d57f4545f74e1c3828552d8700f13ededeef9233f7bca8be/pyramid_beaker-0.8.tar.gz";
       md5 = "22f14be31b06549f80890e2c63a93834";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   pyramid-debugtoolbar = super.buildPythonPackage {
     name = "pyramid-debugtoolbar-2.4.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pyramid pyramid-mako repoze.lru Pygments];
     src = fetchurl {
       url = "https://pypi.python.org/packages/89/00/ed5426ee41ed747ba3ffd30e8230841a6878286ea67d480b1444d24f06a2/pyramid_debugtoolbar-2.4.2.tar.gz";
       md5 = "073ea67086cc4bd5decc3a000853642d";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } pkgs.lib.licenses.bsdOriginal ];
     };
   };
   pyramid-jinja2 = super.buildPythonPackage {
     name = "pyramid-jinja2-2.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pyramid zope.deprecation Jinja2 MarkupSafe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/a1/80/595e26ffab7deba7208676b6936b7e5a721875710f982e59899013cae1ed/pyramid_jinja2-2.5.tar.gz";
       md5 = "07cb6547204ac5e6f0b22a954ccee928";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   pyramid-mako = super.buildPythonPackage {
     name = "pyramid-mako-1.0.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pyramid Mako];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f1/92/7e69bcf09676d286a71cb3bbb887b16595b96f9ba7adbdc239ffdd4b1eb9/pyramid_mako-1.0.2.tar.gz";
       md5 = "ee25343a97eb76bd90abdc2a774eb48a";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   pysqlite = super.buildPythonPackage {
     name = "pysqlite-2.6.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/5c/a6/1c429cd4c8069cf4bfbd0eb4d592b3f4042155a8202df83d7e9b93aa3dc2/pysqlite-2.6.3.tar.gz";
       md5 = "7ff1cedee74646b50117acff87aa1cfa";
     };
     meta = {
       license = [ { fullName = "zlib/libpng License"; } { fullName = "zlib/libpng license"; } ];
     };
   };
   pytest = super.buildPythonPackage {
     name = "pytest-2.8.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [py];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b1/3d/d7ea9b0c51e0cacded856e49859f0a13452747491e842c236bbab3714afe/pytest-2.8.5.zip";
       md5 = "8493b06f700862f1294298d6c1b715a9";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pytest-catchlog = super.buildPythonPackage {
     name = "pytest-catchlog-1.2.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [py pytest];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f2/2b/2faccdb1a978fab9dd0bf31cca9f6847fbe9184a0bdcc3011ac41dd44191/pytest-catchlog-1.2.2.zip";
       md5 = "09d890c54c7456c818102b7ff8c182c8";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pytest-cov = super.buildPythonPackage {
     name = "pytest-cov-1.8.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [py pytest coverage cov-core];
     src = fetchurl {
       url = "https://pypi.python.org/packages/11/4b/b04646e97f1721878eb21e9f779102d84dd044d324382263b1770a3e4838/pytest-cov-1.8.1.tar.gz";
       md5 = "76c778afa2494088270348be42d759fc";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pytest-profiling = super.buildPythonPackage {
     name = "pytest-profiling-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six pytest gprof2dot];
     src = fetchurl {
       url = "https://pypi.python.org/packages/d8/67/8ffab73406e22870e07fa4dc8dce1d7689b26dba8efd00161c9b6fc01ec0/pytest-profiling-1.0.1.tar.gz";
       md5 = "354404eb5b3fd4dc5eb7fffbb3d9b68b";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pytest-runner = super.buildPythonPackage {
     name = "pytest-runner-2.7.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/99/6b/c4ff4418d3424d4475b7af60724fd4a5cdd91ed8e489dc9443281f0052bc/pytest-runner-2.7.1.tar.gz";
       md5 = "e56f0bc8d79a6bd91772b44ef4215c7e";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pytest-timeout = super.buildPythonPackage {
     name = "pytest-timeout-0.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pytest];
     src = fetchurl {
       url = "https://pypi.python.org/packages/24/48/5f6bd4b8026a26e1dd427243d560a29a0f1b24a5c7cffca4bf049a7bb65b/pytest-timeout-0.4.tar.gz";
       md5 = "03b28aff69cbbfb959ed35ade5fde262";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit { fullName = "DFSG approved"; } ];
     };
   };
   python-dateutil = super.buildPythonPackage {
     name = "python-dateutil-1.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b4/7c/df59c89a753eb33c7c44e1dd42de0e9bc2ccdd5a4d576e0bfad97cc280cb/python-dateutil-1.5.tar.gz";
       md5 = "0dcb1de5e5cad69490a3b6ab63f0cfa5";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   python-editor = super.buildPythonPackage {
     name = "python-editor-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/2b/c0/df7b87d5cf016f82eab3b05cd35f53287c1178ad8c42bfb6fa61b89b22f6/python-editor-1.0.1.tar.gz";
       md5 = "e1fa63535b40e022fa4fd646fd8b511a";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   python-ldap = super.buildPythonPackage {
     name = "python-ldap-2.4.19";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/42/81/1b64838c82e64f14d4e246ff00b52e650a35c012551b891ada2b85d40737/python-ldap-2.4.19.tar.gz";
       md5 = "b941bf31d09739492aa19ef679e94ae3";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   python-memcached = super.buildPythonPackage {
     name = "python-memcached-1.57";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six];
     src = fetchurl {
       url = "https://pypi.python.org/packages/52/9d/eebc0dcbc5c7c66840ad207dfc1baa376dadb74912484bff73819cce01e6/python-memcached-1.57.tar.gz";
       md5 = "de21f64b42b2d961f3d4ad7beb5468a1";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   python-pam = super.buildPythonPackage {
     name = "python-pam-1.8.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/de/8c/f8f5d38b4f26893af267ea0b39023d4951705ab0413a39e0cf7cf4900505/python-pam-1.8.2.tar.gz";
       md5 = "db71b6b999246fb05d78ecfbe166629d";
     };
     meta = {
       license = [ { fullName = "License :: OSI Approved :: MIT License"; } pkgs.lib.licenses.mit ];
     };
   };
   pytz = super.buildPythonPackage {
     name = "pytz-2015.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/7e/1a/f43b5c92df7b156822030fed151327ea096bcf417e45acc23bd1df43472f/pytz-2015.4.zip";
       md5 = "233f2a2b370d03f9b5911700cc9ebf3c";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pyzmq = super.buildPythonPackage {
     name = "pyzmq-14.6.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/8a/3b/5463d5a9d712cd8bbdac335daece0d69f6a6792da4e3dd89956c0db4e4e6/pyzmq-14.6.0.tar.gz";
       md5 = "395b5de95a931afa5b14c9349a5b8024";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "LGPL+BSD"; } { fullName = "GNU Library or Lesser General Public License (LGPL)"; } ];
     };
   };
   recaptcha-client = super.buildPythonPackage {
     name = "recaptcha-client-1.0.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/0a/ea/5f2fbbfd894bdac1c68ef8d92019066cfcf9fbff5fe3d728d2b5c25c8db4/recaptcha-client-1.0.6.tar.gz";
       md5 = "74228180f7e1fb76c4d7089160b0d919";
     };
     meta = {
       license = [ { fullName = "MIT/X11"; } ];
     };
   };
   repoze.lru = super.buildPythonPackage {
     name = "repoze.lru-0.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/6e/1e/aa15cc90217e086dc8769872c8778b409812ff036bf021b15795638939e4/repoze.lru-0.6.tar.gz";
       md5 = "2c3b64b17a8e18b405f55d46173e14dd";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   requests = super.buildPythonPackage {
     name = "requests-2.9.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f9/6d/07c44fb1ebe04d069459a189e7dab9e4abfe9432adcd4477367c25332748/requests-2.9.1.tar.gz";
       md5 = "0b7f480d19012ec52bab78292efd976d";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   rhodecode-enterprise-ce = super.buildPythonPackage {
     name = "rhodecode-enterprise-ce-4.3.0";
     buildInputs = with self; [WebTest configobj cssselect flake8 lxml mock pytest pytest-cov pytest-runner];
     doCheck = true;
-    propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments Pylons Pyro4 Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery colander decorator docutils gunicorn infrae.cache ipython iso8601 kombu marshmallow msgpack-python packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt];
+    propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments Pylons Pyro4 Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery colander decorator docutils gunicorn infrae.cache ipython iso8601 kombu msgpack-python packaging psycopg2 py-gfm pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson waitress zope.cachedescriptors dogpile.cache dogpile.core psutil py-bcrypt];
     src = ./.;
     meta = {
       license = [ { fullName = "AGPLv3, and Commercial License"; } ];
     };
   };
   rhodecode-tools = super.buildPythonPackage {
     name = "rhodecode-tools-0.8.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [click future six Mako MarkupSafe requests Whoosh elasticsearch elasticsearch-dsl];
     src = fetchurl {
       url = "https://code.rhodecode.com/rhodecode-tools-ce/archive/v0.8.3.zip";
       md5 = "9acdfd71b8ddf4056057065f37ab9ccb";
     };
     meta = {
       license = [ { fullName = "AGPLv3 and Proprietary"; } ];
     };
   };
   serpent = super.buildPythonPackage {
     name = "serpent-1.12";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/3b/19/1e0e83b47c09edaef8398655088036e7e67386b5c48770218ebb339fbbd5/serpent-1.12.tar.gz";
       md5 = "05869ac7b062828b34f8f927f0457b65";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   setproctitle = super.buildPythonPackage {
     name = "setproctitle-1.1.8";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/33/c3/ad367a4f4f1ca90468863ae727ac62f6edb558fc09a003d344a02cfc6ea6/setproctitle-1.1.8.tar.gz";
       md5 = "728f4c8c6031bbe56083a48594027edd";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   setuptools = super.buildPythonPackage {
     name = "setuptools-20.8.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c4/19/c1bdc88b53da654df43770f941079dbab4e4788c2dcb5658fb86259894c7/setuptools-20.8.1.zip";
       md5 = "fe58a5cac0df20bb83942b252a4b0543";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   setuptools-scm = super.buildPythonPackage {
     name = "setuptools-scm-1.11.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/cd/5f/e3a038292358058d83d764a47d09114aa5a8003ed4529518f9e580f1a94f/setuptools_scm-1.11.0.tar.gz";
       md5 = "4c5c896ba52e134bbc3507bac6400087";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   simplejson = super.buildPythonPackage {
     name = "simplejson-3.7.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/6d/89/7f13f099344eea9d6722779a1f165087cb559598107844b1ac5dbd831fb1/simplejson-3.7.2.tar.gz";
       md5 = "a5fc7d05d4cb38492285553def5d4b46";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit pkgs.lib.licenses.afl21 ];
     };
   };
   six = super.buildPythonPackage {
     name = "six-1.9.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/16/64/1dc5e5976b17466fd7d712e59cbe9fb1e18bec153109e5ba3ed6c9102f1a/six-1.9.0.tar.gz";
       md5 = "476881ef4012262dfc8adc645ee786c4";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   subprocess32 = super.buildPythonPackage {
     name = "subprocess32-3.2.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/28/8d/33ccbff51053f59ae6c357310cac0e79246bbed1d345ecc6188b176d72c3/subprocess32-3.2.6.tar.gz";
       md5 = "754c5ab9f533e764f931136974b618f1";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   supervisor = super.buildPythonPackage {
     name = "supervisor-3.3.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [meld3];
     src = fetchurl {
       url = "https://pypi.python.org/packages/44/80/d28047d120bfcc8158b4e41127706731ee6a3419c661e0a858fb0e7c4b2d/supervisor-3.3.0.tar.gz";
       md5 = "46bac00378d1eddb616752b990c67416";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   transifex-client = super.buildPythonPackage {
     name = "transifex-client-0.10";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f3/4e/7b925192aee656fb3e04fa6381c8b3dc40198047c3b4a356f6cfd642c809/transifex-client-0.10.tar.gz";
       md5 = "5549538d84b8eede6b254cd81ae024fa";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl2 ];
     };
   };
   translationstring = super.buildPythonPackage {
     name = "translationstring-1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/5e/eb/bee578cc150b44c653b63f5ebe258b5d0d812ddac12497e5f80fcad5d0b4/translationstring-1.3.tar.gz";
       md5 = "a4b62e0f3c189c783a1685b3027f7c90";
     };
     meta = {
       license = [ { fullName = "BSD-like (http://repoze.org/license.html)"; } ];
     };
   };
   trollius = super.buildPythonPackage {
     name = "trollius-1.0.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [futures];
     src = fetchurl {
       url = "https://pypi.python.org/packages/aa/e6/4141db437f55e6ee7a3fb69663239e3fde7841a811b4bef293145ad6c836/trollius-1.0.4.tar.gz";
       md5 = "3631a464d49d0cbfd30ab2918ef2b783";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   uWSGI = super.buildPythonPackage {
     name = "uWSGI-2.0.11.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/9b/78/918db0cfab0546afa580c1e565209c49aaf1476bbfe491314eadbe47c556/uwsgi-2.0.11.2.tar.gz";
       md5 = "1f02dcbee7f6f61de4b1fd68350cf16f";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl2 ];
     };
   };
   urllib3 = super.buildPythonPackage {
     name = "urllib3-1.16";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/3b/f0/e763169124e3f5db0926bc3dbfcd580a105f9ca44cf5d8e6c7a803c9f6b5/urllib3-1.16.tar.gz";
       md5 = "fcaab1c5385c57deeb7053d3d7d81d59";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   venusian = super.buildPythonPackage {
     name = "venusian-1.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/86/20/1948e0dfc4930ddde3da8c33612f6a5717c0b4bc28f591a5c5cf014dd390/venusian-1.0.tar.gz";
       md5 = "dccf2eafb7113759d60c86faf5538756";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   waitress = super.buildPythonPackage {
     name = "waitress-0.8.9";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/ee/65/fc9dee74a909a1187ca51e4f15ad9c4d35476e4ab5813f73421505c48053/waitress-0.8.9.tar.gz";
       md5 = "da3f2e62b3676be5dd630703a68e2a04";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
   wsgiref = super.buildPythonPackage {
     name = "wsgiref-0.1.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/41/9e/309259ce8dff8c596e8c26df86dbc4e848b9249fd36797fd60be456f03fc/wsgiref-0.1.2.zip";
       md5 = "29b146e6ebd0f9fb119fe321f7bcf6cb";
     };
     meta = {
       license = [ { fullName = "PSF or ZPL"; } ];
     };
   };
   zope.cachedescriptors = super.buildPythonPackage {
     name = "zope.cachedescriptors-4.0.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/40/33/694b6644c37f28553f4b9f20b3c3a20fb709a22574dff20b5bdffb09ecd5/zope.cachedescriptors-4.0.0.tar.gz";
       md5 = "8d308de8c936792c8e758058fcb7d0f0";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
   zope.deprecation = super.buildPythonPackage {
     name = "zope.deprecation-4.1.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c1/d3/3919492d5e57d8dd01b36f30b34fc8404a30577392b1eb817c303499ad20/zope.deprecation-4.1.2.tar.gz";
       md5 = "e9a663ded58f4f9f7881beb56cae2782";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
   zope.event = super.buildPythonPackage {
     name = "zope.event-4.0.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c1/29/91ba884d7d6d96691df592e9e9c2bfa57a47040ec1ff47eff18c85137152/zope.event-4.0.3.tar.gz";
       md5 = "9a3780916332b18b8b85f522bcc3e249";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
   zope.interface = super.buildPythonPackage {
     name = "zope.interface-4.1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/9d/81/2509ca3c6f59080123c1a8a97125eb48414022618cec0e64eb1313727bfe/zope.interface-4.1.3.tar.gz";
       md5 = "9ae3d24c0c7415deb249dd1a132f0f79";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
 
 ### Test requirements
 
 
 }

requirements.txt

@@ -1,152 +1,151 b''
 Babel==1.3
 Beaker==1.7.0
 CProfileV==1.0.6
 Fabric==1.10.0
 FormEncode==1.2.4
 Jinja2==2.7.3
 Mako==1.0.1
 Markdown==2.6.2
 MarkupSafe==0.23
 MySQL-python==1.2.5
 Paste==2.0.2
 PasteDeploy==1.5.2
 PasteScript==1.7.5
 Pygments==2.1.3
 
 # TODO: This version is not available on PyPI
 # Pylons==1.0.2.dev20160108
 Pylons==1.0.1
 
 # TODO: This version is not available, but newer ones are
 # Pyro4==4.35
 Pyro4==4.41
 
 # TODO: This should probably not be in here
 # -e hg+https://[e-mail]/johbo/rhodecode-fork@3a454bd1f17c0b2b2a951cf2b111e0320d7942a9#egg=RhodeCodeEnterprise-dev
 
 # TODO: This is not really a dependency, we should add it only
 # into the development environment, since there it is useful.
 # RhodeCodeVCSServer==3.9.0
 
 Routes==1.13
 SQLAlchemy==0.9.9
 Sphinx==1.2.2
 Tempita==0.5.2
 URLObject==2.4.0
 WebError==0.10.3
 
 # TODO: This is modified by us, needs a better integration. For now
 # using the latest version before.
 # WebHelpers==1.3.dev20150807
 WebHelpers==1.3
 
 WebHelpers2==2.0
 WebOb==1.3.1
 WebTest==1.4.3
 Whoosh==2.7.0
 alembic==0.8.4
 amqplib==1.0.2
 anyjson==0.3.3
 appenlight-client==0.6.14
 authomatic==0.1.0.post1;
 backport-ipaddress==0.1
 bottle==0.12.8
 bumpversion==0.5.3
 celery==2.2.10
 click==5.1
 colander==1.2
 configobj==5.0.6
 cov-core==1.15.0
 coverage==3.7.1
 cssselect==0.9.1
 decorator==3.4.2
 docutils==0.12
 dogpile.cache==0.6.1
 dogpile.core==0.4.1
 dulwich==0.12.0
 ecdsa==0.11
 flake8==2.4.1
 future==0.14.3
 futures==3.0.2
 gprof2dot==2015.12.1
 gunicorn==19.6.0
 
 # TODO: Needs subvertpy and blows up without Subversion headers,
 # actually we should not need this for Enterprise at all.
 # hgsubversion==1.8.2
 
 gnureadline==6.3.3
 infrae.cache==1.0.1
 invoke==0.13.0
 ipdb==0.8
 ipython==3.1.0
 iso8601==0.1.11
 itsdangerous==0.24
 kombu==1.5.1
 lxml==3.4.4
-marshmallow==2.8.0
 mccabe==0.3
 meld3==1.0.2
 mock==1.0.1
 msgpack-python==0.4.6
 nose==1.3.6
 objgraph==2.0.0
 packaging==15.2
 paramiko==1.15.1
 pep8==1.5.7
 psutil==2.2.1
 psycopg2==2.6.1
 py==1.4.29
 py-bcrypt==0.4
 py-gfm==0.1.3
 pycrypto==2.6.1
 pycurl==7.19.5
 pyflakes==0.8.1
 pyparsing==1.5.7
 pyramid==1.6.1
 pyramid-beaker==0.8
 pyramid-debugtoolbar==2.4.2
 pyramid-jinja2==2.5
 pyramid-mako==1.0.2
 pysqlite==2.6.3
 pytest==2.8.5
 pytest-runner==2.7.1
 pytest-catchlog==1.2.2
 pytest-cov==1.8.1
 pytest-profiling==1.0.1
 pytest-timeout==0.4
 python-dateutil==1.5
 python-ldap==2.4.19
 python-memcached==1.57
 python-pam==1.8.2
 pytz==2015.4
 pyzmq==14.6.0
 
 # TODO: This is not available in public
 # rc-testdata==0.2.0
 
 https://code.rhodecode.com/rhodecode-tools-ce/archive/v0.8.3.zip#md5=9acdfd71b8ddf4056057065f37ab9ccb
 
 
 recaptcha-client==1.0.6
 repoze.lru==0.6
 requests==2.9.1
 serpent==1.12
 setproctitle==1.1.8
 setuptools==20.8.1
 setuptools-scm==1.11.0
 simplejson==3.7.2
 six==1.9.0
 subprocess32==3.2.6
 supervisor==3.3.0
 transifex-client==0.10
 translationstring==1.3
 trollius==1.0.4
 uWSGI==2.0.11.2
 venusian==1.0
 waitress==0.8.9
 wsgiref==0.1.2
 zope.cachedescriptors==4.0.0
 zope.deprecation==4.1.2
 zope.event==4.0.3
 zope.interface==4.1.3

rhodecode/__init__.py

@@ -1,58 +1,58 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 
 RhodeCode, a web based repository management software
 versioning implementation: http://www.python.org/dev/peps/pep-0386/
 """
 
 import os
 import sys
 import platform
 
 VERSION = tuple(open(os.path.join(
     os.path.dirname(__file__), 'VERSION')).read().split('.'))
 
 BACKENDS = {
     'hg': 'Mercurial repository',
     'git': 'Git repository',
     'svn': 'Subversion repository',
 }
 
 CELERY_ENABLED = False
 CELERY_EAGER = False
 
 # link to config for pylons
 CONFIG = {}
 
 # Linked module for extensions
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 54  # defines current db version for migrations
+__dbversion__ = 55  # defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'
 __url__ = 'http://rhodecode.com'
 
 is_windows = __platform__ in ['Windows']
 is_unix = not is_windows
 is_test = False

rhodecode/admin/navigation.py

@@ -1,124 +1,126 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import logging
 import collections
 
 from pylons import url
 from zope.interface import implementer
 
 from rhodecode.admin.interfaces import IAdminNavigationRegistry
 from rhodecode.lib.utils import get_registry
 from rhodecode.translation import _
 
 
 log = logging.getLogger(__name__)
 
 NavListEntry = collections.namedtuple('NavListEntry', ['key', 'name', 'url'])
 
 
 class NavEntry(object):
     """
     Represents an entry in the admin navigation.
 
     :param key: Unique identifier used to store reference in an OrderedDict.
     :param name: Display name, usually a translation string.
     :param view_name: Name of the view, used generate the URL.
     :param pyramid: Indicator to use pyramid for URL generation. This should
         be removed as soon as we are fully migrated to pyramid.
     """
 
     def __init__(self, key, name, view_name, pyramid=False):
         self.key = key
         self.name = name
         self.view_name = view_name
         self.pyramid = pyramid
 
     def generate_url(self, request):
         if self.pyramid:
             if hasattr(request, 'route_path'):
                 return request.route_path(self.view_name)
             else:
                 # TODO: johbo: Remove this after migrating to pyramid.
                 # We need the pyramid request here to generate URLs to pyramid
                 # views from within pylons views.
                 from pyramid.threadlocal import get_current_request
                 pyramid_request = get_current_request()
                 return pyramid_request.route_path(self.view_name)
         else:
             return url(self.view_name)
 
 
 @implementer(IAdminNavigationRegistry)
 class NavigationRegistry(object):
 
     _base_entries = [
         NavEntry('global', _('Global'), 'admin_settings_global'),
         NavEntry('vcs', _('VCS'), 'admin_settings_vcs'),
         NavEntry('visual', _('Visual'), 'admin_settings_visual'),
         NavEntry('mapping', _('Remap and Rescan'), 'admin_settings_mapping'),
         NavEntry('issuetracker', _('Issue Tracker'),
                  'admin_settings_issuetracker'),
         NavEntry('email', _('Email'), 'admin_settings_email'),
         NavEntry('hooks', _('Hooks'), 'admin_settings_hooks'),
         NavEntry('search', _('Full Text Search'), 'admin_settings_search'),
+        NavEntry('integrations', _('Integrations'),
+                 'global_integrations_home', pyramid=True),
         NavEntry('system', _('System Info'), 'admin_settings_system'),
         NavEntry('open_source', _('Open Source Licenses'),
                  'admin_settings_open_source', pyramid=True),
         # TODO: marcink: we disable supervisor now until the supervisor stats
         # page is fixed in the nix configuration
         # NavEntry('supervisor', _('Supervisor'), 'admin_settings_supervisor'),
     ]
 
     _labs_entry = NavEntry('labs', _('Labs'),
                            'admin_settings_labs')
 
     def __init__(self, labs_active=False):
         self._registered_entries = collections.OrderedDict([
             (item.key, item) for item in self.__class__._base_entries
         ])
 
         if labs_active:
             self.add_entry(self._labs_entry)
 
     def add_entry(self, entry):
         self._registered_entries[entry.key] = entry
 
     def get_navlist(self, request):
         navlist = [NavListEntry(i.key, i.name, i.generate_url(request))
                    for i in self._registered_entries.values()]
         return navlist
 
 
 def navigation_registry(request):
     """
     Helper that returns the admin navigation registry.
     """
     pyramid_registry = get_registry(request)
     nav_registry = pyramid_registry.queryUtility(IAdminNavigationRegistry)
     return nav_registry
 
 
 def navigation_list(request):
     """
     Helper that returns the admin navigation as list of NavListEntry objects.
     """
     return navigation_registry(request).get_navlist(request)

rhodecode/config/middleware.py

@@ -1,387 +1,388 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Pylons middleware initialization
 """
 import logging
 
 from paste.registry import RegistryManager
 from paste.gzipper import make_gzip_middleware
 from pylons.wsgiapp import PylonsApp
 from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid.config import Configurator
 from pyramid.static import static_view
 from pyramid.settings import asbool, aslist
 from pyramid.wsgi import wsgiapp
 from pyramid.httpexceptions import HTTPError, HTTPInternalServerError
 import pyramid.httpexceptions as httpexceptions
 from pyramid.renderers import render_to_response, render
 from routes.middleware import RoutesMiddleware
 import routes.util
 
 import rhodecode
 from rhodecode.config import patches
 from rhodecode.config.environment import (
     load_environment, load_pyramid_environment)
 from rhodecode.lib.middleware import csrf
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.disable_vcs import DisableVCSPagesWrapper
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.middleware.vcs import VCSMiddleware
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
 
 
 log = logging.getLogger(__name__)
 
 
 def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     """Create a Pylons WSGI application and return it
 
     ``global_conf``
         The inherited configuration for this application. Normally from
         the [DEFAULT] section of the Paste ini file.
 
     ``full_stack``
         Whether or not this application provides a full WSGI stack (by
         default, meaning it handles its own exceptions and errors).
         Disable full_stack when this application is "managed" by
         another WSGI middleware.
 
     ``app_conf``
         The application's local configuration. Normally specified in
         the [app:<name>] section of the Paste ini file (where <name>
         defaults to main).
 
     """
     # Apply compatibility patches
     patches.kombu_1_5_1_python_2_7_11()
     patches.inspect_getargspec()
 
     # Configure the Pylons environment
     config = load_environment(global_conf, app_conf)
 
     # The Pylons WSGI app
     app = PylonsApp(config=config)
     if rhodecode.is_test:
         app = csrf.CSRFDetector(app)
 
     expected_origin = config.get('expected_origin')
     if expected_origin:
         # The API can be accessed from other Origins.
         app = csrf.OriginChecker(app, expected_origin,
                                  skip_urls=[routes.util.url_for('api')])
 
 
     if asbool(full_stack):
 
         # Appenlight monitoring and error handler
         app, appenlight_client = wrap_in_appenlight_if_enabled(app, config)
 
         # we want our low level middleware to get to the request ASAP. We don't
         # need any pylons stack middleware in them
         app = VCSMiddleware(app, config, appenlight_client)
 
     # Establish the Registry for this application
     app = RegistryManager(app)
 
     app.config = config
 
     return app
 
 
 def make_pyramid_app(global_config, **settings):
     """
     Constructs the WSGI application based on Pyramid and wraps the Pylons based
     application.
 
     Specials:
 
     * We migrate from Pylons to Pyramid. While doing this, we keep both
       frameworks functional. This involves moving some WSGI middlewares around
       and providing access to some data internals, so that the old code is
       still functional.
 
     * The application can also be integrated like a plugin via the call to
       `includeme`. This is accompanied with the other utility functions which
       are called. Changing this should be done with great care to not break
       cases when these fragments are assembled from another place.
 
     """
     # The edition string should be available in pylons too, so we add it here
     # before copying the settings.
     settings.setdefault('rhodecode.edition', 'Community Edition')
 
     # As long as our Pylons application does expect "unprepared" settings, make
     # sure that we keep an unmodified copy. This avoids unintentional change of
     # behavior in the old application.
     settings_pylons = settings.copy()
 
     sanitize_settings_and_apply_defaults(settings)
     config = Configurator(settings=settings)
     add_pylons_compat_data(config.registry, global_config, settings_pylons)
 
     load_pyramid_environment(global_config, settings)
 
     includeme(config)
     includeme_last(config)
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     return pyramid_app
 
 
 def add_pylons_compat_data(registry, global_config, settings):
     """
     Attach data to the registry to support the Pylons integration.
     """
     registry._pylons_compat_global_config = global_config
     registry._pylons_compat_settings = settings
 
 
 def webob_to_pyramid_http_response(webob_response):
     ResponseClass = httpexceptions.status_map[webob_response.status_int]
     pyramid_response = ResponseClass(webob_response.status)
     pyramid_response.status = webob_response.status
     pyramid_response.headers.update(webob_response.headers)
     if pyramid_response.headers['content-type'] == 'text/html':
         pyramid_response.headers['content-type'] = 'text/html; charset=UTF-8'
     return pyramid_response
 
 
 def error_handler(exception, request):
     # TODO: dan: replace the old pylons error controller with this
     from rhodecode.model.settings import SettingsModel
     from rhodecode.lib.utils2 import AttributeDict
 
     try:
         rc_config = SettingsModel().get_all_settings()
     except Exception:
         log.exception('failed to fetch settings')
         rc_config = {}
 
     base_response = HTTPInternalServerError()
     # prefer original exception for the response since it may have headers set
     if isinstance(exception, HTTPError):
         base_response = exception
 
     c = AttributeDict()
     c.error_message = base_response.status
     c.error_explanation = base_response.explanation or str(base_response)
     c.visual = AttributeDict()
 
     c.visual.rhodecode_support_url = (
         request.registry.settings.get('rhodecode_support_url') or
         request.route_url('rhodecode_support')
     )
     c.redirect_time = 0
     c.rhodecode_name = rc_config.get('rhodecode_title', '')
     if not c.rhodecode_name:
         c.rhodecode_name = 'Rhodecode'
 
     response = render_to_response(
         '/errors/error_document.html', {'c': c}, request=request,
         response=base_response)
 
     return response
 
 
 def includeme(config):
     settings = config.registry.settings
 
     if asbool(settings.get('appenlight', 'false')):
         config.include('appenlight_client.ext.pyramid_tween')
 
     # Includes which are required. The application would fail without them.
     config.include('pyramid_mako')
     config.include('pyramid_beaker')
     config.include('rhodecode.admin')
     config.include('rhodecode.authentication')
+    config.include('rhodecode.integrations')
     config.include('rhodecode.login')
     config.include('rhodecode.tweens')
     config.include('rhodecode.api')
     config.add_route(
         'rhodecode_support', 'https://rhodecode.com/help/', static=True)
 
     # Set the authorization policy.
     authz_policy = ACLAuthorizationPolicy()
     config.set_authorization_policy(authz_policy)
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')
 
     # plugin information
     config.registry.rhodecode_plugins = {}
 
     config.add_directive(
         'register_rhodecode_plugin', register_rhodecode_plugin)
     # include RhodeCode plugins
     includes = aslist(settings.get('rhodecode.includes', []))
     for inc in includes:
         config.include(inc)
 
     pylons_app = make_app(
         config.registry._pylons_compat_global_config,
         **config.registry._pylons_compat_settings)
     config.registry._pylons_compat_config = pylons_app.config
 
     pylons_app_as_view = wsgiapp(pylons_app)
 
     # Protect from VCS Server error related pages when server is not available
     vcs_server_enabled = asbool(settings.get('vcs.server.enable', 'true'))
     if not vcs_server_enabled:
         pylons_app_as_view = DisableVCSPagesWrapper(pylons_app_as_view)
 
 
     def pylons_app_with_error_handler(context, request):
         """
         Handle exceptions from rc pylons app:
 
         - old webob type exceptions get converted to pyramid exceptions
         - pyramid exceptions are passed to the error handler view
         """
         try:
             response = pylons_app_as_view(context, request)
             if 400 <= response.status_int <= 599: # webob type error responses
                 return error_handler(
                     webob_to_pyramid_http_response(response), request)
         except HTTPError as e: # pyramid type exceptions
             return error_handler(e, request)
         except Exception:
             if settings.get('debugtoolbar.enabled', False):
                 raise
             return error_handler(HTTPInternalServerError(), request)
         return response
 
     # This is the glue which allows us to migrate in chunks. By registering the
     # pylons based application as the "Not Found" view in Pyramid, we will
     # fallback to the old application each time the new one does not yet know
     # how to handle a request.
     config.add_notfound_view(pylons_app_with_error_handler)
 
     if settings.get('debugtoolbar.enabled', False):
         # if toolbar, then only http type exceptions get caught and rendered
         ExcClass = HTTPError
     else:
         # if no toolbar, then any exception gets caught and rendered
         ExcClass = Exception
     config.add_view(error_handler, context=ExcClass)
 
 
 def includeme_last(config):
     """
     The static file catchall needs to be last in the view configuration.
     """
     settings = config.registry.settings
 
     # Note: johbo: I would prefer to register a prefix for static files at some
     # point, e.g. move them under '_static/'. This would fully avoid that we
     # can have name clashes with a repository name. Imaging someone calling his
     # repo "css" ;-) Also having an external web server to serve out the static
     # files seems to be easier to set up if they have a common prefix.
     #
     # Example: config.add_static_view('_static', path='rhodecode:public')
     #
     # It might be an option to register both paths for a while and then migrate
     # over to the new location.
 
     # Serving static files with a catchall.
     if settings['static_files']:
         config.add_route('catchall_static', '/*subpath')
         config.add_view(
             static_view('rhodecode:public'), route_name='catchall_static')
 
 
 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
     """
     Apply outer WSGI middlewares around the application.
 
     Part of this has been moved up from the Pylons layer, so that the
     data is also available if old Pylons code is hit through an already ported
     view.
     """
     settings = config.registry.settings
 
     # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     pyramid_app = HttpsFixup(pyramid_app, settings)
 
     # Add RoutesMiddleware to support the pylons compatibility tween during
 
     # migration to pyramid.
     pyramid_app = RoutesMiddleware(
         pyramid_app, config.registry._pylons_compat_config['routes.map'])
 
     if asbool(settings.get('appenlight', 'false')):
         pyramid_app, _ = wrap_in_appenlight_if_enabled(
             pyramid_app, config.registry._pylons_compat_config)
 
     # TODO: johbo: Don't really see why we enable the gzip middleware when
     # serving static files, might be something that should have its own setting
     # as well?
     if settings['static_files']:
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
     return pyramid_app
 
 
 def sanitize_settings_and_apply_defaults(settings):
     """
     Applies settings defaults and does all type conversion.
 
     We would move all settings parsing and preparation into this place, so that
     we have only one place left which deals with this part. The remaining parts
     of the application would start to rely fully on well prepared settings.
 
     This piece would later be split up per topic to avoid a big fat monster
     function.
     """
 
     # Pyramid's mako renderer has to search in the templates folder so that the
     # old templates still work. Ported and new templates are expected to use
     # real asset specifications for the includes.
     mako_directories = settings.setdefault('mako.directories', [
         # Base templates of the original Pylons application
         'rhodecode:templates',
     ])
     log.debug(
         "Using the following Mako template directories: %s",
         mako_directories)
 
     # Default includes, possible to change as a user
     pyramid_includes = settings.setdefault('pyramid.includes', [
         'rhodecode.lib.middleware.request_wrapper',
     ])
     log.debug(
         "Using the following pyramid.includes: %s",
         pyramid_includes)
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
     settings.setdefault('rhodecode.api.url', '/_admin/api')
 
     _bool_setting(settings, 'vcs.server.enable', 'true')
     _bool_setting(settings, 'static_files', 'true')
     _bool_setting(settings, 'is_test', 'false')
 
     return settings
 
 
 def _bool_setting(settings, name, default):
     settings[name] = asbool(settings.get(name, default))

rhodecode/config/routing.py

@@ -1,1141 +1,1154 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Routes configuration
 
 The more specific and detailed routes should be defined first so they
 may take precedent over the more generic routes. For more information
 refer to the routes manual at http://routes.groovie.org/docs/
 
 IMPORTANT: if you change any routing here, make sure to take a look at lib/base.py
 and _route_name variable which uses some of stored naming here to do redirects.
 """
 import os
 import re
 from routes import Mapper
 
 from rhodecode.config import routing_links
 
 # prefix for non repository related links needs to be prefixed with `/`
 ADMIN_PREFIX = '/_admin'
 
 # Default requirements for URL parts
 URL_NAME_REQUIREMENTS = {
     # group name can have a slash in them, but they must not end with a slash
     'group_name': r'.*?[^/]',
     # repo names can have a slash in them, but they must not end with a slash
     'repo_name': r'.*?[^/]',
     # file path eats up everything at the end
     'f_path': r'.*',
     # reference types
     'source_ref_type': '(branch|book|tag|rev|\%\(source_ref_type\)s)',
     'target_ref_type': '(branch|book|tag|rev|\%\(target_ref_type\)s)',
 }
 
 
+def add_route_requirements(route_path, requirements):
+    """
+    Adds regex requirements to pyramid routes using a mapping dict
+
+    >>> add_route_requirements('/{action}/{id}', {'id': r'\d+'})
+    '/{action}/{id:\d+}'
+
+     """
+    for key, regex in requirements.items():
+        route_path = route_path.replace('{%s}' % key, '{%s:%s}' % (key, regex))
+    return route_path
+
+
 class JSRoutesMapper(Mapper):
     """
     Wrapper for routes.Mapper to make pyroutes compatible url definitions
     """
     _named_route_regex = re.compile(r'^[a-z-_0-9A-Z]+$')
     _argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')
     def __init__(self, *args, **kw):
         super(JSRoutesMapper, self).__init__(*args, **kw)
         self._jsroutes = []
 
     def connect(self, *args, **kw):
         """
         Wrapper for connect to take an extra argument jsroute=True
 
         :param jsroute: boolean, if True will add the route to the pyroutes list
         """
         if kw.pop('jsroute', False):
             if not self._named_route_regex.match(args[0]):
                 raise Exception('only named routes can be added to pyroutes')
             self._jsroutes.append(args[0])
 
         super(JSRoutesMapper, self).connect(*args, **kw)
 
     def _extract_route_information(self, route):
         """
         Convert a route into tuple(name, path, args), eg:
             ('user_profile', '/profile/%(username)s', ['username'])
         """
         routepath = route.routepath
         def replace(matchobj):
             if matchobj.group(1):
                 return "%%(%s)s" % matchobj.group(1).split(':')[0]
             else:
                 return "%%(%s)s" % matchobj.group(2)
 
         routepath = self._argument_prog.sub(replace, routepath)
         return (
             route.name,
             routepath,
             [(arg[0].split(':')[0] if arg[0] != '' else arg[1])
               for arg in self._argument_prog.findall(route.routepath)]
         )
 
     def jsroutes(self):
         """
         Return a list of pyroutes.js compatible routes
         """
         for route_name in self._jsroutes:
             yield self._extract_route_information(self._routenames[route_name])
 
 
 def make_map(config):
     """Create, configure and return the routes Mapper"""
     rmap = JSRoutesMapper(directory=config['pylons.paths']['controllers'],
                   always_scan=config['debug'])
     rmap.minimization = False
     rmap.explicit = False
 
     from rhodecode.lib.utils2 import str2bool
     from rhodecode.model import repo, repo_group
 
     def check_repo(environ, match_dict):
         """
         check for valid repository for proper 404 handling
 
         :param environ:
         :param match_dict:
         """
         repo_name = match_dict.get('repo_name')
 
         if match_dict.get('f_path'):
             # fix for multiple initial slashes that causes errors
             match_dict['f_path'] = match_dict['f_path'].lstrip('/')
         repo_model = repo.RepoModel()
         by_name_match = repo_model.get_by_repo_name(repo_name)
         # if we match quickly from database, short circuit the operation,
         # and validate repo based on the type.
         if by_name_match:
             return True
 
         by_id_match = repo_model.get_repo_by_id(repo_name)
         if by_id_match:
             repo_name = by_id_match.repo_name
             match_dict['repo_name'] = repo_name
             return True
 
         return False
 
     def check_group(environ, match_dict):
         """
         check for valid repository group path for proper 404 handling
 
         :param environ:
         :param match_dict:
         """
         repo_group_name = match_dict.get('group_name')
         repo_group_model = repo_group.RepoGroupModel()
         by_name_match = repo_group_model.get_by_group_name(repo_group_name)
         if by_name_match:
             return True
 
         return False
 
     def check_user_group(environ, match_dict):
         """
         check for valid user group for proper 404 handling
 
         :param environ:
         :param match_dict:
         """
         return True
 
     def check_int(environ, match_dict):
         return match_dict.get('id').isdigit()
 
 
     #==========================================================================
     # CUSTOM ROUTES HERE
     #==========================================================================
 
     # MAIN PAGE
     rmap.connect('home', '/', controller='home', action='index', jsroute=True)
     rmap.connect('goto_switcher_data', '/_goto_data', controller='home',
                  action='goto_switcher_data')
     rmap.connect('repo_list_data', '/_repos', controller='home',
                  action='repo_list_data')
 
     rmap.connect('user_autocomplete_data', '/_users', controller='home',
                  action='user_autocomplete_data', jsroute=True)
     rmap.connect('user_group_autocomplete_data', '/_user_groups', controller='home',
                  action='user_group_autocomplete_data')
 
     rmap.connect(
         'user_profile', '/_profiles/{username}', controller='users',
         action='user_profile')
 
     # TODO: johbo: Static links, to be replaced by our redirection mechanism
     rmap.connect('rst_help',
                  'http://docutils.sourceforge.net/docs/user/rst/quickref.html',
                  _static=True)
     rmap.connect('markdown_help',
                  'http://daringfireball.net/projects/markdown/syntax',
                  _static=True)
     rmap.connect('rhodecode_official', 'https://rhodecode.com', _static=True)
     rmap.connect('rhodecode_support', 'https://rhodecode.com/help/', _static=True)
     rmap.connect('rhodecode_translations', 'https://rhodecode.com/translate/enterprise', _static=True)
     # TODO: anderson - making this a static link since redirect won't play
     # nice with POST requests
     rmap.connect('enterprise_license_convert_from_old',
                  'https://rhodecode.com/u/license-upgrade',
                  _static=True)
 
     routing_links.connect_redirection_links(rmap)
 
     rmap.connect('ping', '%s/ping' % (ADMIN_PREFIX,), controller='home', action='ping')
     rmap.connect('error_test', '%s/error_test' % (ADMIN_PREFIX,), controller='home', action='error_test')
 
     # ADMIN REPOSITORY ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/repos') as m:
         m.connect('repos', '/repos',
                   action='create', conditions={'method': ['POST']})
         m.connect('repos', '/repos',
                   action='index', conditions={'method': ['GET']})
         m.connect('new_repo', '/create_repository', jsroute=True,
                   action='create_repository', conditions={'method': ['GET']})
         m.connect('/repos/{repo_name}',
                   action='update', conditions={'method': ['PUT'],
                                                'function': check_repo},
                   requirements=URL_NAME_REQUIREMENTS)
         m.connect('delete_repo', '/repos/{repo_name}',
                   action='delete', conditions={'method': ['DELETE']},
                   requirements=URL_NAME_REQUIREMENTS)
         m.connect('repo', '/repos/{repo_name}',
                   action='show', conditions={'method': ['GET'],
                                              'function': check_repo},
                   requirements=URL_NAME_REQUIREMENTS)
 
     # ADMIN REPOSITORY GROUPS ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/repo_groups') as m:
         m.connect('repo_groups', '/repo_groups',
                   action='create', conditions={'method': ['POST']})
         m.connect('repo_groups', '/repo_groups',
                   action='index', conditions={'method': ['GET']})
         m.connect('new_repo_group', '/repo_groups/new',
                   action='new', conditions={'method': ['GET']})
         m.connect('update_repo_group', '/repo_groups/{group_name}',
                   action='update', conditions={'method': ['PUT'],
                                                'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
 
         # EXTRAS REPO GROUP ROUTES
         m.connect('edit_repo_group', '/repo_groups/{group_name}/edit',
                   action='edit',
                   conditions={'method': ['GET'], 'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
         m.connect('edit_repo_group', '/repo_groups/{group_name}/edit',
                   action='edit',
                   conditions={'method': ['PUT'], 'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
 
         m.connect('edit_repo_group_advanced', '/repo_groups/{group_name}/edit/advanced',
                   action='edit_repo_group_advanced',
                   conditions={'method': ['GET'], 'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
         m.connect('edit_repo_group_advanced', '/repo_groups/{group_name}/edit/advanced',
                   action='edit_repo_group_advanced',
                   conditions={'method': ['PUT'], 'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
 
         m.connect('edit_repo_group_perms', '/repo_groups/{group_name}/edit/permissions',
                   action='edit_repo_group_perms',
                   conditions={'method': ['GET'], 'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
         m.connect('edit_repo_group_perms', '/repo_groups/{group_name}/edit/permissions',
                   action='update_perms',
                   conditions={'method': ['PUT'], 'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
 
         m.connect('delete_repo_group', '/repo_groups/{group_name}',
                   action='delete', conditions={'method': ['DELETE'],
                                                'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
 
     # ADMIN USER ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/users') as m:
         m.connect('users', '/users',
                   action='create', conditions={'method': ['POST']})
         m.connect('users', '/users',
                   action='index', conditions={'method': ['GET']})
         m.connect('new_user', '/users/new',
                   action='new', conditions={'method': ['GET']})
         m.connect('update_user', '/users/{user_id}',
                   action='update', conditions={'method': ['PUT']})
         m.connect('delete_user', '/users/{user_id}',
                   action='delete', conditions={'method': ['DELETE']})
         m.connect('edit_user', '/users/{user_id}/edit',
                   action='edit', conditions={'method': ['GET']})
         m.connect('user', '/users/{user_id}',
                   action='show', conditions={'method': ['GET']})
         m.connect('force_password_reset_user', '/users/{user_id}/password_reset',
                   action='reset_password', conditions={'method': ['POST']})
         m.connect('create_personal_repo_group', '/users/{user_id}/create_repo_group',
                   action='create_personal_repo_group', conditions={'method': ['POST']})
 
         # EXTRAS USER ROUTES
         m.connect('edit_user_advanced', '/users/{user_id}/edit/advanced',
                   action='edit_advanced', conditions={'method': ['GET']})
         m.connect('edit_user_advanced', '/users/{user_id}/edit/advanced',
                   action='update_advanced', conditions={'method': ['PUT']})
 
         m.connect('edit_user_auth_tokens', '/users/{user_id}/edit/auth_tokens',
                   action='edit_auth_tokens', conditions={'method': ['GET']})
         m.connect('edit_user_auth_tokens', '/users/{user_id}/edit/auth_tokens',
                   action='add_auth_token', conditions={'method': ['PUT']})
         m.connect('edit_user_auth_tokens', '/users/{user_id}/edit/auth_tokens',
                   action='delete_auth_token', conditions={'method': ['DELETE']})
 
         m.connect('edit_user_global_perms', '/users/{user_id}/edit/global_permissions',
                   action='edit_global_perms', conditions={'method': ['GET']})
         m.connect('edit_user_global_perms', '/users/{user_id}/edit/global_permissions',
                   action='update_global_perms', conditions={'method': ['PUT']})
 
         m.connect('edit_user_perms_summary', '/users/{user_id}/edit/permissions_summary',
                   action='edit_perms_summary', conditions={'method': ['GET']})
 
         m.connect('edit_user_emails', '/users/{user_id}/edit/emails',
                   action='edit_emails', conditions={'method': ['GET']})
         m.connect('edit_user_emails', '/users/{user_id}/edit/emails',
                   action='add_email', conditions={'method': ['PUT']})
         m.connect('edit_user_emails', '/users/{user_id}/edit/emails',
                   action='delete_email', conditions={'method': ['DELETE']})
 
         m.connect('edit_user_ips', '/users/{user_id}/edit/ips',
                   action='edit_ips', conditions={'method': ['GET']})
         m.connect('edit_user_ips', '/users/{user_id}/edit/ips',
                   action='add_ip', conditions={'method': ['PUT']})
         m.connect('edit_user_ips', '/users/{user_id}/edit/ips',
                   action='delete_ip', conditions={'method': ['DELETE']})
 
     # ADMIN USER GROUPS REST ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/user_groups') as m:
         m.connect('users_groups', '/user_groups',
                   action='create', conditions={'method': ['POST']})
         m.connect('users_groups', '/user_groups',
                   action='index', conditions={'method': ['GET']})
         m.connect('new_users_group', '/user_groups/new',
                   action='new', conditions={'method': ['GET']})
         m.connect('update_users_group', '/user_groups/{user_group_id}',
                   action='update', conditions={'method': ['PUT']})
         m.connect('delete_users_group', '/user_groups/{user_group_id}',
                   action='delete', conditions={'method': ['DELETE']})
         m.connect('edit_users_group', '/user_groups/{user_group_id}/edit',
                   action='edit', conditions={'method': ['GET']},
                   function=check_user_group)
 
         # EXTRAS USER GROUP ROUTES
         m.connect('edit_user_group_global_perms',
                   '/user_groups/{user_group_id}/edit/global_permissions',
                   action='edit_global_perms', conditions={'method': ['GET']})
         m.connect('edit_user_group_global_perms',
                   '/user_groups/{user_group_id}/edit/global_permissions',
                   action='update_global_perms', conditions={'method': ['PUT']})
         m.connect('edit_user_group_perms_summary',
                   '/user_groups/{user_group_id}/edit/permissions_summary',
                   action='edit_perms_summary', conditions={'method': ['GET']})
 
         m.connect('edit_user_group_perms',
                   '/user_groups/{user_group_id}/edit/permissions',
                   action='edit_perms', conditions={'method': ['GET']})
         m.connect('edit_user_group_perms',
                   '/user_groups/{user_group_id}/edit/permissions',
                   action='update_perms', conditions={'method': ['PUT']})
 
         m.connect('edit_user_group_advanced',
                   '/user_groups/{user_group_id}/edit/advanced',
                   action='edit_advanced', conditions={'method': ['GET']})
 
         m.connect('edit_user_group_members',
                   '/user_groups/{user_group_id}/edit/members', jsroute=True,
                   action='edit_members', conditions={'method': ['GET']})
 
     # ADMIN PERMISSIONS ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/permissions') as m:
         m.connect('admin_permissions_application', '/permissions/application',
                   action='permission_application_update', conditions={'method': ['POST']})
         m.connect('admin_permissions_application', '/permissions/application',
                   action='permission_application', conditions={'method': ['GET']})
 
         m.connect('admin_permissions_global', '/permissions/global',
                   action='permission_global_update', conditions={'method': ['POST']})
         m.connect('admin_permissions_global', '/permissions/global',
                   action='permission_global', conditions={'method': ['GET']})
 
         m.connect('admin_permissions_object', '/permissions/object',
                   action='permission_objects_update', conditions={'method': ['POST']})
         m.connect('admin_permissions_object', '/permissions/object',
                   action='permission_objects', conditions={'method': ['GET']})
 
         m.connect('admin_permissions_ips', '/permissions/ips',
                   action='permission_ips', conditions={'method': ['POST']})
         m.connect('admin_permissions_ips', '/permissions/ips',
                   action='permission_ips', conditions={'method': ['GET']})
 
         m.connect('admin_permissions_overview', '/permissions/overview',
                   action='permission_perms', conditions={'method': ['GET']})
 
     # ADMIN DEFAULTS REST ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/defaults') as m:
         m.connect('admin_defaults_repositories', '/defaults/repositories',
                   action='update_repository_defaults', conditions={'method': ['POST']})
         m.connect('admin_defaults_repositories', '/defaults/repositories',
                   action='index', conditions={'method': ['GET']})
 
     # ADMIN DEBUG STYLE ROUTES
     if str2bool(config.get('debug_style')):
         with rmap.submapper(path_prefix=ADMIN_PREFIX + '/debug_style',
                             controller='debug_style') as m:
             m.connect('debug_style_home', '',
                       action='index', conditions={'method': ['GET']})
             m.connect('debug_style_template', '/t/{t_path}',
                       action='template', conditions={'method': ['GET']})
 
     # ADMIN SETTINGS ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/settings') as m:
 
         # default
         m.connect('admin_settings', '/settings',
                   action='settings_global_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings', '/settings',
                   action='settings_global', conditions={'method': ['GET']})
 
         m.connect('admin_settings_vcs', '/settings/vcs',
                   action='settings_vcs_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings_vcs', '/settings/vcs',
                   action='settings_vcs',
                   conditions={'method': ['GET']})
         m.connect('admin_settings_vcs', '/settings/vcs',
                   action='delete_svn_pattern',
                   conditions={'method': ['DELETE']})
 
         m.connect('admin_settings_mapping', '/settings/mapping',
                   action='settings_mapping_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings_mapping', '/settings/mapping',
                   action='settings_mapping', conditions={'method': ['GET']})
 
         m.connect('admin_settings_global', '/settings/global',
                   action='settings_global_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings_global', '/settings/global',
                   action='settings_global', conditions={'method': ['GET']})
 
         m.connect('admin_settings_visual', '/settings/visual',
                   action='settings_visual_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings_visual', '/settings/visual',
                   action='settings_visual', conditions={'method': ['GET']})
 
         m.connect('admin_settings_issuetracker',
                   '/settings/issue-tracker', action='settings_issuetracker',
                   conditions={'method': ['GET']})
         m.connect('admin_settings_issuetracker_save',
                   '/settings/issue-tracker/save',
                   action='settings_issuetracker_save',
                   conditions={'method': ['POST']})
         m.connect('admin_issuetracker_test', '/settings/issue-tracker/test',
                   action='settings_issuetracker_test',
                   conditions={'method': ['POST']})
         m.connect('admin_issuetracker_delete',
                   '/settings/issue-tracker/delete',
                   action='settings_issuetracker_delete',
                   conditions={'method': ['DELETE']})
 
         m.connect('admin_settings_email', '/settings/email',
                   action='settings_email_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings_email', '/settings/email',
                   action='settings_email', conditions={'method': ['GET']})
 
         m.connect('admin_settings_hooks', '/settings/hooks',
                   action='settings_hooks_update',
                   conditions={'method': ['POST', 'DELETE']})
         m.connect('admin_settings_hooks', '/settings/hooks',
                   action='settings_hooks', conditions={'method': ['GET']})
 
         m.connect('admin_settings_search', '/settings/search',
                   action='settings_search', conditions={'method': ['GET']})
 
         m.connect('admin_settings_system', '/settings/system',
                   action='settings_system', conditions={'method': ['GET']})
 
         m.connect('admin_settings_system_update', '/settings/system/updates',
                   action='settings_system_update', conditions={'method': ['GET']})
 
         m.connect('admin_settings_supervisor', '/settings/supervisor',
                   action='settings_supervisor', conditions={'method': ['GET']})
         m.connect('admin_settings_supervisor_log', '/settings/supervisor/{procid}/log',
                   action='settings_supervisor_log', conditions={'method': ['GET']})
 
         m.connect('admin_settings_labs', '/settings/labs',
                   action='settings_labs_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings_labs', '/settings/labs',
                   action='settings_labs', conditions={'method': ['GET']})
 
     # ADMIN MY ACCOUNT
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/my_account') as m:
 
         m.connect('my_account', '/my_account',
                   action='my_account', conditions={'method': ['GET']})
         m.connect('my_account_edit', '/my_account/edit',
                   action='my_account_edit', conditions={'method': ['GET']})
         m.connect('my_account', '/my_account',
                   action='my_account_update', conditions={'method': ['POST']})
 
         m.connect('my_account_password', '/my_account/password',
                   action='my_account_password', conditions={'method': ['GET']})
         m.connect('my_account_password', '/my_account/password',
                   action='my_account_password_update', conditions={'method': ['POST']})
 
         m.connect('my_account_repos', '/my_account/repos',
                   action='my_account_repos', conditions={'method': ['GET']})
 
         m.connect('my_account_watched', '/my_account/watched',
                   action='my_account_watched', conditions={'method': ['GET']})
 
         m.connect('my_account_pullrequests', '/my_account/pull_requests',
                   action='my_account_pullrequests', conditions={'method': ['GET']})
 
         m.connect('my_account_perms', '/my_account/perms',
                   action='my_account_perms', conditions={'method': ['GET']})
 
         m.connect('my_account_emails', '/my_account/emails',
                   action='my_account_emails', conditions={'method': ['GET']})
         m.connect('my_account_emails', '/my_account/emails',
                   action='my_account_emails_add', conditions={'method': ['POST']})
         m.connect('my_account_emails', '/my_account/emails',
                   action='my_account_emails_delete', conditions={'method': ['DELETE']})
 
         m.connect('my_account_auth_tokens', '/my_account/auth_tokens',
                   action='my_account_auth_tokens', conditions={'method': ['GET']})
         m.connect('my_account_auth_tokens', '/my_account/auth_tokens',
                   action='my_account_auth_tokens_add', conditions={'method': ['POST']})
         m.connect('my_account_auth_tokens', '/my_account/auth_tokens',
                   action='my_account_auth_tokens_delete', conditions={'method': ['DELETE']})
 
     # NOTIFICATION REST ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/notifications') as m:
         m.connect('notifications', '/notifications',
                   action='index', conditions={'method': ['GET']})
         m.connect('notifications_mark_all_read', '/notifications/mark_all_read',
                   action='mark_all_read', conditions={'method': ['POST']})
 
         m.connect('/notifications/{notification_id}',
                   action='update', conditions={'method': ['PUT']})
         m.connect('/notifications/{notification_id}',
                   action='delete', conditions={'method': ['DELETE']})
         m.connect('notification', '/notifications/{notification_id}',
                   action='show', conditions={'method': ['GET']})
 
     # ADMIN GIST
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/gists') as m:
         m.connect('gists', '/gists',
                   action='create', conditions={'method': ['POST']})
         m.connect('gists', '/gists', jsroute=True,
                   action='index', conditions={'method': ['GET']})
         m.connect('new_gist', '/gists/new', jsroute=True,
                   action='new', conditions={'method': ['GET']})
 
         m.connect('/gists/{gist_id}',
                   action='delete', conditions={'method': ['DELETE']})
         m.connect('edit_gist', '/gists/{gist_id}/edit',
                   action='edit_form', conditions={'method': ['GET']})
         m.connect('edit_gist', '/gists/{gist_id}/edit',
                   action='edit', conditions={'method': ['POST']})
         m.connect(
             'edit_gist_check_revision', '/gists/{gist_id}/edit/check_revision',
             action='check_revision', conditions={'method': ['GET']})
 
         m.connect('gist', '/gists/{gist_id}',
                   action='show', conditions={'method': ['GET']})
         m.connect('gist_rev', '/gists/{gist_id}/{revision}',
                   revision='tip',
                   action='show', conditions={'method': ['GET']})
         m.connect('formatted_gist', '/gists/{gist_id}/{revision}/{format}',
                   revision='tip',
                   action='show', conditions={'method': ['GET']})
         m.connect('formatted_gist_file', '/gists/{gist_id}/{revision}/{format}/{f_path}',
                   revision='tip',
                   action='show', conditions={'method': ['GET']},
                   requirements=URL_NAME_REQUIREMENTS)
 
     # ADMIN MAIN PAGES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/admin') as m:
         m.connect('admin_home', '', action='index')
         m.connect('admin_add_repo', '/add_repo/{new_repo:[a-z0-9\. _-]*}',
                   action='add_repo')
         m.connect(
             'pull_requests_global_0', '/pull_requests/{pull_request_id:[0-9]+}',
             action='pull_requests')
         m.connect(
             'pull_requests_global', '/pull-requests/{pull_request_id:[0-9]+}',
             action='pull_requests')
 
 
     # USER JOURNAL
     rmap.connect('journal', '%s/journal' % (ADMIN_PREFIX,),
                  controller='journal', action='index')
     rmap.connect('journal_rss', '%s/journal/rss' % (ADMIN_PREFIX,),
                  controller='journal', action='journal_rss')
     rmap.connect('journal_atom', '%s/journal/atom' % (ADMIN_PREFIX,),
                  controller='journal', action='journal_atom')
 
     rmap.connect('public_journal', '%s/public_journal' % (ADMIN_PREFIX,),
                  controller='journal', action='public_journal')
 
     rmap.connect('public_journal_rss', '%s/public_journal/rss' % (ADMIN_PREFIX,),
                  controller='journal', action='public_journal_rss')
 
     rmap.connect('public_journal_rss_old', '%s/public_journal_rss' % (ADMIN_PREFIX,),
                  controller='journal', action='public_journal_rss')
 
     rmap.connect('public_journal_atom',
                  '%s/public_journal/atom' % (ADMIN_PREFIX,), controller='journal',
                  action='public_journal_atom')
 
     rmap.connect('public_journal_atom_old',
                  '%s/public_journal_atom' % (ADMIN_PREFIX,), controller='journal',
                  action='public_journal_atom')
 
     rmap.connect('toggle_following', '%s/toggle_following' % (ADMIN_PREFIX,),
                  controller='journal', action='toggle_following', jsroute=True,
                  conditions={'method': ['POST']})
 
     # FULL TEXT SEARCH
     rmap.connect('search', '%s/search' % (ADMIN_PREFIX,),
                  controller='search')
     rmap.connect('search_repo_home', '/{repo_name}/search',
                  controller='search',
                  action='index',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     # FEEDS
     rmap.connect('rss_feed_home', '/{repo_name}/feed/rss',
                  controller='feed', action='rss',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('atom_feed_home', '/{repo_name}/feed/atom',
                  controller='feed', action='atom',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     #==========================================================================
     # REPOSITORY ROUTES
     #==========================================================================
 
     rmap.connect('repo_creating_home', '/{repo_name}/repo_creating',
                  controller='admin/repos', action='repo_creating',
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_check_home', '/{repo_name}/crepo_check',
                  controller='admin/repos', action='repo_check',
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('repo_stats', '/{repo_name}/repo_stats/{commit_id}',
                  controller='summary', action='repo_stats',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('repo_refs_data', '/{repo_name}/refs-data',
                  controller='summary', action='repo_refs_data', jsroute=True,
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_refs_changelog_data', '/{repo_name}/refs-data-changelog',
                  controller='summary', action='repo_refs_changelog_data',
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('changeset_home', '/{repo_name}/changeset/{revision}',
                  controller='changeset', revision='tip', jsroute=True,
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('changeset_children', '/{repo_name}/changeset_children/{revision}',
                  controller='changeset', revision='tip', action='changeset_children',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('changeset_parents', '/{repo_name}/changeset_parents/{revision}',
                  controller='changeset', revision='tip', action='changeset_parents',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     # repo edit options
     rmap.connect('edit_repo', '/{repo_name}/settings', jsroute=True,
                  controller='admin/repos', action='edit',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_perms', '/{repo_name}/settings/permissions',
                  jsroute=True,
                  controller='admin/repos', action='edit_permissions',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('edit_repo_perms_update', '/{repo_name}/settings/permissions',
                  controller='admin/repos', action='edit_permissions_update',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_fields', '/{repo_name}/settings/fields',
                  controller='admin/repos', action='edit_fields',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('create_repo_fields', '/{repo_name}/settings/fields/new',
                  controller='admin/repos', action='create_repo_field',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('delete_repo_fields', '/{repo_name}/settings/fields/{field_id}',
                  controller='admin/repos', action='delete_repo_field',
                  conditions={'method': ['DELETE'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_advanced', '/{repo_name}/settings/advanced',
                  controller='admin/repos', action='edit_advanced',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_advanced_locking', '/{repo_name}/settings/advanced/locking',
                  controller='admin/repos', action='edit_advanced_locking',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('toggle_locking', '/{repo_name}/settings/advanced/locking_toggle',
                  controller='admin/repos', action='toggle_locking',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_advanced_journal', '/{repo_name}/settings/advanced/journal',
                  controller='admin/repos', action='edit_advanced_journal',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_advanced_fork', '/{repo_name}/settings/advanced/fork',
                  controller='admin/repos', action='edit_advanced_fork',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_caches', '/{repo_name}/settings/caches',
                  controller='admin/repos', action='edit_caches_form',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('edit_repo_caches', '/{repo_name}/settings/caches',
                  controller='admin/repos', action='edit_caches',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_remote', '/{repo_name}/settings/remote',
                  controller='admin/repos', action='edit_remote_form',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('edit_repo_remote', '/{repo_name}/settings/remote',
                  controller='admin/repos', action='edit_remote',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_statistics', '/{repo_name}/settings/statistics',
                  controller='admin/repos', action='edit_statistics_form',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('edit_repo_statistics', '/{repo_name}/settings/statistics',
                  controller='admin/repos', action='edit_statistics',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_settings_issuetracker',
                  '/{repo_name}/settings/issue-tracker',
                  controller='admin/repos', action='repo_issuetracker',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_issuetracker_test',
                  '/{repo_name}/settings/issue-tracker/test',
                  controller='admin/repos', action='repo_issuetracker_test',
                  conditions={'method': ['POST'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_issuetracker_delete',
                  '/{repo_name}/settings/issue-tracker/delete',
                  controller='admin/repos', action='repo_issuetracker_delete',
                  conditions={'method': ['DELETE'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_issuetracker_save',
                  '/{repo_name}/settings/issue-tracker/save',
                  controller='admin/repos', action='repo_issuetracker_save',
                  conditions={'method': ['POST'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_vcs_settings', '/{repo_name}/settings/vcs',
                  controller='admin/repos', action='repo_settings_vcs_update',
                  conditions={'method': ['POST'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_vcs_settings', '/{repo_name}/settings/vcs',
                  controller='admin/repos', action='repo_settings_vcs',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_vcs_settings', '/{repo_name}/settings/vcs',
                  controller='admin/repos', action='repo_delete_svn_pattern',
                  conditions={'method': ['DELETE'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     # still working url for backward compat.
     rmap.connect('raw_changeset_home_depraced',
                  '/{repo_name}/raw-changeset/{revision}',
                  controller='changeset', action='changeset_raw',
                  revision='tip', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     # new URLs
     rmap.connect('changeset_raw_home',
                  '/{repo_name}/changeset-diff/{revision}',
                  controller='changeset', action='changeset_raw',
                  revision='tip', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changeset_patch_home',
                  '/{repo_name}/changeset-patch/{revision}',
                  controller='changeset', action='changeset_patch',
                  revision='tip', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changeset_download_home',
                  '/{repo_name}/changeset-download/{revision}',
                  controller='changeset', action='changeset_download',
                  revision='tip', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changeset_comment',
                  '/{repo_name}/changeset/{revision}/comment', jsroute=True,
                  controller='changeset', revision='tip', action='comment',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changeset_comment_preview',
                  '/{repo_name}/changeset/comment/preview', jsroute=True,
                  controller='changeset', action='preview_comment',
                  conditions={'function': check_repo, 'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changeset_comment_delete',
                  '/{repo_name}/changeset/comment/{comment_id}/delete',
                  controller='changeset', action='delete_comment',
                  conditions={'function': check_repo, 'method': ['DELETE']},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('changeset_info', '/changeset_info/{repo_name}/{revision}',
                  controller='changeset', action='changeset_info',
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('compare_home',
                  '/{repo_name}/compare',
                  controller='compare', action='index',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('compare_url',
                  '/{repo_name}/compare/{source_ref_type}@{source_ref:.*?}...{target_ref_type}@{target_ref:.*?}',
                  controller='compare', action='compare',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_home',
                  '/{repo_name}/pull-request/new', controller='pullrequests',
                  action='index', conditions={'function': check_repo,
                                              'method': ['GET']},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest',
                  '/{repo_name}/pull-request/new', controller='pullrequests',
                  action='create', conditions={'function': check_repo,
                                               'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_repo_refs',
                  '/{repo_name}/pull-request/refs/{target_repo_name:.*?[^/]}',
                  controller='pullrequests',
                  action='get_repo_refs',
                  conditions={'function': check_repo, 'method': ['GET']},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_repo_destinations',
                  '/{repo_name}/pull-request/repo-destinations',
                  controller='pullrequests',
                  action='get_repo_destinations',
                  conditions={'function': check_repo, 'method': ['GET']},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_show',
                  '/{repo_name}/pull-request/{pull_request_id}',
                  controller='pullrequests',
                  action='show', conditions={'function': check_repo,
                                             'method': ['GET']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('pullrequest_update',
                  '/{repo_name}/pull-request/{pull_request_id}',
                  controller='pullrequests',
                  action='update', conditions={'function': check_repo,
                                               'method': ['PUT']},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_merge',
                  '/{repo_name}/pull-request/{pull_request_id}',
                  controller='pullrequests',
                  action='merge', conditions={'function': check_repo,
                                              'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('pullrequest_delete',
                  '/{repo_name}/pull-request/{pull_request_id}',
                  controller='pullrequests',
                  action='delete', conditions={'function': check_repo,
                                               'method': ['DELETE']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('pullrequest_show_all',
                  '/{repo_name}/pull-request',
                  controller='pullrequests',
                  action='show_all', conditions={'function': check_repo,
                                                 'method': ['GET']},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_comment',
                  '/{repo_name}/pull-request-comment/{pull_request_id}',
                  controller='pullrequests',
                  action='comment', conditions={'function': check_repo,
                                                'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_comment_delete',
                  '/{repo_name}/pull-request-comment/{comment_id}/delete',
                  controller='pullrequests', action='delete_comment',
                  conditions={'function': check_repo, 'method': ['DELETE']},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('summary_home_explicit', '/{repo_name}/summary',
                  controller='summary', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('branches_home', '/{repo_name}/branches',
                  controller='branches', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('tags_home', '/{repo_name}/tags',
                  controller='tags', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('bookmarks_home', '/{repo_name}/bookmarks',
                  controller='bookmarks', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changelog_home', '/{repo_name}/changelog', jsroute=True,
                  controller='changelog', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changelog_summary_home', '/{repo_name}/changelog_summary',
                  controller='changelog', action='changelog_summary',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changelog_file_home',
                  '/{repo_name}/changelog/{revision}/{f_path}',
                  controller='changelog', f_path=None,
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('changelog_details', '/{repo_name}/changelog_details/{cs}',
                  controller='changelog', action='changelog_details',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_home',  '/{repo_name}/files/{revision}/{f_path}',
                  controller='files', revision='tip', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('files_home_simple_catchrev',
                  '/{repo_name}/files/{revision}',
                  controller='files', revision='tip', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_home_simple_catchall',
                  '/{repo_name}/files',
                  controller='files', revision='tip', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_history_home',
                  '/{repo_name}/history/{revision}/{f_path}',
                  controller='files', action='history', revision='tip', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('files_authors_home',
                  '/{repo_name}/authors/{revision}/{f_path}',
                  controller='files', action='authors', revision='tip', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('files_diff_home', '/{repo_name}/diff/{f_path}',
                  controller='files', action='diff', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_diff_2way_home',
                  '/{repo_name}/diff-2way/{f_path}',
                  controller='files', action='diff_2way', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_rawfile_home',
                  '/{repo_name}/rawfile/{revision}/{f_path}',
                  controller='files', action='rawfile', revision='tip',
                  f_path='', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_raw_home',
                  '/{repo_name}/raw/{revision}/{f_path}',
                  controller='files', action='raw', revision='tip', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_render_home',
                  '/{repo_name}/render/{revision}/{f_path}',
                  controller='files', action='index', revision='tip', f_path='',
                  rendered=True, conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_annotate_home',
                  '/{repo_name}/annotate/{revision}/{f_path}',
                  controller='files', action='index', revision='tip',
                  f_path='', annotate=True, conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_edit',
                  '/{repo_name}/edit/{revision}/{f_path}',
                  controller='files', action='edit', revision='tip',
                  f_path='',
                  conditions={'function': check_repo, 'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_edit_home',
                  '/{repo_name}/edit/{revision}/{f_path}',
                  controller='files', action='edit_home', revision='tip',
                  f_path='', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_add',
                  '/{repo_name}/add/{revision}/{f_path}',
                  controller='files', action='add', revision='tip',
                  f_path='',
                  conditions={'function': check_repo, 'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_add_home',
                  '/{repo_name}/add/{revision}/{f_path}',
                  controller='files', action='add_home', revision='tip',
                  f_path='', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_delete',
                  '/{repo_name}/delete/{revision}/{f_path}',
                  controller='files', action='delete', revision='tip',
                  f_path='',
                  conditions={'function': check_repo, 'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_delete_home',
                  '/{repo_name}/delete/{revision}/{f_path}',
                  controller='files', action='delete_home', revision='tip',
                  f_path='', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_archive_home', '/{repo_name}/archive/{fname}',
                  controller='files', action='archivefile',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('files_nodelist_home',
                  '/{repo_name}/nodelist/{revision}/{f_path}',
                  controller='files', action='nodelist',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('files_metadata_list_home',
                  '/{repo_name}/metadata_list/{revision}/{f_path}',
                  controller='files', action='metadata_list',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('repo_fork_create_home', '/{repo_name}/fork',
                  controller='forks', action='fork_create',
                  conditions={'function': check_repo, 'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('repo_fork_home', '/{repo_name}/fork',
                  controller='forks', action='fork',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('repo_forks_home', '/{repo_name}/forks',
                  controller='forks', action='forks',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('repo_followers_home', '/{repo_name}/followers',
                  controller='followers', action='followers',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     # must be here for proper group/repo catching pattern
     _connect_with_slash(
         rmap, 'repo_group_home', '/{group_name}',
         controller='home', action='index_repo_group',
         conditions={'function': check_group},
         requirements=URL_NAME_REQUIREMENTS)
 
     # catch all, at the end
     _connect_with_slash(
         rmap, 'summary_home', '/{repo_name}', jsroute=True,
         controller='summary', action='index',
         conditions={'function': check_repo},
         requirements=URL_NAME_REQUIREMENTS)
 
     return rmap
 
 
 def _connect_with_slash(mapper, name, path, *args, **kwargs):
     """
     Connect a route with an optional trailing slash in `path`.
     """
     mapper.connect(name + '_slash', path + '/', *args, **kwargs)
     mapper.connect(name, path, *args, **kwargs)

rhodecode/events/__init__.py

@@ -1,57 +1,66 b''
 # Copyright (C) 2016-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from pyramid.threadlocal import get_current_registry
 
 
-def trigger(event):
+def trigger(event, registry=None):
     """
     Helper method to send an event. This wraps the pyramid logic to send an
     event.
     """
     # For the first step we are using pyramids thread locals here. If the
     # event mechanism works out as a good solution we should think about
     # passing the registry as an argument to get rid of it.
-    registry = get_current_registry()
+    registry = registry or get_current_registry()
     registry.notify(event)
 
+    # Until we can work around the problem that VCS operations do not have a
+    # pyramid context to work with, we send the events to integrations directly
+
+    # Later it will be possible to use regular pyramid subscribers ie:
+    #   config.add_subscriber(integrations_event_handler, RhodecodeEvent)
+    from rhodecode.integrations import integrations_event_handler
+    if isinstance(event, RhodecodeEvent):
+        integrations_event_handler(event)
+
 
 from rhodecode.events.base import RhodecodeEvent
 
 from rhodecode.events.user import (
     UserPreCreate,
     UserPreUpdate,
     UserRegistered
 )
 
 from rhodecode.events.repo import (
     RepoEvent,
-    RepoPreCreateEvent, RepoCreatedEvent,
-    RepoPreDeleteEvent, RepoDeletedEvent,
+    RepoPreCreateEvent, RepoCreateEvent,
+    RepoPreDeleteEvent, RepoDeleteEvent,
     RepoPrePushEvent,   RepoPushEvent,
     RepoPrePullEvent,   RepoPullEvent,
 )
 
 from rhodecode.events.pullrequest import (
     PullRequestEvent,
     PullRequestCreateEvent,
     PullRequestUpdateEvent,
     PullRequestReviewEvent,
     PullRequestMergeEvent,
     PullRequestCloseEvent,
-)
  No newline at end of file
+)

rhodecode/events/base.py

@@ -1,71 +1,59 b''
 # Copyright (C) 2016-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from datetime import datetime
-from marshmallow import Schema, fields
 from pyramid.threadlocal import get_current_request
 from rhodecode.lib.utils2 import AttributeDict
 
 
 # this is a user object to be used for events caused by the system (eg. shell)
 SYSTEM_USER = AttributeDict(dict(
     username='__SYSTEM__'
 ))
 
 
-class UserSchema(Schema):
-    """
-    Marshmallow schema for a user
-    """
-    username = fields.Str()
-
-
-class RhodecodeEventSchema(Schema):
-    """
-    Marshmallow schema for a rhodecode event
-    """
-    utc_timestamp = fields.DateTime()
-    actor = fields.Nested(UserSchema)
-    actor_ip = fields.Str()
-    name = fields.Str()
-
-
 class RhodecodeEvent(object):
     """
     Base event class for all Rhodecode events
     """
-    MarshmallowSchema = RhodecodeEventSchema
-
     def __init__(self):
         self.request = get_current_request()
         self.utc_timestamp = datetime.utcnow()
 
     @property
     def actor(self):
         if self.request:
             return self.request.user.get_instance()
         return SYSTEM_USER
 
     @property
     def actor_ip(self):
         if self.request:
             return self.request.user.ip_addr
         return '<no ip available>'
 
     def as_dict(self):
-        return self.MarshmallowSchema().dump(self).data
+        data = {
+            'name': self.name,
+            'utc_timestamp': self.utc_timestamp,
+            'actor_ip': self.actor_ip,
+            'actor': {
+                'username': self.actor.username
+            }
+        }
+        return data
  No newline at end of file

rhodecode/events/pullrequest.py

@@ -1,97 +1,97 b''
 # Copyright (C) 2016-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-from marshmallow import Schema, fields
 
+from rhodecode.translation import lazy_ugettext
 from rhodecode.events.repo import RepoEvent
 
 
-def get_pull_request_url(pull_request):
-    from rhodecode.model.pull_request import PullRequestModel
-    return PullRequestModel().get_url(pull_request)
-
-
-class PullRequestSchema(Schema):
-    """
-    Marshmallow schema for a pull request
-    """
-    pull_request_id = fields.Integer()
-    url = fields.Function(get_pull_request_url)
-    title = fields.Str()
-
-
-class PullRequestEventSchema(RepoEvent.MarshmallowSchema):
-    """
-    Marshmallow schema for a pull request event
-    """
-    pullrequest = fields.Nested(PullRequestSchema)
-
-
 class PullRequestEvent(RepoEvent):
     """
     Base class for pull request events.
 
     :param pullrequest: a :class:`PullRequest` instance
     """
-    MarshmallowSchema = PullRequestEventSchema
 
     def __init__(self, pullrequest):
         super(PullRequestEvent, self).__init__(pullrequest.target_repo)
         self.pullrequest = pullrequest
 
+    def as_dict(self):
+        from rhodecode.model.pull_request import PullRequestModel
+        data = super(PullRequestEvent, self).as_dict()
+
+        commits = self._commits_as_dict(self.pullrequest.revisions)
+        issues = self._issues_as_dict(commits)
+
+        data.update({
+            'pullrequest': {
+                'title': self.pullrequest.title,
+                'issues': issues,
+                'pull_request_id': self.pullrequest.pull_request_id,
+                'url': PullRequestModel().get_url(self.pullrequest)
+            }
+        })
+        return data
+
 
 class PullRequestCreateEvent(PullRequestEvent):
     """
     An instance of this class is emitted as an :term:`event` after a pull
     request is created.
     """
     name = 'pullrequest-create'
+    display_name = lazy_ugettext('pullrequest created')
 
 
 class PullRequestCloseEvent(PullRequestEvent):
     """
     An instance of this class is emitted as an :term:`event` after a pull
     request is closed.
     """
     name = 'pullrequest-close'
+    display_name = lazy_ugettext('pullrequest closed')
 
 
 class PullRequestUpdateEvent(PullRequestEvent):
     """
     An instance of this class is emitted as an :term:`event` after a pull
     request is updated.
     """
     name = 'pullrequest-update'
+    display_name = lazy_ugettext('pullrequest updated')
 
 
 class PullRequestMergeEvent(PullRequestEvent):
     """
     An instance of this class is emitted as an :term:`event` after a pull
     request is merged.
     """
     name = 'pullrequest-merge'
+    display_name = lazy_ugettext('pullrequest merged')
 
 
 class PullRequestReviewEvent(PullRequestEvent):
     """
     An instance of this class is emitted as an :term:`event` after a pull
     request is reviewed.
     """
     name = 'pullrequest-review'
+    display_name = lazy_ugettext('pullrequest reviewed')
 
 

rhodecode/events/repo.py

@@ -1,149 +1,219 b''
 # Copyright (C) 2016-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-from marshmallow import Schema, fields
+import logging
 
+from rhodecode.translation import lazy_ugettext
 from rhodecode.model.db import User, Repository, Session
 from rhodecode.events.base import RhodecodeEvent
 
-
-def get_repo_url(repo):
-    from rhodecode.model.repo import RepoModel
-    return RepoModel().get_url(repo)
-
-
-class RepositorySchema(Schema):
-    """
-    Marshmallow schema for a repository
-    """
-    repo_id = fields.Integer()
-    repo_name = fields.Str()
-    url = fields.Function(get_repo_url)
-
-
-class RepoEventSchema(RhodecodeEvent.MarshmallowSchema):
-    """
-    Marshmallow schema for a repository event
-    """
-    repo = fields.Nested(RepositorySchema)
+log = logging.getLogger()
 
 
 class RepoEvent(RhodecodeEvent):
     """
     Base class for events acting on a repository.
 
     :param repo: a :class:`Repository` instance
     """
-    MarshmallowSchema = RepoEventSchema
 
     def __init__(self, repo):
         super(RepoEvent, self).__init__()
         self.repo = repo
 
+    def as_dict(self):
+        from rhodecode.model.repo import RepoModel
+        data = super(RepoEvent, self).as_dict()
+        data.update({
+            'repo': {
+                'repo_id': self.repo.repo_id,
+                'repo_name': self.repo.repo_name,
+                'url': RepoModel().get_url(self.repo)
+            }
+        })
+        return data
+
+    def _commits_as_dict(self, commit_ids):
+        """ Helper function to serialize commit_ids """
+
+        from rhodecode.lib.utils2 import extract_mentioned_users
+        from rhodecode.model.db import Repository
+        from rhodecode.lib import helpers as h
+        from rhodecode.lib.helpers import process_patterns
+        from rhodecode.lib.helpers import urlify_commit_message
+        if not commit_ids:
+            return []
+        commits = []
+        reviewers = []
+        vcs_repo = self.repo.scm_instance(cache=False)
+        try:
+            for commit_id in commit_ids:
+                cs = vcs_repo.get_changeset(commit_id)
+                cs_data = cs.__json__()
+                cs_data['mentions'] = extract_mentioned_users(cs_data['message'])
+                cs_data['reviewers'] = reviewers
+                cs_data['url'] = h.url('changeset_home',
+                    repo_name=self.repo.repo_name,
+                    revision=cs_data['raw_id'],
+                    qualified=True
+                )
+                urlified_message, issues_data = process_patterns(
+                    cs_data['message'], self.repo.repo_name)
+                cs_data['issues'] = issues_data
+                cs_data['message_html'] = urlify_commit_message(cs_data['message'],
+                    self.repo.repo_name)
+                commits.append(cs_data)
+        except Exception as e:
+            log.exception(e)
+            # we don't send any commits when crash happens, only full list matters
+            # we short circuit then.
+            return []
+        return commits
+
+    def _issues_as_dict(self, commits):
+        """ Helper function to serialize issues from commits """
+        issues = {}
+        for commit in commits:
+            for issue in commit['issues']:
+                issues[issue['id']] = issue
+        return issues
+
 
 class RepoPreCreateEvent(RepoEvent):
     """
     An instance of this class is emitted as an :term:`event` before a repo is
     created.
     """
     name = 'repo-pre-create'
+    display_name = lazy_ugettext('repository pre create')
 
 
-class RepoCreatedEvent(RepoEvent):
+class RepoCreateEvent(RepoEvent):
     """
     An instance of this class is emitted as an :term:`event` whenever a repo is
     created.
     """
-    name = 'repo-created'
+    name = 'repo-create'
+    display_name = lazy_ugettext('repository created')
 
 
 class RepoPreDeleteEvent(RepoEvent):
     """
     An instance of this class is emitted as an :term:`event` whenever a repo is
     created.
     """
     name = 'repo-pre-delete'
+    display_name = lazy_ugettext('repository pre delete')
 
 
-class RepoDeletedEvent(RepoEvent):
+class RepoDeleteEvent(RepoEvent):
     """
     An instance of this class is emitted as an :term:`event` whenever a repo is
     created.
     """
-    name = 'repo-deleted'
+    name = 'repo-delete'
+    display_name = lazy_ugettext('repository deleted')
 
 
 class RepoVCSEvent(RepoEvent):
     """
     Base class for events triggered by the VCS
     """
     def __init__(self, repo_name, extras):
         self.repo = Repository.get_by_repo_name(repo_name)
         if not self.repo:
             raise Exception('repo by this name %s does not exist' % repo_name)
         self.extras = extras
         super(RepoVCSEvent, self).__init__(self.repo)
 
     @property
     def actor(self):
         if self.extras.get('username'):
             return User.get_by_username(self.extras['username'])
 
     @property
     def actor_ip(self):
         if self.extras.get('ip'):
             return self.extras['ip']
 
 
 class RepoPrePullEvent(RepoVCSEvent):
     """
     An instance of this class is emitted as an :term:`event` before commits
     are pulled from a repo.
     """
     name = 'repo-pre-pull'
+    display_name = lazy_ugettext('repository pre pull')
 
 
 class RepoPullEvent(RepoVCSEvent):
     """
     An instance of this class is emitted as an :term:`event` after commits
     are pulled from a repo.
     """
     name = 'repo-pull'
+    display_name = lazy_ugettext('repository pull')
 
 
 class RepoPrePushEvent(RepoVCSEvent):
     """
     An instance of this class is emitted as an :term:`event` before commits
     are pushed to a repo.
     """
     name = 'repo-pre-push'
+    display_name = lazy_ugettext('repository pre push')
 
 
 class RepoPushEvent(RepoVCSEvent):
     """
     An instance of this class is emitted as an :term:`event` after commits
     are pushed to a repo.
 
     :param extras: (optional) dict of data from proxied VCS actions
     """
     name = 'repo-push'
+    display_name = lazy_ugettext('repository push')
 
     def __init__(self, repo_name, pushed_commit_ids, extras):
         super(RepoPushEvent, self).__init__(repo_name, extras)
         self.pushed_commit_ids = pushed_commit_ids
 
+    def as_dict(self):
+        data = super(RepoPushEvent, self).as_dict()
+        branch_url = repo_url = data['repo']['url']
+
+        commits = self._commits_as_dict(self.pushed_commit_ids)
+        issues = self._issues_as_dict(commits)
+
+        branches = set(
+            commit['branch'] for commit in commits if commit['branch'])
+        branches = [
+            {
+                'name': branch,
+                'url': '{}/changelog?branch={}'.format(
+                    data['repo']['url'], branch)
+            }
+            for branch in branches
+        ]
+
+        data['push'] = {
+            'commits': commits,
+            'issues': issues,
+            'branches': branches,
+        }
+        return data
  No newline at end of file

rhodecode/events/user.py

@@ -1,55 +1,65 b''
 # Copyright (C) 2016-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from zope.interface import implementer
 
+from rhodecode.translation import lazy_ugettext
 from rhodecode.events.base import RhodecodeEvent
 from rhodecode.events.interfaces import (
     IUserRegistered, IUserPreCreate, IUserPreUpdate)
 
 
 @implementer(IUserRegistered)
 class UserRegistered(RhodecodeEvent):
     """
     An instance of this class is emitted as an :term:`event` whenever a user
     account is registered.
     """
+    name = 'user-register'
+    display_name = lazy_ugettext('user registered')
+
     def __init__(self, user, session):
         self.user = user
         self.session = session
 
 
 @implementer(IUserPreCreate)
 class UserPreCreate(RhodecodeEvent):
     """
     An instance of this class is emitted as an :term:`event` before a new user
     object is created.
     """
+    name = 'user-pre-create'
+    display_name = lazy_ugettext('user pre create')
+
     def __init__(self, user_data):
         self.user_data = user_data
 
 
 @implementer(IUserPreUpdate)
 class UserPreUpdate(RhodecodeEvent):
     """
     An instance of this class is emitted as an :term:`event` before a user
     object is updated.
     """
+    name = 'user-pre-update'
+    display_name = lazy_ugettext('user pre update')
+
     def __init__(self, user, user_data):
         self.user = user
         self.user_data = user_data

rhodecode/lib/helpers.py

@@ -1,1900 +1,1931 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Helper functions
 
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
 
 import random
 import hashlib
 import StringIO
 import urllib
 import math
 import logging
 import re
 import urlparse
 import time
 import string
 import hashlib
 import pygments
 
 from datetime import datetime
 from functools import partial
 from pygments.formatters.html import HtmlFormatter
 from pygments import highlight as code_highlight
 from pygments.lexers import (
     get_lexer_by_name, get_lexer_for_filename, get_lexer_for_mimetype)
-from pylons import url
+from pylons import url as pylons_url
 from pylons.i18n.translation import _, ungettext
 from pyramid.threadlocal import get_current_request
 
 from webhelpers.html import literal, HTML, escape
 from webhelpers.html.tools import *
 from webhelpers.html.builder import make_tag
 from webhelpers.html.tags import auto_discovery_link, checkbox, css_classes, \
     end_form, file, form as wh_form, hidden, image, javascript_link, link_to, \
     link_to_if, link_to_unless, ol, required_legend, select, stylesheet_link, \
     submit, text, password, textarea, title, ul, xml_declaration, radio
 from webhelpers.html.tools import auto_link, button_to, highlight, \
     js_obfuscate, mail_to, strip_links, strip_tags, tag_re
 from webhelpers.pylonslib import Flash as _Flash
 from webhelpers.text import chop_at, collapse, convert_accented_entities, \
     convert_misc_entities, lchop, plural, rchop, remove_formatting, \
     replace_whitespace, urlify, truncate, wrap_paragraphs
 from webhelpers.date import time_ago_in_words
 from webhelpers.paginate import Page as _Page
 from webhelpers.html.tags import _set_input_attrs, _set_id_attr, \
     convert_boolean_attrs, NotGiven, _make_safe_id_component
 from webhelpers2.number import format_byte_size
 
 from rhodecode.lib.annotate import annotate_highlight
 from rhodecode.lib.action_parser import action_parser
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils import repo_name_slug, get_custom_lexer
 from rhodecode.lib.utils2 import str2bool, safe_unicode, safe_str, \
     get_commit_safe, datetime_to_time, time_to_datetime, time_to_utcdatetime, \
     AttributeDict, safe_int, md5, md5_safe
 from rhodecode.lib.markup_renderer import MarkupRenderer
 from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
 from rhodecode.lib.vcs.backends.base import BaseChangeset, EmptyCommit
 from rhodecode.config.conf import DATE_FORMAT, DATETIME_FORMAT
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.db import Permission, User, Repository
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import IssueTrackerSettingsModel
 
 log = logging.getLogger(__name__)
 
 DEFAULT_USER = User.DEFAULT_USER
 DEFAULT_USER_EMAIL = User.DEFAULT_USER_EMAIL
 
+def url(*args, **kw):
+    return pylons_url(*args, **kw)
+
+def pylons_url_current(*args, **kw):
+    """
+    This function overrides pylons.url.current() which returns the current
+    path so that it will also work from a pyramid only context. This
+    should be removed once port to pyramid is complete.
+    """
+    if not args and not kw:
+        request = get_current_request()
+        return request.path
+    return pylons_url.current(*args, **kw)
+
+url.current = pylons_url_current
+
 
 def html_escape(text, html_escape_table=None):
     """Produce entities within text."""
     if not html_escape_table:
         html_escape_table = {
             "&": "&amp;",
             '"': "&quot;",
             "'": "&apos;",
             ">": "&gt;",
             "<": "&lt;",
         }
     return "".join(html_escape_table.get(c, c) for c in text)
 
 
 def chop_at_smart(s, sub, inclusive=False, suffix_if_chopped=None):
     """
     Truncate string ``s`` at the first occurrence of ``sub``.
 
     If ``inclusive`` is true, truncate just after ``sub`` rather than at it.
     """
     suffix_if_chopped = suffix_if_chopped or ''
     pos = s.find(sub)
     if pos == -1:
         return s
 
     if inclusive:
         pos += len(sub)
 
     chopped = s[:pos]
     left = s[pos:].strip()
 
     if left and suffix_if_chopped:
         chopped += suffix_if_chopped
 
     return chopped
 
 
 def shorter(text, size=20):
     postfix = '...'
     if len(text) > size:
         return text[:size - len(postfix)] + postfix
     return text
 
 
 def _reset(name, value=None, id=NotGiven, type="reset", **attrs):
     """
     Reset button
     """
     _set_input_attrs(attrs, type, name, value)
     _set_id_attr(attrs, id, name)
     convert_boolean_attrs(attrs, ["disabled"])
     return HTML.input(**attrs)
 
 reset = _reset
 safeid = _make_safe_id_component
 
 
 def branding(name, length=40):
     return truncate(name, length, indicator="")
 
 
 def FID(raw_id, path):
     """
     Creates a unique ID for filenode based on it's hash of path and commit
     it's safe to use in urls
 
     :param raw_id:
     :param path:
     """
 
     return 'c-%s-%s' % (short_id(raw_id), md5_safe(path)[:12])
 
 
 class _GetError(object):
     """Get error from form_errors, and represent it as span wrapped error
     message
 
     :param field_name: field to fetch errors for
     :param form_errors: form errors dict
     """
 
     def __call__(self, field_name, form_errors):
         tmpl = """<span class="error_msg">%s</span>"""
         if form_errors and field_name in form_errors:
             return literal(tmpl % form_errors.get(field_name))
 
 get_error = _GetError()
 
 
 class _ToolTip(object):
 
     def __call__(self, tooltip_title, trim_at=50):
         """
         Special function just to wrap our text into nice formatted
         autowrapped text
 
         :param tooltip_title:
         """
         tooltip_title = escape(tooltip_title)
         tooltip_title = tooltip_title.replace('<', '&lt;').replace('>', '&gt;')
         return tooltip_title
 tooltip = _ToolTip()
 
 
 def files_breadcrumbs(repo_name, commit_id, file_path):
     if isinstance(file_path, str):
         file_path = safe_unicode(file_path)
 
     # TODO: johbo: Is this always a url like path, or is this operating
     # system dependent?
     path_segments = file_path.split('/')
 
     repo_name_html = escape(repo_name)
     if len(path_segments) == 1 and path_segments[0] == '':
         url_segments = [repo_name_html]
     else:
         url_segments = [
             link_to(
                 repo_name_html,
                 url('files_home',
                     repo_name=repo_name,
                     revision=commit_id,
                     f_path=''),
                 class_='pjax-link')]
 
     last_cnt = len(path_segments) - 1
     for cnt, segment in enumerate(path_segments):
         if not segment:
             continue
         segment_html = escape(segment)
 
         if cnt != last_cnt:
             url_segments.append(
                 link_to(
                     segment_html,
                     url('files_home',
                         repo_name=repo_name,
                         revision=commit_id,
                         f_path='/'.join(path_segments[:cnt + 1])),
                     class_='pjax-link'))
         else:
             url_segments.append(segment_html)
 
     return literal('/'.join(url_segments))
 
 
 class CodeHtmlFormatter(HtmlFormatter):
     """
     My code Html Formatter for source codes
     """
 
     def wrap(self, source, outfile):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<div id="L%s">%s</div>' % (cnt + 1, t)
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         dummyoutfile = StringIO.StringIO()
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             dummyoutfile.write(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         nocls = self.noclasses
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s%d" class="special">%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         # in case you wonder about the seemingly redundant <div> here: since the
         # content in the other cell also is wrapped in a div, some browsers in
         # some configurations seem to mess up the formatting...
         if nocls:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td><div class="linenodiv" '
                       'style="background-color: #f0f0f0; padding-right: 10px">'
                       '<pre style="line-height: 125%">' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         else:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td class="linenos"><div class="linenodiv"><pre>' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         yield 0, dummyoutfile.getvalue()
         yield 0, '</td></tr></table>'
 
 
 class SearchContentCodeHtmlFormatter(CodeHtmlFormatter):
     def __init__(self, **kw):
         # only show these line numbers if set
         self.only_lines = kw.pop('only_line_numbers', [])
         self.query_terms = kw.pop('query_terms', [])
         self.max_lines = kw.pop('max_lines', 5)
         self.line_context = kw.pop('line_context', 3)
         self.url = kw.pop('url', None)
 
         super(CodeHtmlFormatter, self).__init__(**kw)
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<pre>%s</pre>' % t
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         yield 0, '<table class="code-highlight %stable">' % self.cssclass
 
         last_shown_line_number = 0
         current_line_number = 1
 
         for t, line in inner:
             if not t:
                 yield t, line
                 continue
 
             if current_line_number in self.only_lines:
                 if last_shown_line_number + 1 != current_line_number:
                     yield 0, '<tr>'
                     yield 0, '<td class="line">...</td>'
                     yield 0, '<td id="hlcode" class="code"></td>'
                     yield 0, '</tr>'
 
                 yield 0, '<tr>'
                 if self.url:
                     yield 0, '<td class="line"><a href="%s#L%i">%i</a></td>' % (
                         self.url, current_line_number, current_line_number)
                 else:
                     yield 0, '<td class="line"><a href="">%i</a></td>' % (
                         current_line_number)
                 yield 0, '<td id="hlcode" class="code">' + line + '</td>'
                 yield 0, '</tr>'
 
                 last_shown_line_number = current_line_number
 
             current_line_number += 1
 
 
         yield 0, '</table>'
 
 
 def extract_phrases(text_query):
     """
     Extracts phrases from search term string making sure phrases
     contained in double quotes are kept together - and discarding empty values
     or fully whitespace values eg.
 
     'some   text "a phrase" more' => ['some', 'text', 'a phrase', 'more']
 
     """
 
     in_phrase = False
     buf = ''
     phrases = []
     for char in text_query:
         if in_phrase:
             if char == '"': # end phrase
                 phrases.append(buf)
                 buf = ''
                 in_phrase = False
                 continue
             else:
                 buf += char
                 continue
         else:
             if char == '"': # start phrase
                 in_phrase = True
                 phrases.append(buf)
                 buf = ''
                 continue
             elif char == ' ':
                 phrases.append(buf)
                 buf = ''
                 continue
             else:
                 buf += char
 
     phrases.append(buf)
     phrases = [phrase.strip() for phrase in phrases if phrase.strip()]
     return phrases
 
 
 def get_matching_offsets(text, phrases):
     """
     Returns a list of string offsets in `text` that the list of `terms` match
 
     >>> get_matching_offsets('some text here', ['some', 'here'])
     [(0, 4), (10, 14)]
 
     """
     offsets = []
     for phrase in phrases:
         for match in re.finditer(phrase, text):
             offsets.append((match.start(), match.end()))
 
     return offsets
 
 
 def normalize_text_for_matching(x):
     """
     Replaces all non alnum characters to spaces and lower cases the string,
     useful for comparing two text strings without punctuation
     """
     return re.sub(r'[^\w]', ' ', x.lower())
 
 
 def get_matching_line_offsets(lines, terms):
     """ Return a set of `lines` indices (starting from 1) matching a
     text search query, along with `context` lines above/below matching lines
 
     :param lines: list of strings representing lines
     :param terms: search term string to match in lines eg. 'some text'
     :param context: number of lines above/below a matching line to add to result
     :param max_lines: cut off for lines of interest
      eg.
 
     text = '''
     words words words
     words words words
     some text some
     words words words
     words words words
     text here what
     '''
     get_matching_line_offsets(text, 'text', context=1)
     {3: [(5, 9)], 6: [(0, 4)]]
 
     """
     matching_lines = {}
     phrases = [normalize_text_for_matching(phrase)
                for phrase in extract_phrases(terms)]
 
     for line_index, line in enumerate(lines, start=1):
         match_offsets = get_matching_offsets(
             normalize_text_for_matching(line), phrases)
         if match_offsets:
             matching_lines[line_index] = match_offsets
 
     return matching_lines
 
 
 def get_lexer_safe(mimetype=None, filepath=None):
     """
     Tries to return a relevant pygments lexer using mimetype/filepath name,
     defaulting to plain text if none could be found
     """
     lexer = None
     try:
         if mimetype:
             lexer = get_lexer_for_mimetype(mimetype)
         if not lexer:
             lexer = get_lexer_for_filename(filepath)
     except pygments.util.ClassNotFound:
         pass
 
     if not lexer:
         lexer = get_lexer_by_name('text')
 
     return lexer
 
 
 def pygmentize(filenode, **kwargs):
     """
     pygmentize function using pygments
 
     :param filenode:
     """
     lexer = get_custom_lexer(filenode.extension) or filenode.lexer
     return literal(code_highlight(filenode.content, lexer,
                                   CodeHtmlFormatter(**kwargs)))
 
 
 def pygmentize_annotation(repo_name, filenode, **kwargs):
     """
     pygmentize function for annotation
 
     :param filenode:
     """
 
     color_dict = {}
 
     def gen_color(n=10000):
         """generator for getting n of evenly distributed colors using
         hsv color and golden ratio. It always return same order of colors
 
         :returns: RGB tuple
         """
 
         def hsv_to_rgb(h, s, v):
             if s == 0.0:
                 return v, v, v
             i = int(h * 6.0)  # XXX assume int() truncates!
             f = (h * 6.0) - i
             p = v * (1.0 - s)
             q = v * (1.0 - s * f)
             t = v * (1.0 - s * (1.0 - f))
             i = i % 6
             if i == 0:
                 return v, t, p
             if i == 1:
                 return q, v, p
             if i == 2:
                 return p, v, t
             if i == 3:
                 return p, q, v
             if i == 4:
                 return t, p, v
             if i == 5:
                 return v, p, q
 
         golden_ratio = 0.618033988749895
         h = 0.22717784590367374
 
         for _ in xrange(n):
             h += golden_ratio
             h %= 1
             HSV_tuple = [h, 0.95, 0.95]
             RGB_tuple = hsv_to_rgb(*HSV_tuple)
             yield map(lambda x: str(int(x * 256)), RGB_tuple)
 
     cgenerator = gen_color()
 
     def get_color_string(commit_id):
         if commit_id in color_dict:
             col = color_dict[commit_id]
         else:
             col = color_dict[commit_id] = cgenerator.next()
         return "color: rgb(%s)! important;" % (', '.join(col))
 
     def url_func(repo_name):
 
         def _url_func(commit):
             author = commit.author
             date = commit.date
             message = tooltip(commit.message)
 
             tooltip_html = ("<div style='font-size:0.8em'><b>Author:</b>"
                             " %s<br/><b>Date:</b> %s</b><br/><b>Message:"
                             "</b> %s<br/></div>")
 
             tooltip_html = tooltip_html % (author, date, message)
             lnk_format = '%5s:%s' % ('r%s' % commit.idx, commit.short_id)
             uri = link_to(
                 lnk_format,
                 url('changeset_home', repo_name=repo_name,
                     revision=commit.raw_id),
                 style=get_color_string(commit.raw_id),
                 class_='tooltip',
                 title=tooltip_html
             )
 
             uri += '\n'
             return uri
         return _url_func
 
     return literal(annotate_highlight(filenode, url_func(repo_name), **kwargs))
 
 
 def is_following_repo(repo_name, user_id):
     from rhodecode.model.scm import ScmModel
     return ScmModel().is_following_repo(repo_name, user_id)
 
 
 class _Message(object):
     """A message returned by ``Flash.pop_messages()``.
 
     Converting the message to a string returns the message text. Instances
     also have the following attributes:
 
     * ``message``: the message text.
     * ``category``: the category specified when the message was created.
     """
 
     def __init__(self, category, message):
         self.category = category
         self.message = message
 
     def __str__(self):
         return self.message
 
     __unicode__ = __str__
 
     def __html__(self):
         return escape(safe_unicode(self.message))
 
 
 class Flash(_Flash):
 
     def pop_messages(self):
         """Return all accumulated messages and delete them from the session.
 
         The return value is a list of ``Message`` objects.
         """
         from pylons import session
 
         messages = []
 
         # Pop the 'old' pylons flash messages. They are tuples of the form
         # (category, message)
         for cat, msg in session.pop(self.session_key, []):
             messages.append(_Message(cat, msg))
 
         # Pop the 'new' pyramid flash messages for each category as list
         # of strings.
         for cat in self.categories:
             for msg in session.pop_flash(queue=cat):
                 messages.append(_Message(cat, msg))
         # Map messages from the default queue to the 'notice' category.
         for msg in session.pop_flash():
             messages.append(_Message('notice', msg))
 
         session.save()
         return messages
 
 flash = Flash()
 
 #==============================================================================
 # SCM FILTERS available via h.
 #==============================================================================
 from rhodecode.lib.vcs.utils import author_name, author_email
 from rhodecode.lib.utils2 import credentials_filter, age as _age
 from rhodecode.model.db import User, ChangesetStatus
 
 age = _age
 capitalize = lambda x: x.capitalize()
 email = author_email
 short_id = lambda x: x[:12]
 hide_credentials = lambda x: ''.join(credentials_filter(x))
 
 
 def age_component(datetime_iso, value=None, time_is_local=False):
     title = value or format_date(datetime_iso)
 
     # detect if we have a timezone info, otherwise, add it
     if isinstance(datetime_iso, datetime) and not datetime_iso.tzinfo:
         tzinfo = '+00:00'
 
         if time_is_local:
             tzinfo = time.strftime("+%H:%M",
                 time.gmtime(
                     (datetime.now() - datetime.utcnow()).seconds + 1
                 )
             )
 
     return literal(
         '<time class="timeago tooltip" '
         'title="{1}" datetime="{0}{2}">{1}</time>'.format(
             datetime_iso, title, tzinfo))
 
 
 def _shorten_commit_id(commit_id):
     from rhodecode import CONFIG
     def_len = safe_int(CONFIG.get('rhodecode_show_sha_length', 12))
     return commit_id[:def_len]
 
 
 def show_id(commit):
     """
     Configurable function that shows ID
     by default it's r123:fffeeefffeee
 
     :param commit: commit instance
     """
     from rhodecode import CONFIG
     show_idx = str2bool(CONFIG.get('rhodecode_show_revision_number', True))
 
     raw_id = _shorten_commit_id(commit.raw_id)
     if show_idx:
         return 'r%s:%s' % (commit.idx, raw_id)
     else:
         return '%s' % (raw_id, )
 
 
 def format_date(date):
     """
     use a standardized formatting for dates used in RhodeCode
 
     :param date: date/datetime object
     :return: formatted date
     """
 
     if date:
         _fmt = "%a, %d %b %Y %H:%M:%S"
         return safe_unicode(date.strftime(_fmt))
 
     return u""
 
 
 class _RepoChecker(object):
 
     def __init__(self, backend_alias):
         self._backend_alias = backend_alias
 
     def __call__(self, repository):
         if hasattr(repository, 'alias'):
             _type = repository.alias
         elif hasattr(repository, 'repo_type'):
             _type = repository.repo_type
         else:
             _type = repository
         return _type == self._backend_alias
 
 is_git = _RepoChecker('git')
 is_hg = _RepoChecker('hg')
 is_svn = _RepoChecker('svn')
 
 
 def get_repo_type_by_name(repo_name):
     repo = Repository.get_by_repo_name(repo_name)
     return repo.repo_type
 
 
 def is_svn_without_proxy(repository):
     from rhodecode import CONFIG
     if is_svn(repository):
         if not CONFIG.get('rhodecode_proxy_subversion_http_requests', False):
             return True
     return False
 
 
 def discover_user(author):
     """
     Tries to discover RhodeCode User based on the autho string. Author string
     is typically `FirstName LastName <[e-mail]>`
     """
 
     # if author is already an instance use it for extraction
     if isinstance(author, User):
         return author
 
     # Valid email in the attribute passed, see if they're in the system
     _email = author_email(author)
     if _email != '':
         user = User.get_by_email(_email, case_insensitive=True, cache=True)
         if user is not None:
             return user
 
     # Maybe it's a username, we try to extract it and fetch by username ?
     _author = author_name(author)
     user = User.get_by_username(_author, case_insensitive=True, cache=True)
     if user is not None:
         return user
 
     return None
 
 
 def email_or_none(author):
     # extract email from the commit string
     _email = author_email(author)
 
     # If we have an email, use it, otherwise
     # see if it contains a username we can get an email from
     if _email != '':
         return _email
     else:
         user = User.get_by_username(author_name(author), case_insensitive=True,
                                 cache=True)
     
     if user is not None:
             return user.email
 
     # No valid email, not a valid user in the system, none!
     return None
 
 
 def link_to_user(author, length=0, **kwargs):
     user = discover_user(author)
     # user can be None, but if we have it already it means we can re-use it
     # in the person() function, so we save 1 intensive-query
     if user:
         author = user
 
     display_person = person(author, 'username_or_name_or_email')
     if length:
         display_person = shorter(display_person, length)
 
     if user:
         return link_to(
             escape(display_person),
             url('user_profile', username=user.username),
             **kwargs)
     else:
         return escape(display_person)
 
 
 def person(author, show_attr="username_and_name"):
     user = discover_user(author)
     if user:
         return getattr(user, show_attr)
     else:
         _author = author_name(author)
         _email = email(author)
         return _author or _email
 
 
 def author_string(email):
     if email:
         user = User.get_by_email(email, case_insensitive=True, cache=True)
         if user:
             if user.firstname or user.lastname:
                 return '%s %s &lt;%s&gt;' % (user.firstname, user.lastname, email)
             else:
                 return email
         else:
             return email
     else:
         return None
 
 
 def person_by_id(id_, show_attr="username_and_name"):
     # attr to return from fetched user
     person_getter = lambda usr: getattr(usr, show_attr)
 
     #maybe it's an ID ?
     if str(id_).isdigit() or isinstance(id_, int):
         id_ = int(id_)
         user = User.get(id_)
         if user is not None:
             return person_getter(user)
     return id_
 
 
 def gravatar_with_user(author, show_disabled=False):
     from rhodecode.lib.utils import PartialRenderer
     _render = PartialRenderer('base/base.html')
     return _render('gravatar_with_user', author, show_disabled=show_disabled)
 
 
 def desc_stylize(value):
     """
     converts tags from value into html equivalent
 
     :param value:
     """
     if not value:
         return ''
 
     value = re.sub(r'\[see\ \=\>\ *([a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\]',
                    '<div class="metatag" tag="see">see =&gt; \\1 </div>', value)
     value = re.sub(r'\[license\ \=\>\ *([a-zA-Z0-9\/\=\?\&\ \:\/\.\-]*)\]',
                    '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/\\1">\\1</a></div>', value)
     value = re.sub(r'\[(requires|recommends|conflicts|base)\ \=\>\ *([a-zA-Z0-9\-\/]*)\]',
                    '<div class="metatag" tag="\\1">\\1 =&gt; <a href="/\\2">\\2</a></div>', value)
     value = re.sub(r'\[(lang|language)\ \=\>\ *([a-zA-Z\-\/\#\+]*)\]',
                    '<div class="metatag" tag="lang">\\2</div>', value)
     value = re.sub(r'\[([a-z]+)\]',
                    '<div class="metatag" tag="\\1">\\1</div>', value)
 
     return value
 
 
 def escaped_stylize(value):
     """
     converts tags from value into html equivalent, but escaping its value first
     """
     if not value:
         return ''
 
     # Using default webhelper escape method, but has to force it as a
     # plain unicode instead of a markup tag to be used in regex expressions
     value = unicode(escape(safe_unicode(value)))
 
     value = re.sub(r'\[see\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]',
                    '<div class="metatag" tag="see">see =&gt; \\1 </div>', value)
     value = re.sub(r'\[license\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]',
                    '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/\\1">\\1</a></div>', value)
     value = re.sub(r'\[(requires|recommends|conflicts|base)\ \=\&gt;\ *([a-zA-Z0-9\-\/]*)\]',
                    '<div class="metatag" tag="\\1">\\1 =&gt; <a href="/\\2">\\2</a></div>', value)
     value = re.sub(r'\[(lang|language)\ \=\&gt;\ *([a-zA-Z\-\/\#\+]*)\]',
                    '<div class="metatag" tag="lang">\\2</div>', value)
     value = re.sub(r'\[([a-z]+)\]',
                    '<div class="metatag" tag="\\1">\\1</div>', value)
 
     return value
 
 
 def bool2icon(value):
     """
     Returns boolean value of a given value, represented as html element with
     classes that will represent icons
 
     :param value: given value to convert to html node
     """
 
     if value:  # does bool conversion
         return HTML.tag('i', class_="icon-true")
     else:  # not true as bool
         return HTML.tag('i', class_="icon-false")
 
 
 #==============================================================================
 # PERMS
 #==============================================================================
 from rhodecode.lib.auth import HasPermissionAny, HasPermissionAll, \
 HasRepoPermissionAny, HasRepoPermissionAll, HasRepoGroupPermissionAll, \
 HasRepoGroupPermissionAny, HasRepoPermissionAnyApi, get_csrf_token
 
 
 #==============================================================================
 # GRAVATAR URL
 #==============================================================================
 class InitialsGravatar(object):
     def __init__(self, email_address, first_name, last_name, size=30,
                  background=None, text_color='#fff'):
         self.size = size
         self.first_name = first_name
         self.last_name = last_name
         self.email_address = email_address
         self.background = background or self.str2color(email_address)
         self.text_color = text_color
 
     def get_color_bank(self):
         """
         returns a predefined list of colors that gravatars can use.
         Those are randomized distinct colors that guarantee readability and
         uniqueness.
 
         generated with: http://phrogz.net/css/distinct-colors.html
         """
         return [
             '#bf3030', '#a67f53', '#00ff00', '#5989b3', '#392040', '#d90000',
             '#402910', '#204020', '#79baf2', '#a700b3', '#bf6060', '#7f5320',
             '#008000', '#003059', '#ee00ff', '#ff0000', '#8c4b00', '#007300',
             '#005fb3', '#de73e6', '#ff4040', '#ffaa00', '#3df255', '#203140',
             '#47004d', '#591616', '#664400', '#59b365', '#0d2133', '#83008c',
             '#592d2d', '#bf9f60', '#73e682', '#1d3f73', '#73006b', '#402020',
             '#b2862d', '#397341', '#597db3', '#e600d6', '#a60000', '#736039',
             '#00b318', '#79aaf2', '#330d30', '#ff8080', '#403010', '#16591f',
             '#002459', '#8c4688', '#e50000', '#ffbf40', '#00732e', '#102340',
             '#bf60ac', '#8c4646', '#cc8800', '#00a642', '#1d3473', '#b32d98',
             '#660e00', '#ffd580', '#80ffb2', '#7391e6', '#733967', '#d97b6c',
             '#8c5e00', '#59b389', '#3967e6', '#590047', '#73281d', '#665200',
             '#00e67a', '#2d50b3', '#8c2377', '#734139', '#b2982d', '#16593a',
             '#001859', '#ff00aa', '#a65e53', '#ffcc00', '#0d3321', '#2d3959',
             '#731d56', '#401610', '#4c3d00', '#468c6c', '#002ca6', '#d936a3',
             '#d94c36', '#403920', '#36d9a3', '#0d1733', '#592d4a', '#993626',
             '#cca300', '#00734d', '#46598c', '#8c005e', '#7f1100', '#8c7000',
             '#00a66f', '#7382e6', '#b32d74', '#d9896c', '#ffe680', '#1d7362',
             '#364cd9', '#73003d', '#d93a00', '#998a4d', '#59b3a1', '#5965b3',
             '#e5007a', '#73341d', '#665f00', '#00b38f', '#0018b3', '#59163a',
             '#b2502d', '#bfb960', '#00ffcc', '#23318c', '#a6537f', '#734939',
             '#b2a700', '#104036', '#3d3df2', '#402031', '#e56739', '#736f39',
             '#79f2ea', '#000059', '#401029', '#4c1400', '#ffee00', '#005953',
             '#101040', '#990052', '#402820', '#403d10', '#00ffee', '#0000d9',
             '#ff80c4', '#a66953', '#eeff00', '#00ccbe', '#8080ff', '#e673a1',
             '#a62c00', '#474d00', '#1a3331', '#46468c', '#733950', '#662900',
             '#858c23', '#238c85', '#0f0073', '#b20047', '#d9986c', '#becc00',
             '#396f73', '#281d73', '#ff0066', '#ff6600', '#dee673', '#59adb3',
             '#6559b3', '#590024', '#b2622d', '#98b32d', '#36ced9', '#332d59',
             '#40001a', '#733f1d', '#526600', '#005359', '#242040', '#bf6079',
             '#735039', '#cef23d', '#007780', '#5630bf', '#66001b', '#b24700',
             '#acbf60', '#1d6273', '#25008c', '#731d34', '#a67453', '#50592d',
             '#00ccff', '#6600ff', '#ff0044', '#4c1f00', '#8a994d', '#79daf2',
             '#a173e6', '#d93662', '#402310', '#aaff00', '#2d98b3', '#8c40ff',
             '#592d39', '#ff8c40', '#354020', '#103640', '#1a0040', '#331a20',
             '#331400', '#334d00', '#1d5673', '#583973', '#7f0022', '#4c3626',
             '#88cc00', '#36a3d9', '#3d0073', '#d9364c', '#33241a', '#698c23',
             '#5995b3', '#300059', '#e57382', '#7f3300', '#366600', '#00aaff',
             '#3a1659', '#733941', '#663600', '#74b32d', '#003c59', '#7f53a6',
             '#73000f', '#ff8800', '#baf279', '#79caf2', '#291040', '#a6293a',
             '#b2742d', '#587339', '#0077b3', '#632699', '#400009', '#d9a66c',
             '#294010', '#2d4a59', '#aa00ff', '#4c131b', '#b25f00', '#5ce600',
             '#267399', '#a336d9', '#990014', '#664e33', '#86bf60', '#0088ff',
             '#7700b3', '#593a16', '#073300', '#1d4b73', '#ac60bf', '#e59539',
             '#4f8c46', '#368dd9', '#5c0073'
         ]
 
     def rgb_to_hex_color(self, rgb_tuple):
         """
         Converts an rgb_tuple passed to an hex color.
 
         :param rgb_tuple: tuple with 3 ints represents rgb color space
         """
         return '#' + ("".join(map(chr, rgb_tuple)).encode('hex'))
 
     def email_to_int_list(self, email_str):
         """
         Get every byte of the hex digest value of email and turn it to integer.
         It's going to be always between 0-255
         """
         digest = md5_safe(email_str.lower())
         return [int(digest[i * 2:i * 2 + 2], 16) for i in range(16)]
 
     def pick_color_bank_index(self, email_str, color_bank):
         return self.email_to_int_list(email_str)[0] % len(color_bank)
 
     def str2color(self, email_str):
         """
         Tries to map in a stable algorithm an email to color
 
         :param email_str:
         """
         color_bank = self.get_color_bank()
         # pick position (module it's length so we always find it in the
         # bank even if it's smaller than 256 values
         pos = self.pick_color_bank_index(email_str, color_bank)
         return color_bank[pos]
 
     def normalize_email(self, email_address):
         import unicodedata
         # default host used to fill in the fake/missing email
         default_host = u'localhost'
 
         if not email_address:
             email_address = u'%s@%s' % (User.DEFAULT_USER, default_host)
 
         email_address = safe_unicode(email_address)
 
         if u'@' not in email_address:
             email_address = u'%s@%s' % (email_address, default_host)
 
         if email_address.endswith(u'@'):
             email_address = u'%s%s' % (email_address, default_host)
 
         email_address = unicodedata.normalize('NFKD', email_address)\
             .encode('ascii', 'ignore')
         return email_address
 
     def get_initials(self):
         """
         Returns 2 letter initials calculated based on the input.
         The algorithm picks first given email address, and takes first letter
         of part before @, and then the first letter of server name. In case
         the part before @ is in a format of `somestring.somestring2` it replaces
         the server letter with first letter of somestring2
 
         In case function was initialized with both first and lastname, this
         overrides the extraction from email by first letter of the first and
         last name. We add special logic to that functionality, In case Full name
         is compound, like Guido Von Rossum, we use last part of the last name
         (Von Rossum) picking `R`.
 
         Function also normalizes the non-ascii characters to they ascii
         representation, eg Ą => A
         """
         import unicodedata
         # replace non-ascii to ascii
         first_name = unicodedata.normalize(
             'NFKD', safe_unicode(self.first_name)).encode('ascii', 'ignore')
         last_name = unicodedata.normalize(
             'NFKD', safe_unicode(self.last_name)).encode('ascii', 'ignore')
 
         # do NFKD encoding, and also make sure email has proper format
         email_address = self.normalize_email(self.email_address)
 
         # first push the email initials
         prefix, server = email_address.split('@', 1)
 
         # check if prefix is maybe a 'firstname.lastname' syntax
         _dot_split = prefix.rsplit('.', 1)
         if len(_dot_split) == 2:
             initials = [_dot_split[0][0], _dot_split[1][0]]
         else:
             initials = [prefix[0], server[0]]
 
         # then try to replace either firtname or lastname
         fn_letter = (first_name or " ")[0].strip()
         ln_letter = (last_name.split(' ', 1)[-1] or " ")[0].strip()
 
         if fn_letter:
             initials[0] = fn_letter
 
         if ln_letter:
             initials[1] = ln_letter
 
         return ''.join(initials).upper()
 
     def get_img_data_by_type(self, font_family, img_type):
         default_user = """
         <svg xmlns="http://www.w3.org/2000/svg"
         version="1.1" x="0px" y="0px" width="{size}" height="{size}"
         viewBox="-15 -10 439.165 429.164"
 
         xml:space="preserve"
         style="background:{background};" >
 
         <path d="M204.583,216.671c50.664,0,91.74-48.075,
                  91.74-107.378c0-82.237-41.074-107.377-91.74-107.377
                  c-50.668,0-91.74,25.14-91.74,107.377C112.844,
                  168.596,153.916,216.671,
                  204.583,216.671z" fill="{text_color}"/>
         <path d="M407.164,374.717L360.88,
                  270.454c-2.117-4.771-5.836-8.728-10.465-11.138l-71.83-37.392
                  c-1.584-0.823-3.502-0.663-4.926,0.415c-20.316,
                  15.366-44.203,23.488-69.076,23.488c-24.877,
                  0-48.762-8.122-69.078-23.488
                  c-1.428-1.078-3.346-1.238-4.93-0.415L58.75,
                  259.316c-4.631,2.41-8.346,6.365-10.465,11.138L2.001,374.717
                  c-3.191,7.188-2.537,15.412,1.75,22.005c4.285,
                  6.592,11.537,10.526,19.4,10.526h362.861c7.863,0,15.117-3.936,
                  19.402-10.527 C409.699,390.129,
                  410.355,381.902,407.164,374.717z" fill="{text_color}"/>
         </svg>""".format(
             size=self.size,
             background='#979797',  # @grey4
             text_color=self.text_color,
             font_family=font_family)
 
         return {
             "default_user": default_user
         }[img_type]
 
     def get_img_data(self, svg_type=None):
         """
         generates the svg metadata for image
         """
 
         font_family = ','.join([
             'proximanovaregular',
             'Proxima Nova Regular',
             'Proxima Nova',
             'Arial',
             'Lucida Grande',
             'sans-serif'
             ])
         if svg_type:
             return self.get_img_data_by_type(font_family, svg_type)
 
         initials = self.get_initials()
         img_data = """
         <svg xmlns="http://www.w3.org/2000/svg" pointer-events="none"
              width="{size}" height="{size}"
              style="width: 100%; height: 100%; background-color: {background}"
              viewBox="0 0 {size} {size}">
             <text text-anchor="middle" y="50%" x="50%" dy="0.35em"
                   pointer-events="auto" fill="{text_color}"
                   font-family="{font_family}"
                   style="font-weight: 400; font-size: {f_size}px;">{text}
             </text>
         </svg>""".format(
             size=self.size,
             f_size=self.size/1.85,  # scale the text inside the box nicely
             background=self.background,
             text_color=self.text_color,
             text=initials.upper(),
             font_family=font_family)
 
         return img_data
 
     def generate_svg(self, svg_type=None):
         img_data = self.get_img_data(svg_type)
         return "data:image/svg+xml;base64,%s" % img_data.encode('base64')
 
 
 def initials_gravatar(email_address, first_name, last_name, size=30):
     svg_type = None
     if email_address == User.DEFAULT_USER_EMAIL:
         svg_type = 'default_user'
     klass = InitialsGravatar(email_address, first_name, last_name, size)
     return klass.generate_svg(svg_type=svg_type)
 
 
 def gravatar_url(email_address, size=30):
     # doh, we need to re-import those to mock it later
     from pylons import tmpl_context as c
 
     _use_gravatar = c.visual.use_gravatar
     _gravatar_url = c.visual.gravatar_url or User.DEFAULT_GRAVATAR_URL
 
     email_address = email_address or User.DEFAULT_USER_EMAIL
     if isinstance(email_address, unicode):
         # hashlib crashes on unicode items
         email_address = safe_str(email_address)
 
     # empty email or default user
     if not email_address or email_address == User.DEFAULT_USER_EMAIL:
         return initials_gravatar(User.DEFAULT_USER_EMAIL, '', '', size=size)
 
     if _use_gravatar:
         # TODO: Disuse pyramid thread locals. Think about another solution to
         # get the host and schema here.
         request = get_current_request()
         tmpl = safe_str(_gravatar_url)
         tmpl = tmpl.replace('{email}', email_address)\
                    .replace('{md5email}', md5_safe(email_address.lower())) \
                    .replace('{netloc}', request.host)\
                    .replace('{scheme}', request.scheme)\
                    .replace('{size}', safe_str(size))
         return tmpl
     else:
         return initials_gravatar(email_address, '', '', size=size)
 
 
 class Page(_Page):
     """
     Custom pager to match rendering style with paginator
     """
 
     def _get_pos(self, cur_page, max_page, items):
         edge = (items / 2) + 1
         if (cur_page <= edge):
             radius = max(items / 2, items - cur_page)
         elif (max_page - cur_page) < edge:
             radius = (items - 1) - (max_page - cur_page)
         else:
             radius = items / 2
 
         left = max(1, (cur_page - (radius)))
         right = min(max_page, cur_page + (radius))
         return left, cur_page, right
 
     def _range(self, regexp_match):
         """
         Return range of linked pages (e.g. '1 2 [3] 4 5 6 7 8').
 
         Arguments:
 
         regexp_match
             A "re" (regular expressions) match object containing the
             radius of linked pages around the current page in
             regexp_match.group(1) as a string
 
         This function is supposed to be called as a callable in
         re.sub.
 
         """
         radius = int(regexp_match.group(1))
 
         # Compute the first and last page number within the radius
         # e.g. '1 .. 5 6 [7] 8 9 .. 12'
         # -> leftmost_page  = 5
         # -> rightmost_page = 9
         leftmost_page, _cur, rightmost_page = self._get_pos(self.page,
                                                             self.last_page,
                                                             (radius * 2) + 1)
         nav_items = []
 
         # Create a link to the first page (unless we are on the first page
         # or there would be no need to insert '..' spacers)
         if self.page != self.first_page and self.first_page < leftmost_page:
             nav_items.append(self._pagerlink(self.first_page, self.first_page))
 
         # Insert dots if there are pages between the first page
         # and the currently displayed page range
         if leftmost_page - self.first_page > 1:
             # Wrap in a SPAN tag if nolink_attr is set
             text = '..'
             if self.dotdot_attr:
                 text = HTML.span(c=text, **self.dotdot_attr)
             nav_items.append(text)
 
         for thispage in xrange(leftmost_page, rightmost_page + 1):
             # Hilight the current page number and do not use a link
             if thispage == self.page:
                 text = '%s' % (thispage,)
                 # Wrap in a SPAN tag if nolink_attr is set
                 if self.curpage_attr:
                     text = HTML.span(c=text, **self.curpage_attr)
                 nav_items.append(text)
             # Otherwise create just a link to that page
             else:
                 text = '%s' % (thispage,)
                 nav_items.append(self._pagerlink(thispage, text))
 
         # Insert dots if there are pages between the displayed
         # page numbers and the end of the page range
         if self.last_page - rightmost_page > 1:
             text = '..'
             # Wrap in a SPAN tag if nolink_attr is set
             if self.dotdot_attr:
                 text = HTML.span(c=text, **self.dotdot_attr)
             nav_items.append(text)
 
         # Create a link to the very last page (unless we are on the last
         # page or there would be no need to insert '..' spacers)
         if self.page != self.last_page and rightmost_page < self.last_page:
             nav_items.append(self._pagerlink(self.last_page, self.last_page))
 
         ## prerender links
         #_page_link = url.current()
         #nav_items.append(literal('<link rel="prerender" href="%s?page=%s">' % (_page_link, str(int(self.page)+1))))
         #nav_items.append(literal('<link rel="prefetch" href="%s?page=%s">' % (_page_link, str(int(self.page)+1))))
         return self.separator.join(nav_items)
 
     def pager(self, format='~2~', page_param='page', partial_param='partial',
         show_if_single_page=False, separator=' ', onclick=None,
         symbol_first='<<', symbol_last='>>',
         symbol_previous='<', symbol_next='>',
         link_attr={'class': 'pager_link', 'rel': 'prerender'},
         curpage_attr={'class': 'pager_curpage'},
         dotdot_attr={'class': 'pager_dotdot'}, **kwargs):
 
         self.curpage_attr = curpage_attr
         self.separator = separator
         self.pager_kwargs = kwargs
         self.page_param = page_param
         self.partial_param = partial_param
         self.onclick = onclick
         self.link_attr = link_attr
         self.dotdot_attr = dotdot_attr
 
         # Don't show navigator if there is no more than one page
         if self.page_count == 0 or (self.page_count == 1 and not show_if_single_page):
             return ''
 
         from string import Template
         # Replace ~...~ in token format by range of pages
         result = re.sub(r'~(\d+)~', self._range, format)
 
         # Interpolate '%' variables
         result = Template(result).safe_substitute({
             'first_page': self.first_page,
             'last_page': self.last_page,
             'page': self.page,
             'page_count': self.page_count,
             'items_per_page': self.items_per_page,
             'first_item': self.first_item,
             'last_item': self.last_item,
             'item_count': self.item_count,
             'link_first': self.page > self.first_page and \
                     self._pagerlink(self.first_page, symbol_first) or '',
             'link_last': self.page < self.last_page and \
                     self._pagerlink(self.last_page, symbol_last) or '',
             'link_previous': self.previous_page and \
                     self._pagerlink(self.previous_page, symbol_previous) \
                     or HTML.span(symbol_previous, class_="pg-previous disabled"),
             'link_next': self.next_page and \
                     self._pagerlink(self.next_page, symbol_next) \
                     or HTML.span(symbol_next, class_="pg-next disabled")
         })
 
         return literal(result)
 
 
 #==============================================================================
 # REPO PAGER, PAGER FOR REPOSITORY
 #==============================================================================
 class RepoPage(Page):
 
     def __init__(self, collection, page=1, items_per_page=20,
                  item_count=None, url=None, **kwargs):
 
         """Create a "RepoPage" instance. special pager for paging
         repository
         """
         self._url_generator = url
 
         # Safe the kwargs class-wide so they can be used in the pager() method
         self.kwargs = kwargs
 
         # Save a reference to the collection
         self.original_collection = collection
 
         self.collection = collection
 
         # The self.page is the number of the current page.
         # The first page has the number 1!
         try:
             self.page = int(page)  # make it int() if we get it as a string
         except (ValueError, TypeError):
             self.page = 1
 
         self.items_per_page = items_per_page
 
         # Unless the user tells us how many items the collections has
         # we calculate that ourselves.
         if item_count is not None:
             self.item_count = item_count
         else:
             self.item_count = len(self.collection)
 
         # Compute the number of the first and last available page
         if self.item_count > 0:
             self.first_page = 1
             self.page_count = int(math.ceil(float(self.item_count) /
                                             self.items_per_page))
             self.last_page = self.first_page + self.page_count - 1
 
             # Make sure that the requested page number is the range of
             # valid pages
             if self.page > self.last_page:
                 self.page = self.last_page
             elif self.page < self.first_page:
                 self.page = self.first_page
 
             # Note: the number of items on this page can be less than
             #       items_per_page if the last page is not full
             self.first_item = max(0, (self.item_count) - (self.page *
                                                           items_per_page))
             self.last_item = ((self.item_count - 1) - items_per_page *
                               (self.page - 1))
 
             self.items = list(self.collection[self.first_item:self.last_item + 1])
 
             # Links to previous and next page
             if self.page > self.first_page:
                 self.previous_page = self.page - 1
             else:
                 self.previous_page = None
 
             if self.page < self.last_page:
                 self.next_page = self.page + 1
             else:
                 self.next_page = None
 
         # No items available
         else:
             self.first_page = None
             self.page_count = 0
             self.last_page = None
             self.first_item = None
             self.last_item = None
             self.previous_page = None
             self.next_page = None
             self.items = []
 
         # This is a subclass of the 'list' type. Initialise the list now.
         list.__init__(self, reversed(self.items))
 
 
 def changed_tooltip(nodes):
     """
     Generates a html string for changed nodes in commit page.
     It limits the output to 30 entries
 
     :param nodes: LazyNodesGenerator
     """
     if nodes:
         pref = ': <br/> '
         suf = ''
         if len(nodes) > 30:
             suf = '<br/>' + _(' and %s more') % (len(nodes) - 30)
         return literal(pref + '<br/> '.join([safe_unicode(x.path)
                                              for x in nodes[:30]]) + suf)
     else:
         return ': ' + _('No Files')
 
 
 def breadcrumb_repo_link(repo):
     """
     Makes a breadcrumbs path link to repo
 
     ex::
         group >> subgroup >> repo
 
     :param repo: a Repository instance
     """
 
     path = [
         link_to(group.name, url('repo_group_home', group_name=group.group_name))
         for group in repo.groups_with_parents
     ] + [
         link_to(repo.just_name, url('summary_home', repo_name=repo.repo_name))
     ]
 
     return literal(' &raquo; '.join(path))
 
 
 def format_byte_size_binary(file_size):
     """
     Formats file/folder sizes to standard.
     """
     formatted_size = format_byte_size(file_size, binary=True)
     return formatted_size
 
 
 def fancy_file_stats(stats):
     """
     Displays a fancy two colored bar for number of added/deleted
     lines of code on file
 
     :param stats: two element list of added/deleted lines of code
     """
     from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
         MOD_FILENODE, RENAMED_FILENODE, CHMOD_FILENODE, BIN_FILENODE
 
     def cgen(l_type, a_v, d_v):
         mapping = {'tr': 'top-right-rounded-corner-mid',
                    'tl': 'top-left-rounded-corner-mid',
                    'br': 'bottom-right-rounded-corner-mid',
                    'bl': 'bottom-left-rounded-corner-mid'}
         map_getter = lambda x: mapping[x]
 
         if l_type == 'a' and d_v:
             #case when added and deleted are present
             return ' '.join(map(map_getter, ['tl', 'bl']))
 
         if l_type == 'a' and not d_v:
             return ' '.join(map(map_getter, ['tr', 'br', 'tl', 'bl']))
 
         if l_type == 'd' and a_v:
             return ' '.join(map(map_getter, ['tr', 'br']))
 
         if l_type == 'd' and not a_v:
             return ' '.join(map(map_getter, ['tr', 'br', 'tl', 'bl']))
 
     a, d = stats['added'], stats['deleted']
     width = 100
 
     if stats['binary']:  # binary operations like chmod/rename etc
         lbl = []
         bin_op = 0  # undefined
 
         # prefix with bin for binary files
         if BIN_FILENODE in stats['ops']:
             lbl += ['bin']
 
         if NEW_FILENODE in stats['ops']:
             lbl += [_('new file')]
             bin_op = NEW_FILENODE
         elif MOD_FILENODE in stats['ops']:
             lbl += [_('mod')]
             bin_op = MOD_FILENODE
         elif DEL_FILENODE in stats['ops']:
             lbl += [_('del')]
             bin_op = DEL_FILENODE
         elif RENAMED_FILENODE in stats['ops']:
             lbl += [_('rename')]
             bin_op = RENAMED_FILENODE
 
         # chmod can go with other operations, so we add a + to lbl if needed
         if CHMOD_FILENODE in stats['ops']:
             lbl += [_('chmod')]
             if bin_op == 0:
                 bin_op = CHMOD_FILENODE
 
         lbl = '+'.join(lbl)
         b_a = '<div class="bin bin%s %s" style="width:100%%">%s</div>' \
               % (bin_op, cgen('a', a_v='', d_v=0), lbl)
         b_d = '<div class="bin bin1" style="width:0%%"></div>'
         return literal('<div style="width:%spx">%s%s</div>' % (width, b_a, b_d))
 
     t = stats['added'] + stats['deleted']
     unit = float(width) / (t or 1)
 
     # needs > 9% of width to be visible or 0 to be hidden
     a_p = max(9, unit * a) if a > 0 else 0
     d_p = max(9, unit * d) if d > 0 else 0
     p_sum = a_p + d_p
 
     if p_sum > width:
         #adjust the percentage to be == 100% since we adjusted to 9
         if a_p > d_p:
             a_p = a_p - (p_sum - width)
         else:
             d_p = d_p - (p_sum - width)
 
     a_v = a if a > 0 else ''
     d_v = d if d > 0 else ''
 
     d_a = '<div class="added %s" style="width:%s%%">%s</div>' % (
         cgen('a', a_v, d_v), a_p, a_v
     )
     d_d = '<div class="deleted %s" style="width:%s%%">%s</div>' % (
         cgen('d', a_v, d_v), d_p, d_v
     )
     return literal('<div style="width:%spx">%s%s</div>' % (width, d_a, d_d))
 
 
 def urlify_text(text_, safe=True):
     """
     Extrac urls from text and make html links out of them
 
     :param text_:
     """
 
     url_pat = re.compile(r'''(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@#.&+]'''
                          '''|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)''')
 
     def url_func(match_obj):
         url_full = match_obj.groups()[0]
         return '<a href="%(url)s">%(url)s</a>' % ({'url': url_full})
     _newtext = url_pat.sub(url_func, text_)
     if safe:
         return literal(_newtext)
     return _newtext
 
 
 def urlify_commits(text_, repository):
     """
     Extract commit ids from text and make link from them
 
     :param text_:
     :param repository: repo name to build the URL with
     """
     from pylons import url  # doh, we need to re-import url to mock it later
     URL_PAT = re.compile(r'(^|\s)([0-9a-fA-F]{12,40})($|\s)')
 
     def url_func(match_obj):
         commit_id = match_obj.groups()[1]
         pref = match_obj.groups()[0]
         suf = match_obj.groups()[2]
 
         tmpl = (
             '%(pref)s<a class="%(cls)s" href="%(url)s">'
             '%(commit_id)s</a>%(suf)s'
         )
         return tmpl % {
             'pref': pref,
             'cls': 'revision-link',
             'url': url('changeset_home', repo_name=repository,
-                       revision=commit_id),
+                       revision=commit_id, qualified=True),
             'commit_id': commit_id,
             'suf': suf
         }
 
     newtext = URL_PAT.sub(url_func, text_)
 
     return newtext
 
 
-def _process_url_func(match_obj, repo_name, uid, entry):
+def _process_url_func(match_obj, repo_name, uid, entry,
+                      return_raw_data=False):
     pref = ''
     if match_obj.group().startswith(' '):
         pref = ' '
 
     issue_id = ''.join(match_obj.groups())
     tmpl = (
         '%(pref)s<a class="%(cls)s" href="%(url)s">'
         '%(issue-prefix)s%(id-repr)s'
         '</a>')
 
     (repo_name_cleaned,
      parent_group_name) = RepoGroupModel().\
         _get_group_name_and_parent(repo_name)
 
     # variables replacement
     named_vars = {
         'id': issue_id,
         'repo': repo_name,
         'repo_name': repo_name_cleaned,
         'group_name': parent_group_name
     }
     # named regex variables
     named_vars.update(match_obj.groupdict())
     _url = string.Template(entry['url']).safe_substitute(**named_vars)
 
-    return tmpl % {
+    data = {
         'pref': pref,
         'cls': 'issue-tracker-link',
         'url': _url,
         'id-repr': issue_id,
         'issue-prefix': entry['pref'],
         'serv': entry['url'],
     }
+    if return_raw_data:
+        return {
+            'id': issue_id,
+            'url': _url
+        }
+    return tmpl % data
 
 
-def process_patterns(text_string, repo_name, config):
+def process_patterns(text_string, repo_name, config=None):
     repo = None
     if repo_name:
         # Retrieving repo_name to avoid invalid repo_name to explode on
         # IssueTrackerSettingsModel but still passing invalid name further down
         repo = Repository.get_by_repo_name(repo_name, cache=True)
 
     settings_model = IssueTrackerSettingsModel(repo=repo)
     active_entries = settings_model.get_settings(cache=True)
 
+    issues_data = []
     newtext = text_string
     for uid, entry in active_entries.items():
-        url_func = partial(
-            _process_url_func, repo_name=repo_name, entry=entry, uid=uid)
-
         log.debug('found issue tracker entry with uid %s' % (uid,))
 
         if not (entry['pat'] and entry['url']):
             log.debug('skipping due to missing data')
             continue
 
         log.debug('issue tracker entry: uid: `%s` PAT:%s URL:%s PREFIX:%s'
                   % (uid, entry['pat'], entry['url'], entry['pref']))
 
         try:
             pattern = re.compile(r'%s' % entry['pat'])
         except re.error:
             log.exception(
                 'issue tracker pattern: `%s` failed to compile',
                 entry['pat'])
             continue
 
+        data_func = partial(
+            _process_url_func, repo_name=repo_name, entry=entry, uid=uid,
+            return_raw_data=True)
+
+        for match_obj in pattern.finditer(text_string):
+            issues_data.append(data_func(match_obj))
+
+        url_func = partial(
+            _process_url_func, repo_name=repo_name, entry=entry, uid=uid)
+
         newtext = pattern.sub(url_func, newtext)
         log.debug('processed prefix:uid `%s`' % (uid,))
 
-    return newtext
+    return newtext, issues_data
 
 
 def urlify_commit_message(commit_text, repository=None):
     """
     Parses given text message and makes proper links.
     issues are linked to given issue-server, and rest is a commit link
 
     :param commit_text:
     :param repository:
     """
     from pylons import url  # doh, we need to re-import url to mock it later
-    from rhodecode import CONFIG
 
     def escaper(string):
         return string.replace('<', '&lt;').replace('>', '&gt;')
 
     newtext = escaper(commit_text)
+
+    # extract http/https links and make them real urls
+    newtext = urlify_text(newtext, safe=False)
+
     # urlify commits - extract commit ids and make link out of them, if we have
     # the scope of repository present.
     if repository:
         newtext = urlify_commits(newtext, repository)
 
-    # extract http/https links and make them real urls
-    newtext = urlify_text(newtext, safe=False)
-
     # process issue tracker patterns
-    newtext = process_patterns(newtext, repository or '', CONFIG)
+    newtext, issues = process_patterns(newtext, repository or '')
 
     return literal(newtext)
 
 
 def rst(source, mentions=False):
     return literal('<div class="rst-block">%s</div>' %
                    MarkupRenderer.rst(source, mentions=mentions))
 
 
 def markdown(source, mentions=False):
     return literal('<div class="markdown-block">%s</div>' %
                    MarkupRenderer.markdown(source, flavored=True,
                                            mentions=mentions))
 
 def renderer_from_filename(filename, exclude=None):
     return MarkupRenderer.renderer_from_filename(filename, exclude=exclude)
 
 
 def render(source, renderer='rst', mentions=False):
     if renderer == 'rst':
         return rst(source, mentions=mentions)
     if renderer == 'markdown':
         return markdown(source, mentions=mentions)
 
 
 def commit_status(repo, commit_id):
     return ChangesetStatusModel().get_status(repo, commit_id)
 
 
 def commit_status_lbl(commit_status):
     return dict(ChangesetStatus.STATUSES).get(commit_status)
 
 
 def commit_time(repo_name, commit_id):
     repo = Repository.get_by_repo_name(repo_name)
     commit = repo.get_commit(commit_id=commit_id)
     return commit.date
 
 
 def get_permission_name(key):
     return dict(Permission.PERMS).get(key)
 
 
 def journal_filter_help():
     return _(
         'Example filter terms:\n' +
         '     repository:vcs\n' +
         '     username:marcin\n' +
         '     action:*push*\n' +
         '     ip:127.0.0.1\n' +
         '     date:20120101\n' +
         '     date:[20120101100000 TO 20120102]\n' +
         '\n' +
         'Generate wildcards using \'*\' character:\n' +
         '     "repository:vcs*" - search everything starting with \'vcs\'\n' +
         '     "repository:*vcs*" - search for repository containing \'vcs\'\n' +
         '\n' +
         'Optional AND / OR operators in queries\n' +
         '     "repository:vcs OR repository:test"\n' +
         '     "username:test AND repository:test*"\n'
     )
 
 
 def not_mapped_error(repo_name):
     flash(_('%s repository is not mapped to db perhaps'
             ' it was created or renamed from the filesystem'
             ' please run the application again'
             ' in order to rescan repositories') % repo_name, category='error')
 
 
 def ip_range(ip_addr):
     from rhodecode.model.db import UserIpMap
     s, e = UserIpMap._get_ip_range(ip_addr)
     return '%s - %s' % (s, e)
 
 
 def form(url, method='post', needs_csrf_token=True, **attrs):
     """Wrapper around webhelpers.tags.form to prevent CSRF attacks."""
     if method.lower() != 'get' and needs_csrf_token:
         raise Exception(
             'Forms to POST/PUT/DELETE endpoints should have (in general) a ' +
             'CSRF token. If the endpoint does not require such token you can ' +
             'explicitly set the parameter needs_csrf_token to false.')
 
     return wh_form(url, method=method, **attrs)
 
 
 def secure_form(url, method="POST", multipart=False, **attrs):
     """Start a form tag that points the action to an url. This
     form tag will also include the hidden field containing
     the auth token.
 
     The url options should be given either as a string, or as a
     ``url()`` function. The method for the form defaults to POST.
 
     Options:
 
     ``multipart``
         If set to True, the enctype is set to "multipart/form-data".
     ``method``
         The method to use when submitting the form, usually either
         "GET" or "POST". If "PUT", "DELETE", or another verb is used, a
         hidden input with name _method is added to simulate the verb
         over POST.
 
     """
     from webhelpers.pylonslib.secure_form import insecure_form
     from rhodecode.lib.auth import get_csrf_token, csrf_token_key
     form = insecure_form(url, method, multipart, **attrs)
     token = HTML.div(hidden(csrf_token_key, get_csrf_token()), style="display: none;")
     return literal("%s\n%s" % (form, token))
 
 def dropdownmenu(name, selected, options, enable_filter=False, **attrs):
     select_html = select(name, selected, options, **attrs)
     select2 = """
         <script>
             $(document).ready(function() {
                   $('#%s').select2({
                       containerCssClass: 'drop-menu',
                       dropdownCssClass: 'drop-menu-dropdown',
                       dropdownAutoWidth: true%s
                   });
             });
         </script>
     """
     filter_option = """,
                         minimumResultsForSearch: -1
     """
     input_id = attrs.get('id') or name
     filter_enabled = "" if enable_filter else filter_option
     select_script = literal(select2 % (input_id, filter_enabled))
 
     return literal(select_html+select_script)
 
 
 def get_visual_attr(tmpl_context_var, attr_name):
     """
     A safe way to get a variable from visual variable of template context
 
     :param tmpl_context_var: instance of tmpl_context, usually present as `c`
     :param attr_name: name of the attribute we fetch from the c.visual
     """
     visual = getattr(tmpl_context_var, 'visual', None)
     if not visual:
         return
     else:
         return getattr(visual, attr_name, None)
 
 
 def get_last_path_part(file_node):
     if not file_node.path:
         return u''
 
     path = safe_unicode(file_node.path.split('/')[-1])
     return u'../' + path
 
 
 def route_path(*args, **kwds):
     """
     Wrapper around pyramids `route_path` function. It is used to generate
     URLs from within pylons views or templates. This will be removed when
     pyramid migration if finished.
     """
     req = get_current_request()
     return req.route_path(*args, **kwds)
 
 
 def resource_path(*args, **kwds):
     """
     Wrapper around pyramids `route_path` function. It is used to generate
     URLs from within pylons views or templates. This will be removed when
     pyramid migration if finished.
     """
     req = get_current_request()
     return req.resource_path(*args, **kwds)

rhodecode/lib/hooks_daemon.py

@@ -1,260 +1,278 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import json
 import logging
+import urlparse
 import threading
 from BaseHTTPServer import BaseHTTPRequestHandler
 from SocketServer import TCPServer
+from routes.util import URLGenerator
 
 import Pyro4
+import pylons
+import rhodecode
 
 from rhodecode.lib import hooks_base
 from rhodecode.lib.utils2 import AttributeDict
 
 
 log = logging.getLogger(__name__)
 
 
 class HooksHttpHandler(BaseHTTPRequestHandler):
     def do_POST(self):
         method, extras = self._read_request()
         try:
             result = self._call_hook(method, extras)
         except Exception as e:
             result = {
                 'exception': e.__class__.__name__,
                 'exception_args': e.args
             }
         self._write_response(result)
 
     def _read_request(self):
         length = int(self.headers['Content-Length'])
         body = self.rfile.read(length).decode('utf-8')
         data = json.loads(body)
         return data['method'], data['extras']
 
     def _write_response(self, result):
         self.send_response(200)
         self.send_header("Content-type", "text/json")
         self.end_headers()
         self.wfile.write(json.dumps(result))
 
     def _call_hook(self, method, extras):
         hooks = Hooks()
         result = getattr(hooks, method)(extras)
         return result
 
     def log_message(self, format, *args):
         """
         This is an overriden method of BaseHTTPRequestHandler which logs using
         logging library instead of writing directly to stderr.
         """
 
         message = format % args
 
         # TODO: mikhail: add different log levels support
         log.debug(
             "%s - - [%s] %s", self.client_address[0],
             self.log_date_time_string(), message)
 
 
 class DummyHooksCallbackDaemon(object):
     def __init__(self):
         self.hooks_module = Hooks.__module__
 
     def __enter__(self):
         log.debug('Running dummy hooks callback daemon')
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         log.debug('Exiting dummy hooks callback daemon')
 
 
 class ThreadedHookCallbackDaemon(object):
 
     _callback_thread = None
     _daemon = None
     _done = False
 
     def __init__(self):
         self._prepare()
 
     def __enter__(self):
         self._run()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self._stop()
 
     def _prepare(self):
         raise NotImplementedError()
 
     def _run(self):
         raise NotImplementedError()
 
     def _stop(self):
         raise NotImplementedError()
 
 
 class Pyro4HooksCallbackDaemon(ThreadedHookCallbackDaemon):
     """
     Context manager which will run a callback daemon in a background thread.
     """
 
     hooks_uri = None
 
     def _prepare(self):
         log.debug("Preparing callback daemon and registering hook object")
         self._daemon = Pyro4.Daemon()
         hooks_interface = Hooks()
         self.hooks_uri = str(self._daemon.register(hooks_interface))
         log.debug("Hooks uri is: %s", self.hooks_uri)
 
     def _run(self):
         log.debug("Running event loop of callback daemon in background thread")
         callback_thread = threading.Thread(
             target=self._daemon.requestLoop,
             kwargs={'loopCondition': lambda: not self._done})
         callback_thread.daemon = True
         callback_thread.start()
         self._callback_thread = callback_thread
 
     def _stop(self):
         log.debug("Waiting for background thread to finish.")
         self._done = True
         self._callback_thread.join()
         self._daemon.close()
         self._daemon = None
         self._callback_thread = None
 
 
 class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
     """
     Context manager which will run a callback daemon in a background thread.
     """
 
     hooks_uri = None
 
     IP_ADDRESS = '127.0.0.1'
 
     # From Python docs: Polling reduces our responsiveness to a shutdown
     # request and wastes cpu at all other times.
     POLL_INTERVAL = 0.1
 
     def _prepare(self):
         log.debug("Preparing callback daemon and registering hook object")
 
         self._done = False
         self._daemon = TCPServer((self.IP_ADDRESS, 0), HooksHttpHandler)
         _, port = self._daemon.server_address
         self.hooks_uri = '{}:{}'.format(self.IP_ADDRESS, port)
 
         log.debug("Hooks uri is: %s", self.hooks_uri)
 
     def _run(self):
         log.debug("Running event loop of callback daemon in background thread")
         callback_thread = threading.Thread(
             target=self._daemon.serve_forever,
             kwargs={'poll_interval': self.POLL_INTERVAL})
         callback_thread.daemon = True
         callback_thread.start()
         self._callback_thread = callback_thread
 
     def _stop(self):
         log.debug("Waiting for background thread to finish.")
         self._daemon.shutdown()
         self._callback_thread.join()
         self._daemon = None
         self._callback_thread = None
 
 
 def prepare_callback_daemon(extras, protocol=None, use_direct_calls=False):
     callback_daemon = None
     protocol = protocol.lower() if protocol else None
 
     if use_direct_calls:
         callback_daemon = DummyHooksCallbackDaemon()
         extras['hooks_module'] = callback_daemon.hooks_module
     else:
         callback_daemon = (
             Pyro4HooksCallbackDaemon()
             if protocol == 'pyro4'
             else HttpHooksCallbackDaemon())
         extras['hooks_uri'] = callback_daemon.hooks_uri
         extras['hooks_protocol'] = protocol
 
     return callback_daemon, extras
 
 
 class Hooks(object):
     """
     Exposes the hooks for remote call backs
     """
 
     @Pyro4.callback
     def repo_size(self, extras):
         log.debug("Called repo_size of Hooks object")
         return self._call_hook(hooks_base.repo_size, extras)
 
     @Pyro4.callback
     def pre_pull(self, extras):
         log.debug("Called pre_pull of Hooks object")
         return self._call_hook(hooks_base.pre_pull, extras)
 
     @Pyro4.callback
     def post_pull(self, extras):
         log.debug("Called post_pull of Hooks object")
         return self._call_hook(hooks_base.post_pull, extras)
 
     @Pyro4.callback
     def pre_push(self, extras):
         log.debug("Called pre_push of Hooks object")
         return self._call_hook(hooks_base.pre_push, extras)
 
     @Pyro4.callback
     def post_push(self, extras):
         log.debug("Called post_push of Hooks object")
         return self._call_hook(hooks_base.post_push, extras)
 
     def _call_hook(self, hook, extras):
         extras = AttributeDict(extras)
+        netloc = urlparse.urlparse(extras.server_url).netloc
+        environ = {
+            'SERVER_NAME': netloc.split(':')[0],
+            'SERVER_PORT': ':' in netloc and netloc.split(':')[1] or '80',
+            'SCRIPT_NAME': '',
+            'PATH_INFO': '/',
+            'HTTP_HOST': 'localhost',
+            'REQUEST_METHOD': 'GET',
+        }
+        pylons_router = URLGenerator(rhodecode.CONFIG['routes.map'], environ)
+        pylons.url._push_object(pylons_router)
 
         try:
             result = hook(extras)
         except Exception as error:
             log.exception('Exception when handling hook %s', hook)
             error_args = error.args
             return {
                 'status': 128,
                 'output': '',
                 'exception': type(error).__name__,
                 'exception_args': error_args,
             }
+        finally:
+            pylons.url._pop_object()
+
         return {
             'status': result.status,
             'output': result.output,
         }
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         pass