Merge branch default into stable

docs/release-notes/release-notes-4.1.0.rst

@@ -0,0 +1,55 b''
+|RCE| 4.1.0 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2016-06-XX
+
+General
+^^^^^^^
+
+- Migrated more views to Pyramid. Those now include login, social plugins, search
+- Started Implementing Pyramid Events system in exchange to rcextensions callbacks
+- JS routes assets are now generated in development mode automatically
+- ini: Add fallback authentication plugin setting. In case only one
+  authentication backend is enabled users can now enable fallback auth if
+  they cannot log-in due to external servers being down
+- Bumped Mercurial to 3.8.3 version
+- Bumped RhodeCode Tools to 0.8.3 version
+
+New Features
+^^^^^^^^^^^^
+
+- search: add syntax highlighting, line numbers and line context to file
+  content search results
+- Go To switcher now searches commit hashes as well
+- Token based authentication is now in CE edition as well
+- User groups: added autocomplete widget to be able to select members of
+  other group as part of current group.
+
+Security
+^^^^^^^^
+
+- Added new action loggers for actions like adding/revoking permissions.
+- permissions: show origin of permissions in permissions summary. Allows users
+  to see where and how permissions are inherited
+
+Performance
+^^^^^^^^^^^
+
+
+
+Fixes
+^^^^^
+
+- api: gracefully handle errors on repos that are damaged or missing
+  from filesystem.
+- logging: log the original error when a merge failure occurs
+- #3965 Cannot change the owner of a user's group by using the API
+- database is now initialized inside pyramid context
+- fixed wrong check on LDAP plugin about missing ldap server
+- Bring back multi-threaded workers to gunicorn for backward compatibility with
+  previous RhodeCode versions
+- Commit dates are now properly handled as UTC. This fixes some issues
+  with displaying age of commits
  No newline at end of file

rhodecode/authentication/plugins/auth_headers.py

@@ -0,0 +1,225 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import colander
+import logging
+
+from sqlalchemy.ext.hybrid import hybrid_property
+
+from rhodecode.authentication.base import RhodeCodeExternalAuthPlugin
+from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
+from rhodecode.authentication.routes import AuthnPluginResourceBase
+from rhodecode.lib.colander_utils import strip_whitespace
+from rhodecode.lib.utils2 import str2bool, safe_unicode
+from rhodecode.model.db import User
+from rhodecode.translation import _
+
+
+log = logging.getLogger(__name__)
+
+
+def plugin_factory(plugin_id, *args, **kwds):
+    """
+    Factory function that is called during plugin discovery.
+    It returns the plugin instance.
+    """
+    plugin = RhodeCodeAuthPlugin(plugin_id)
+    return plugin
+
+
+class HeadersAuthnResource(AuthnPluginResourceBase):
+    pass
+
+
+class HeadersSettingsSchema(AuthnPluginSettingsSchemaBase):
+    header = colander.SchemaNode(
+        colander.String(),
+        default='REMOTE_USER',
+        description=_('Header to extract the user from'),
+        preparer=strip_whitespace,
+        title=_('Header'),
+        widget='string')
+    fallback_header = colander.SchemaNode(
+        colander.String(),
+        default='HTTP_X_FORWARDED_USER',
+        description=_('Header to extract the user from when main one fails'),
+        preparer=strip_whitespace,
+        title=_('Fallback header'),
+        widget='string')
+    clean_username = colander.SchemaNode(
+        colander.Boolean(),
+        default=True,
+        description=_('Perform cleaning of user, if passed user has @ in '
+                      'username then first part before @ is taken. '
+                      'If there\'s \\ in the username only the part after '
+                      ' \\ is taken'),
+        missing=False,
+        title=_('Clean username'),
+        widget='bool')
+
+
+class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
+
+    def includeme(self, config):
+        config.add_authn_plugin(self)
+        config.add_authn_resource(self.get_id(), HeadersAuthnResource(self))
+        config.add_view(
+            'rhodecode.authentication.views.AuthnPluginViewBase',
+            attr='settings_get',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
+            request_method='GET',
+            route_name='auth_home',
+            context=HeadersAuthnResource)
+        config.add_view(
+            'rhodecode.authentication.views.AuthnPluginViewBase',
+            attr='settings_post',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
+            request_method='POST',
+            route_name='auth_home',
+            context=HeadersAuthnResource)
+
+    def get_display_name(self):
+        return _('Headers')
+
+    def get_settings_schema(self):
+        return HeadersSettingsSchema()
+
+    @hybrid_property
+    def name(self):
+        return 'headers'
+
+    @property
+    def is_headers_auth(self):
+        return True
+
+    def use_fake_password(self):
+        return True
+
+    def user_activation_state(self):
+        def_user_perms = User.get_default_user().AuthUser.permissions['global']
+        return 'hg.extern_activate.auto' in def_user_perms
+
+    def _clean_username(self, username):
+        # Removing realm and domain from username
+        username = username.split('@')[0]
+        username = username.rsplit('\\')[-1]
+        return username
+
+    def _get_username(self, environ, settings):
+        username = None
+        environ = environ or {}
+        if not environ:
+            log.debug('got empty environ: %s' % environ)
+
+        settings = settings or {}
+        if settings.get('header'):
+            header = settings.get('header')
+            username = environ.get(header)
+            log.debug('extracted %s:%s' % (header, username))
+
+        # fallback mode
+        if not username and settings.get('fallback_header'):
+            header = settings.get('fallback_header')
+            username = environ.get(header)
+            log.debug('extracted %s:%s' % (header, username))
+
+        if username and str2bool(settings.get('clean_username')):
+            log.debug('Received username `%s` from headers' % username)
+            username = self._clean_username(username)
+            log.debug('New cleanup user is:%s' % username)
+        return username
+
+    def get_user(self, username=None, **kwargs):
+        """
+        Helper method for user fetching in plugins, by default it's using
+        simple fetch by username, but this method can be custimized in plugins
+        eg. headers auth plugin to fetch user by environ params
+        :param username: username if given to fetch
+        :param kwargs: extra arguments needed for user fetching.
+        """
+        environ = kwargs.get('environ') or {}
+        settings = kwargs.get('settings') or {}
+        username = self._get_username(environ, settings)
+        # we got the username, so use default method now
+        return super(RhodeCodeAuthPlugin, self).get_user(username)
+
+    def auth(self, userobj, username, password, settings, **kwargs):
+        """
+        Get's the headers_auth username (or email). It tries to get username
+        from REMOTE_USER if this plugin is enabled, if that fails
+        it tries to get username from HTTP_X_FORWARDED_USER if fallback header
+        is set. clean_username extracts the username from this data if it's
+        having @ in it.
+        Return None on failure. On success, return a dictionary of the form:
+
+            see: RhodeCodeAuthPluginBase.auth_func_attrs
+
+        :param userobj:
+        :param username:
+        :param password:
+        :param settings:
+        :param kwargs:
+        """
+        environ = kwargs.get('environ')
+        if not environ:
+            log.debug('Empty environ data skipping...')
+            return None
+
+        if not userobj:
+            userobj = self.get_user('', environ=environ, settings=settings)
+
+        # we don't care passed username/password for headers auth plugins.
+        # only way to log in is using environ
+        username = None
+        if userobj:
+            username = getattr(userobj, 'username')
+
+        if not username:
+            # we don't have any objects in DB user doesn't exist extract
+            # username from environ based on the settings
+            username = self._get_username(environ, settings)
+
+        # if cannot fetch username, it's a no-go for this plugin to proceed
+        if not username:
+            return None
+
+        # old attrs fetched from RhodeCode database
+        admin = getattr(userobj, 'admin', False)
+        active = getattr(userobj, 'active', True)
+        email = getattr(userobj, 'email', '')
+        firstname = getattr(userobj, 'firstname', '')
+        lastname = getattr(userobj, 'lastname', '')
+        extern_type = getattr(userobj, 'extern_type', '')
+
+        user_attrs = {
+            'username': username,
+            'firstname': safe_unicode(firstname or username),
+            'lastname': safe_unicode(lastname or ''),
+            'groups': [],
+            'email': email or '',
+            'admin': admin or False,
+            'active': active,
+            'active_from_extern': True,
+            'extern_name': username,
+            'extern_type': extern_type,
+        }
+
+        log.info('user `%s` authenticated correctly' % user_attrs['username'])
+        return user_attrs

rhodecode/authentication/plugins/auth_token.py

@@ -0,0 +1,136 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2016-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+"""
+RhodeCode authentication token plugin for built in internal auth
+"""
+
+import logging
+
+from sqlalchemy.ext.hybrid import hybrid_property
+
+from rhodecode.translation import _
+from rhodecode.authentication.base import RhodeCodeAuthPluginBase, VCS_TYPE
+from rhodecode.authentication.routes import AuthnPluginResourceBase
+from rhodecode.model.db import User, UserApiKeys
+
+
+log = logging.getLogger(__name__)
+
+
+def plugin_factory(plugin_id, *args, **kwds):
+    plugin = RhodeCodeAuthPlugin(plugin_id)
+    return plugin
+
+
+class RhodecodeAuthnResource(AuthnPluginResourceBase):
+    pass
+
+
+class RhodeCodeAuthPlugin(RhodeCodeAuthPluginBase):
+    """
+    Enables usage of authentication tokens for vcs operations.
+    """
+
+    def includeme(self, config):
+        config.add_authn_plugin(self)
+        config.add_authn_resource(self.get_id(), RhodecodeAuthnResource(self))
+        config.add_view(
+            'rhodecode.authentication.views.AuthnPluginViewBase',
+            attr='settings_get',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
+            request_method='GET',
+            route_name='auth_home',
+            context=RhodecodeAuthnResource)
+        config.add_view(
+            'rhodecode.authentication.views.AuthnPluginViewBase',
+            attr='settings_post',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
+            request_method='POST',
+            route_name='auth_home',
+            context=RhodecodeAuthnResource)
+
+    def get_display_name(self):
+        return _('Rhodecode Token Auth')
+
+    @hybrid_property
+    def name(self):
+        return "authtoken"
+
+    def user_activation_state(self):
+        def_user_perms = User.get_default_user().AuthUser.permissions['global']
+        return 'hg.register.auto_activate' in def_user_perms
+
+    def allows_authentication_from(
+            self, user, allows_non_existing_user=True,
+            allowed_auth_plugins=None, allowed_auth_sources=None):
+        """
+        Custom method for this auth that doesn't accept empty users. And also
+        allows rhodecode and authtoken extern_type to auth with this. But only
+        via vcs mode
+        """
+        # only this and rhodecode plugins can use this type
+        from rhodecode.authentication.plugins import auth_rhodecode
+        allowed_auth_plugins = [
+            self.name, auth_rhodecode.RhodeCodeAuthPlugin.name]
+        # only for vcs operations
+        allowed_auth_sources = [VCS_TYPE]
+
+        return super(RhodeCodeAuthPlugin, self).allows_authentication_from(
+            user, allows_non_existing_user=False,
+            allowed_auth_plugins=allowed_auth_plugins,
+            allowed_auth_sources=allowed_auth_sources)
+
+    def auth(self, userobj, username, password, settings, **kwargs):
+        if not userobj:
+            log.debug('userobj was:%s skipping' % (userobj, ))
+            return None
+
+        user_attrs = {
+            "username": userobj.username,
+            "firstname": userobj.firstname,
+            "lastname": userobj.lastname,
+            "groups": [],
+            "email": userobj.email,
+            "admin": userobj.admin,
+            "active": userobj.active,
+            "active_from_extern": userobj.active,
+            "extern_name": userobj.user_id,
+            "extern_type": userobj.extern_type,
+        }
+
+        log.debug('Authenticating user with args %s', user_attrs)
+        if userobj.active:
+            role = UserApiKeys.ROLE_VCS
+            active_tokens = [x.api_key for x in
+                             User.extra_valid_auth_tokens(userobj, role=role)]
+            if userobj.username == username and password in active_tokens:
+                log.info(
+                    'user `%s` successfully authenticated via %s',
+                    user_attrs['username'], self.name)
+                return user_attrs
+            log.error(
+                'user `%s` failed to authenticate via %s, reason: bad or '
+                'inactive token.', username, self.name)
+        else:
+            log.warning(
+                'user `%s` failed to authenticate via %s, reason: account not '
+                'active.', username, self.name)
+        return None

rhodecode/config/jsroutes.py

@@ -0,0 +1,42 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2016  RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+def generate_jsroutes_content(jsroutes):
+    statements = []
+    for url_name, url, fields in jsroutes:
+        statements.append(
+            "pyroutes.register('%s', '%s', %s);" % (url_name, url, fields))
+    return u'''
+/******************************************************************************
+ *                                                                            *
+ *                      DO NOT CHANGE THIS FILE MANUALLY                      *
+ *                                                                            *
+ *                                                                            *
+ *        This file is automatically generated when the app starts up.        *
+ *                                                                            *
+ *  To add a route here pass jsroute=True to the route definition in the app  *
+ *                                                                            *
+ ******************************************************************************/
+function registerRCRoutes() {
+    // routes registration
+    %s
+}
+''' % '\n    '.join(statements)
+

.bumpversion.cfg

@@ -1,6 +1,6 b''
 [bumpversion]
-current_version = 4.0.1
+current_version = 4.1.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:rhodecode/VERSION]
 

Gruntfile.js

@@ -1,137 +1,138 b''
 module.exports = function(grunt) {
   grunt.initConfig({
 
     dirs: {
       css: "rhodecode/public/css",
       js: {
         "src": "rhodecode/public/js/src",
         "dest": "rhodecode/public/js"
       }
     },
 
     concat: {
       dist: {
         src: [
           // Base libraries
           '<%= dirs.js.src %>/jquery-1.11.1.min.js',
           '<%= dirs.js.src %>/logging.js',
           '<%= dirs.js.src %>/bootstrap.js',
           '<%= dirs.js.src %>/mousetrap.js',
           '<%= dirs.js.src %>/moment.js',
           '<%= dirs.js.src %>/appenlight-client-0.4.1.min.js',
 
           // Plugins
           '<%= dirs.js.src %>/plugins/jquery.pjax.js',
           '<%= dirs.js.src %>/plugins/jquery.dataTables.js',
           '<%= dirs.js.src %>/plugins/flavoured_checkbox.js',
           '<%= dirs.js.src %>/plugins/jquery.auto-grow-input.js',
           '<%= dirs.js.src %>/plugins/jquery.autocomplete.js',
           '<%= dirs.js.src %>/plugins/jquery.debounce.js',
+          '<%= dirs.js.src %>/plugins/jquery.mark.js',
           '<%= dirs.js.src %>/plugins/jquery.timeago.js',
           '<%= dirs.js.src %>/plugins/jquery.timeago-extension.js',
 
           // Select2
           '<%= dirs.js.src %>/select2/select2.js',
-          
+
           // Code-mirror
           '<%= dirs.js.src %>/codemirror/codemirror.js',
           '<%= dirs.js.src %>/codemirror/codemirror_loadmode.js',
           '<%= dirs.js.src %>/codemirror/codemirror_hint.js',
           '<%= dirs.js.src %>/codemirror/codemirror_overlay.js',
           '<%= dirs.js.src %>/codemirror/codemirror_placeholder.js',
           // TODO: mikhail: this is an exception. Since the code mirror modes
           // are loaded "on the fly", we need to keep them in a public folder
           '<%= dirs.js.dest %>/mode/meta.js',
           '<%= dirs.js.dest %>/mode/meta_ext.js',
           '<%= dirs.js.dest %>/rhodecode/i18n/select2/translations.js',
 
           // Rhodecode utilities
           '<%= dirs.js.src %>/rhodecode/utils/array.js',
           '<%= dirs.js.src %>/rhodecode/utils/string.js',
           '<%= dirs.js.src %>/rhodecode/utils/pyroutes.js',
           '<%= dirs.js.src %>/rhodecode/utils/ajax.js',
           '<%= dirs.js.src %>/rhodecode/utils/autocomplete.js',
           '<%= dirs.js.src %>/rhodecode/utils/colorgenerator.js',
           '<%= dirs.js.src %>/rhodecode/utils/ie.js',
           '<%= dirs.js.src %>/rhodecode/utils/os.js',
 
           // Rhodecode widgets
           '<%= dirs.js.src %>/rhodecode/widgets/multiselect.js',
 
           // Rhodecode components
-          '<%= dirs.js.src %>/rhodecode/pyroutes.js',
+          '<%= dirs.js.src %>/rhodecode/init.js',
           '<%= dirs.js.src %>/rhodecode/codemirror.js',
           '<%= dirs.js.src %>/rhodecode/comments.js',
           '<%= dirs.js.src %>/rhodecode/constants.js',
           '<%= dirs.js.src %>/rhodecode/files.js',
           '<%= dirs.js.src %>/rhodecode/followers.js',
           '<%= dirs.js.src %>/rhodecode/menus.js',
           '<%= dirs.js.src %>/rhodecode/notifications.js',
           '<%= dirs.js.src %>/rhodecode/permissions.js',
           '<%= dirs.js.src %>/rhodecode/pjax.js',
           '<%= dirs.js.src %>/rhodecode/pullrequests.js',
           '<%= dirs.js.src %>/rhodecode/settings.js',
           '<%= dirs.js.src %>/rhodecode/select2_widgets.js',
           '<%= dirs.js.src %>/rhodecode/tooltips.js',
           '<%= dirs.js.src %>/rhodecode/users.js',
           '<%= dirs.js.src %>/rhodecode/appenlight.js',
 
           // Rhodecode main module
           '<%= dirs.js.src %>/rhodecode.js'
         ],
         dest: '<%= dirs.js.dest %>/scripts.js',
         nonull: true
       }
     },
 
     less: {
       development: {
         options: {
           compress: false,
           yuicompress: false,
           optimization: 0
         },
         files: {
           "<%= dirs.css %>/style.css": "<%= dirs.css %>/main.less"
         }
       },
       production: {
         options: {
           compress: true,
           yuicompress: true,
           optimization: 2
         },
         files: {
           "<%= dirs.css %>/style.css": "<%= dirs.css %>/main.less"
         }
       }
     },
 
     watch: {
       less: {
         files: ["<%= dirs.css %>/*.less"],
         tasks: ["less:production"]
       },
       js: {
         files: ["<%= dirs.js.src %>/**/*.js"],
         tasks: ["concat:dist"]
       }
     },
 
     jshint: {
       rhodecode: {
         src: '<%= dirs.js.src %>/rhodecode/**/*.js',
         options: {
           jshintrc: '.jshintrc'
         }
       }
     }
   });
 
   grunt.loadNpmTasks('grunt-contrib-less');
   grunt.loadNpmTasks('grunt-contrib-concat');
   grunt.loadNpmTasks('grunt-contrib-watch');
   grunt.loadNpmTasks('grunt-contrib-jshint');
 
   grunt.registerTask('default', ['less:production', 'concat:dist']);
 };

configs/development.ini

@@ -1,577 +1,608 b''
 ################################################################################
 ################################################################################
 # RhodeCode Enterprise - configuration file                                    #
 # Built-in functions and variables                                             #
 # The %(here)s variable will be replaced with the parent directory of this file#
 #                                                                              #
 ################################################################################
 
 [DEFAULT]
 debug = true
 pdebug = false
 ################################################################################
 ## Uncomment and replace with the email address which should receive          ##
 ## any error reports after an application crash                               ##
 ## Additionally these settings will be used by the RhodeCode mailing system   ##
 ################################################################################
 #email_to = admin@localhost
 #error_email_from = paste_error@localhost
 #app_email_from = rhodecode-noreply@localhost
 #error_message =
 #email_prefix = [RhodeCode]
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 ## Specify available auth parameters here (e.g. LOGIN PLAIN CRAM-MD5, etc.)
 #smtp_auth =
 
 [server:main]
 ## COMMON ##
 host = 127.0.0.1
 port = 5000
 
-##########################
-## WAITRESS WSGI SERVER ##
-##########################
+##################################
+##     WAITRESS WSGI SERVER     ##
+## Recommended for Development  ##
+##################################
 use = egg:waitress#main
 ## number of worker threads
 threads = 5
 ## MAX BODY SIZE 100GB
 max_request_body_size = 107374182400
 ## Use poll instead of select, fixes file descriptors limits problems.
 ## May not work on old windows systems.
 asyncore_use_poll = true
 
 
 ##########################
 ## GUNICORN WSGI SERVER ##
 ##########################
 ## run with gunicorn --log-config <inifile.ini> --paste <inifile.ini>
 #use = egg:gunicorn#main
 ## Sets the number of process workers. You must set `instance_id = *`
 ## when this option is set to more than one worker, recommended
 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
 ## The `instance_id = *` must be set in the [app:main] section below
-#workers = 1
+#workers = 2
 ## number of threads for each of the worker, must be set to 1 for gevent
 ## generally recommened to be at 1
 #threads = 1
 ## process name
 #proc_name = rhodecode
 ## type of worker class, one of sync, gevent
 ## recommended for bigger setup is using of of other than sync one
 #worker_class = sync
 ## The maximum number of simultaneous clients. Valid only for Gevent
 #worker_connections = 10
 ## max number of requests that worker will handle before being gracefully
 ## restarted, could prevent memory leaks
 #max_requests = 1000
 #max_requests_jitter = 30
-## ammount of time a worker can spend with handling a request before it
+## amount of time a worker can spend with handling a request before it
 ## gets killed and restarted. Set to 6hrs
 #timeout = 21600
 
 
 ## prefix middleware for RhodeCode, disables force_https flag.
 ## allows to set RhodeCode under a prefix in server.
 ## eg https://server.com/<prefix>. Enable `filter-with =` option below as well.
 #[filter:proxy-prefix]
 #use = egg:PasteDeploy#prefix
 #prefix = /<your-prefix>
 
 [app:main]
 use = egg:rhodecode-enterprise-ce
 ## enable proxy prefix middleware, defined below
 #filter-with = proxy-prefix
 
 # During development the we want to have the debug toolbar enabled
 pyramid.includes =
     pyramid_debugtoolbar
     rhodecode.utils.debugtoolbar
     rhodecode.lib.middleware.request_wrapper
 
 pyramid.reload_templates = true
 
 debugtoolbar.hosts = 0.0.0.0/0
 debugtoolbar.exclude_prefixes =
     /css
     /fonts
     /images
     /js
 
 ## RHODECODE PLUGINS ##
 rhodecode.includes =
     rhodecode.api
 
 
 # api prefix url
 rhodecode.api.url = /_admin/api
 
 
 ## END RHODECODE PLUGINS ##
 
 full_stack = true
 
 ## Serve static files via RhodeCode, disable to serve them via HTTP server
 static_files = true
 
 ## Optional Languages
 ## en(default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ## perform a full repository scan on each server start, this should be
 ## set to false after first startup, to allow faster server restarts.
 startup.import_repos = false
 
 ## Uncomment and set this path to use archive download cache.
 ## Once enabled, generated archives will be cached at this location
 ## and served from the cache during subsequent requests for the same archive of
 ## the repository.
 #archive_cache_dir = /tmp/tarballcache
 
 ## change this to unique ID for security
 app_instance_uuid = rc-production
 
 ## cut off limit for large diffs (size in bytes)
 cut_off_limit_diff = 1024000
 cut_off_limit_file = 256000
 
 ## use cache version of scm repo everywhere
 vcs_full_cache = true
 
 ## force https in RhodeCode, fixes https redirects, assumes it's always https
 ## Normally this is controlled by proper http flags sent from http server
 force_https = false
 
 ## use Strict-Transport-Security headers
 use_htsts = false
 
 ## number of commits stats will parse on each iteration
 commit_parse_limit = 25
 
 ## git rev filter option, --all is the default filter, if you need to
 ## hide all refs in changelog switch this to --branches --tags
 git_rev_filter = --branches --tags
 
 # Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ## RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ## gist URL alias, used to create nicer urls for gist. This should be an
 ## url that does rewrites to _admin/gists/<gistid>.
 ## example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ## RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/<gistid>
 gist_alias_url =
 
 ## List of controllers (using glob pattern syntax) that AUTH TOKENS could be
 ## used for access.
 ## Adding ?auth_token = <token> to the url authenticates this request as if it
 ## came from the the logged in user who own this authentication token.
 ##
 ## Syntax is <ControllerClass>:<function_pattern>.
 ## To enable access to raw_files put `FilesController:raw`.
 ## To enable access to patches add `ChangesetController:changeset_patch`.
 ## The list should be "," separated and on a single line.
 ##
 ## Recommended controllers to enable:
 #    ChangesetController:changeset_patch,
 #    ChangesetController:changeset_raw,
 #    FilesController:raw,
 #    FilesController:archivefile,
 #    GistsController:*,
 api_access_controllers_whitelist =
 
 ## default encoding used to convert from and to unicode
 ## can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ## instance-id prefix
 ## a prefix key for this instance used for cache invalidation when running
 ## multiple instances of rhodecode, make sure it's globally unique for
 ## all running rhodecode instances. Leave empty if you don't use it
 instance_id =
 
+## Fallback authentication plugin. Set this to a plugin ID to force the usage
+## of an authentication plugin also if it is disabled by it's settings.
+## This could be useful if you are unable to log in to the system due to broken
+## authentication settings. Then you can enable e.g. the internal rhodecode auth
+## module to log in again and fix the settings.
+##
+## Available builtin plugin IDs (hash is part of the ID):
+## egg:rhodecode-enterprise-ce#rhodecode
+## egg:rhodecode-enterprise-ce#pam
+## egg:rhodecode-enterprise-ce#ldap
+## egg:rhodecode-enterprise-ce#jasig_cas
+## egg:rhodecode-enterprise-ce#headers
+## egg:rhodecode-enterprise-ce#crowd
+#rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
+
 ## alternative return HTTP header for failed authentication. Default HTTP
 ## response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ## handling that causing a series of failed authentication calls.
 ## Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ## This will be served instead of default 401 on bad authnetication
 auth_ret_code =
 
 ## use special detection method when serving auth_ret_code, instead of serving
 ## ret_code directly, use 401 initially (Which triggers credentials prompt)
 ## and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ## locking return code. When repository is locked return this HTTP code. 2XX
 ## codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ## allows to change the repository location in settings page
 allow_repo_location_change = true
 
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ## generated license token, goto license page in RhodeCode settings to obtain
 ## new token
 license_token =
 
 ## supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 ## supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = dev
 
 ## Display extended labs settings
 labs_settings_active = true
 
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 broker.host = localhost
 broker.vhost = rabbitmqhost
 broker.port = 5672
 broker.user = rabbitmq
 broker.password = qweqwe
 
 celery.imports = rhodecode.lib.celerylib.tasks
 
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 
 celeryd.concurrency = 2
 #celeryd.log.file = celeryd.log
 celeryd.log.level = debug
 celeryd.max.tasks.per.child = 1
 
 ## tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 # default cache dir for templates.  Putting this into a ramdisk
 ## can boost performance, eg. %(here)s/data_ramdisk
 cache_dir = %(here)s/data
 
 ## locking and default file storage for Beaker. Putting this into a ramdisk
 ## can boost performance, eg. %(here)s/data_ramdisk/cache/beaker_data
 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
 
 beaker.cache.regions = super_short_term, short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
 
 beaker.cache.super_short_term.type = memory
 beaker.cache.super_short_term.expire = 10
 beaker.cache.super_short_term.key_length = 256
 
 beaker.cache.short_term.type = memory
 beaker.cache.short_term.expire = 60
 beaker.cache.short_term.key_length = 256
 
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 
 beaker.cache.sql_cache_short.type = memory
 beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.key_length = 256
 
 # default is memory cache, configure only if required
 # using multi-node or multi-worker setup
 #beaker.cache.auth_plugins.type = ext:database
 #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
 #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.cache.auth_plugins.sa.pool_recycle = 3600
 #beaker.cache.auth_plugins.sa.pool_size = 10
 #beaker.cache.auth_plugins.sa.max_overflow = 0
 
 beaker.cache.repo_cache_long.type = memorylru_base
 beaker.cache.repo_cache_long.max_items = 4096
 beaker.cache.repo_cache_long.expire = 2592000
 
 # default is memorylru_base cache, configure only if required
 # using multi-node or multi-worker setup
 #beaker.cache.repo_cache_long.type = ext:memcached
 #beaker.cache.repo_cache_long.url = localhost:11211
 #beaker.cache.repo_cache_long.expire = 1209600
 #beaker.cache.repo_cache_long.key_length = 256
 
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 
 ## .session.type is type of storage options for the session, current allowed
-## types are file, ext:memcached, ext:database, and memory(default).
+## types are file, ext:memcached, ext:database, and memory (default).
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/data/sessions/data
 
 ## db based session, fast, and allows easy management over logged in users ##
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = develop-rc-uytcxaz
 beaker.session.lock_dir = %(here)s/data/sessions/lock
 
 ## Secure encrypted cookie. Requires AES and AES python libraries
 ## you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = <key_for_encryption>
 #beaker.session.validate_key = <validation_key>
 
 ## sets session as invalid(also logging out user) if it haven not been
 ## accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 #beaker.session.cookie_path = /<your-prefix>
 
 ## uncomment for https secure cookie
 beaker.session.secure = false
 
 ## auto save the session to not to use .save()
 beaker.session.auto = false
 
 ## default cookie expiration time in seconds, set to `true` to set expire
 ## at browser close
 #beaker.session.cookie_expires = 3600
 
 ###################################
 ## SEARCH INDEXING CONFIGURATION ##
 ###################################
+## Full text search indexer is available in rhodecode-tools under
+## `rhodecode-tools index` command
 
+# WHOOSH Backend, doesn't require additional services to run
+# it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
+
 ###################################
-## ERROR AND LOG HANDLING SYSTEM ##
+##       APPENLIGHT CONFIG       ##
 ###################################
 
 ## Appenlight is tailored to work with RhodeCode, see
 ## http://appenlight.com for details how to obtain an account
 
 ## appenlight integration enabled
 appenlight = false
 
 appenlight.server_url = https://api.appenlight.com
 appenlight.api_key = YOUR_API_KEY
-;appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
+#appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
 
 # used for JS client
 appenlight.api_public_key = YOUR_API_PUBLIC_KEY
 
 ## TWEAK AMOUNT OF INFO SENT HERE
 
 ## enables 404 error logging (default False)
 appenlight.report_404 = false
 
 ## time in seconds after request is considered being slow (default 1)
 appenlight.slow_request_time = 1
 
 ## record slow requests in application
 ## (needs to be enabled for slow datastore recording and time tracking)
 appenlight.slow_requests = true
 
 ## enable hooking to application loggers
 appenlight.logging = true
 
 ## minimum log level for log capture
 appenlight.logging.level = WARNING
 
 ## send logs only from erroneous/slow requests
 ## (saves API quota for intensive logging)
 appenlight.logging_on_error = false
 
 ## list of additonal keywords that should be grabbed from environ object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always send following info:
 ## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
 ## start with HTTP* this list be extended with additional keywords here
 appenlight.environ_keys_whitelist =
 
 ## list of keywords that should be blanked from request object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always blank keys that contain following words
 ## 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
 ## this list be extended with additional keywords set here
 appenlight.request_keys_blacklist =
 
 ## list of namespaces that should be ignores when gathering log entries
 ## can be string with comma separated list of namespaces
 ## (by default the client ignores own entries: appenlight_client.client)
 appenlight.log_namespace_blacklist =
 
 
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 #set debug = false
 
 
 ##############
 ## STYLING  ##
 ##############
 debug_style = true
 
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode
 
 # see sqlalchemy docs for other advanced settings
 
 ## print the sql statements to output
 sqlalchemy.db1.echo = false
 ## recycle the connections after this ammount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.db1.convert_unicode = true
 
 ## the number of connections to keep open inside the connection pool.
 ## 0 indicates no limit
 #sqlalchemy.db1.pool_size = 5
 
 ## the number of connections to allow in connection pool "overflow", that is
 ## connections that can be opened above and beyond the pool_size setting,
 ## which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 
 ##################
 ### VCS CONFIG ###
 ##################
 vcs.server.enable = true
 vcs.server = localhost:9900
-# Available protocols: pyro4, http
-vcs.server.protocol = pyro4
 
-# available impl:
-# vcsserver.scm_app (EE only, for testing),
-# rhodecode.lib.middleware.utils.scm_app_http
-# pyro4
+## Web server connectivity protocol, responsible for web based VCS operatations
+## Available protocols are:
+## `pyro4` - using pyro4 server
+## `http` - using http-rpc backend
+#vcs.server.protocol = http
+
+## Push/Pull operations protocol, available options are:
+## `pyro4` - using pyro4 server
+## `rhodecode.lib.middleware.utils.scm_app_http` - Http based, recommended
+## `vcsserver.scm_app` - internal app (EE only)
 #vcs.scm_app_implementation = rhodecode.lib.middleware.utils.scm_app_http
 
+## Push/Pull operations hooks protocol, available options are:
+## `pyro4` - using pyro4 server
+## `http` - using http-rpc backend
+#vcs.hooks.protocol = http
+
 vcs.server.log_level = debug
+## Start VCSServer with this instance as a subprocess, usefull for development
 vcs.start_server = true
 vcs.backends = hg, git, svn
 vcs.connection_timeout = 3600
 ## Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ## Available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible
 #vcs.svn.compatible_version = pre-1.8-compatible
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, rhodecode, sqlalchemy, beaker, pyro4, templates, whoosh_indexer
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_routes]
 level = DEBUG
 handlers =
 qualname = routes.middleware
 ## "level = DEBUG" logs the route matched and routing variables.
 propagate = 1
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_pyro4]
 level = DEBUG
 handlers =
 qualname = Pyro4
 propagate = 1
 
 [logger_templates]
 level = INFO
 handlers =
 qualname = pylons.templating
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_whoosh_indexer]
 level = DEBUG
 handlers =
 qualname = whoosh_indexer
 propagate = 1
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = DEBUG
 formatter = color_formatter
 
 [handler_console_sql]
 class = StreamHandler
 args = (sys.stderr,)
 level = DEBUG
 formatter = color_formatter_sql
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.Pyro4AwareFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

configs/production.ini

@@ -1,551 +1,577 b''
 ################################################################################
 ################################################################################
 # RhodeCode Enterprise - configuration file                                    #
 # Built-in functions and variables                                             #
 # The %(here)s variable will be replaced with the parent directory of this file#
 #                                                                              #
 ################################################################################
 
 [DEFAULT]
 debug = true
 pdebug = false
 ################################################################################
 ## Uncomment and replace with the email address which should receive          ##
 ## any error reports after an application crash                               ##
 ## Additionally these settings will be used by the RhodeCode mailing system   ##
 ################################################################################
 #email_to = admin@localhost
 #error_email_from = paste_error@localhost
 #app_email_from = rhodecode-noreply@localhost
 #error_message =
 #email_prefix = [RhodeCode]
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 ## Specify available auth parameters here (e.g. LOGIN PLAIN CRAM-MD5, etc.)
 #smtp_auth =
 
 [server:main]
 ## COMMON ##
 host = 127.0.0.1
 port = 5000
 
-##########################
-## WAITRESS WSGI SERVER ##
-##########################
-use = egg:waitress#main
+##################################
+##     WAITRESS WSGI SERVER     ##
+## Recommended for Development  ##
+##################################
+#use = egg:waitress#main
 ## number of worker threads
-threads = 5
+#threads = 5
 ## MAX BODY SIZE 100GB
-max_request_body_size = 107374182400
+#max_request_body_size = 107374182400
 ## Use poll instead of select, fixes file descriptors limits problems.
 ## May not work on old windows systems.
-asyncore_use_poll = true
+#asyncore_use_poll = true
 
 
 ##########################
 ## GUNICORN WSGI SERVER ##
 ##########################
 ## run with gunicorn --log-config <inifile.ini> --paste <inifile.ini>
-#use = egg:gunicorn#main
+use = egg:gunicorn#main
 ## Sets the number of process workers. You must set `instance_id = *`
 ## when this option is set to more than one worker, recommended
 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
 ## The `instance_id = *` must be set in the [app:main] section below
-#workers = 1
+workers = 2
 ## number of threads for each of the worker, must be set to 1 for gevent
 ## generally recommened to be at 1
 #threads = 1
 ## process name
-#proc_name = rhodecode
+proc_name = rhodecode
 ## type of worker class, one of sync, gevent
 ## recommended for bigger setup is using of of other than sync one
-#worker_class = sync
+worker_class = sync
 ## The maximum number of simultaneous clients. Valid only for Gevent
 #worker_connections = 10
 ## max number of requests that worker will handle before being gracefully
 ## restarted, could prevent memory leaks
-#max_requests = 1000
-#max_requests_jitter = 30
-## ammount of time a worker can spend with handling a request before it
+max_requests = 1000
+max_requests_jitter = 30
+## amount of time a worker can spend with handling a request before it
 ## gets killed and restarted. Set to 6hrs
-#timeout = 21600
+timeout = 21600
 
 
 ## prefix middleware for RhodeCode, disables force_https flag.
 ## allows to set RhodeCode under a prefix in server.
 ## eg https://server.com/<prefix>. Enable `filter-with =` option below as well.
 #[filter:proxy-prefix]
 #use = egg:PasteDeploy#prefix
 #prefix = /<your-prefix>
 
 [app:main]
 use = egg:rhodecode-enterprise-ce
 ## enable proxy prefix middleware, defined below
 #filter-with = proxy-prefix
 
 full_stack = true
 
 ## Serve static files via RhodeCode, disable to serve them via HTTP server
 static_files = true
 
 ## Optional Languages
 ## en(default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ## perform a full repository scan on each server start, this should be
 ## set to false after first startup, to allow faster server restarts.
 startup.import_repos = false
 
 ## Uncomment and set this path to use archive download cache.
 ## Once enabled, generated archives will be cached at this location
 ## and served from the cache during subsequent requests for the same archive of
 ## the repository.
 #archive_cache_dir = /tmp/tarballcache
 
 ## change this to unique ID for security
 app_instance_uuid = rc-production
 
 ## cut off limit for large diffs (size in bytes)
 cut_off_limit_diff = 1024000
 cut_off_limit_file = 256000
 
 ## use cache version of scm repo everywhere
 vcs_full_cache = true
 
 ## force https in RhodeCode, fixes https redirects, assumes it's always https
 ## Normally this is controlled by proper http flags sent from http server
 force_https = false
 
 ## use Strict-Transport-Security headers
 use_htsts = false
 
 ## number of commits stats will parse on each iteration
 commit_parse_limit = 25
 
 ## git rev filter option, --all is the default filter, if you need to
 ## hide all refs in changelog switch this to --branches --tags
 git_rev_filter = --branches --tags
 
 # Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ## RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ## gist URL alias, used to create nicer urls for gist. This should be an
 ## url that does rewrites to _admin/gists/<gistid>.
 ## example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ## RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/<gistid>
 gist_alias_url =
 
 ## List of controllers (using glob pattern syntax) that AUTH TOKENS could be
 ## used for access.
 ## Adding ?auth_token = <token> to the url authenticates this request as if it
 ## came from the the logged in user who own this authentication token.
 ##
 ## Syntax is <ControllerClass>:<function_pattern>.
 ## To enable access to raw_files put `FilesController:raw`.
 ## To enable access to patches add `ChangesetController:changeset_patch`.
 ## The list should be "," separated and on a single line.
 ##
 ## Recommended controllers to enable:
 #    ChangesetController:changeset_patch,
 #    ChangesetController:changeset_raw,
 #    FilesController:raw,
 #    FilesController:archivefile,
 #    GistsController:*,
 api_access_controllers_whitelist =
 
 ## default encoding used to convert from and to unicode
 ## can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ## instance-id prefix
 ## a prefix key for this instance used for cache invalidation when running
 ## multiple instances of rhodecode, make sure it's globally unique for
 ## all running rhodecode instances. Leave empty if you don't use it
 instance_id =
 
+## Fallback authentication plugin. Set this to a plugin ID to force the usage
+## of an authentication plugin also if it is disabled by it's settings.
+## This could be useful if you are unable to log in to the system due to broken
+## authentication settings. Then you can enable e.g. the internal rhodecode auth
+## module to log in again and fix the settings.
+##
+## Available builtin plugin IDs (hash is part of the ID):
+## egg:rhodecode-enterprise-ce#rhodecode
+## egg:rhodecode-enterprise-ce#pam
+## egg:rhodecode-enterprise-ce#ldap
+## egg:rhodecode-enterprise-ce#jasig_cas
+## egg:rhodecode-enterprise-ce#headers
+## egg:rhodecode-enterprise-ce#crowd
+#rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
+
 ## alternative return HTTP header for failed authentication. Default HTTP
 ## response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ## handling that causing a series of failed authentication calls.
 ## Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ## This will be served instead of default 401 on bad authnetication
 auth_ret_code =
 
 ## use special detection method when serving auth_ret_code, instead of serving
 ## ret_code directly, use 401 initially (Which triggers credentials prompt)
 ## and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ## locking return code. When repository is locked return this HTTP code. 2XX
 ## codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ## allows to change the repository location in settings page
 allow_repo_location_change = true
 
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ## generated license token, goto license page in RhodeCode settings to obtain
 ## new token
 license_token =
 
 ## supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 ## supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = prod
 
 ## Display extended labs settings
 labs_settings_active = true
 
 ####################################
 ###        CELERY CONFIG        ####
 ####################################
 use_celery = false
 broker.host = localhost
 broker.vhost = rabbitmqhost
 broker.port = 5672
 broker.user = rabbitmq
 broker.password = qweqwe
 
 celery.imports = rhodecode.lib.celerylib.tasks
 
 celery.result.backend = amqp
 celery.result.dburi = amqp://
 celery.result.serialier = json
 
 #celery.send.task.error.emails = true
 #celery.amqp.task.result.expires = 18000
 
 celeryd.concurrency = 2
 #celeryd.log.file = celeryd.log
 celeryd.log.level = debug
 celeryd.max.tasks.per.child = 1
 
 ## tasks will never be sent to the queue, but executed locally instead.
 celery.always.eager = false
 
 ####################################
 ###         BEAKER CACHE        ####
 ####################################
 # default cache dir for templates.  Putting this into a ramdisk
 ## can boost performance, eg. %(here)s/data_ramdisk
 cache_dir = %(here)s/data
 
 ## locking and default file storage for Beaker. Putting this into a ramdisk
 ## can boost performance, eg. %(here)s/data_ramdisk/cache/beaker_data
 beaker.cache.data_dir = %(here)s/data/cache/beaker_data
 beaker.cache.lock_dir = %(here)s/data/cache/beaker_lock
 
 beaker.cache.regions = super_short_term, short_term, long_term, sql_cache_short, auth_plugins, repo_cache_long
 
 beaker.cache.super_short_term.type = memory
 beaker.cache.super_short_term.expire = 10
 beaker.cache.super_short_term.key_length = 256
 
 beaker.cache.short_term.type = memory
 beaker.cache.short_term.expire = 60
 beaker.cache.short_term.key_length = 256
 
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 
 beaker.cache.sql_cache_short.type = memory
 beaker.cache.sql_cache_short.expire = 10
 beaker.cache.sql_cache_short.key_length = 256
 
 # default is memory cache, configure only if required
 # using multi-node or multi-worker setup
 #beaker.cache.auth_plugins.type = ext:database
 #beaker.cache.auth_plugins.lock_dir = %(here)s/data/cache/auth_plugin_lock
 #beaker.cache.auth_plugins.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.cache.auth_plugins.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.cache.auth_plugins.sa.pool_recycle = 3600
 #beaker.cache.auth_plugins.sa.pool_size = 10
 #beaker.cache.auth_plugins.sa.max_overflow = 0
 
 beaker.cache.repo_cache_long.type = memorylru_base
 beaker.cache.repo_cache_long.max_items = 4096
 beaker.cache.repo_cache_long.expire = 2592000
 
 # default is memorylru_base cache, configure only if required
 # using multi-node or multi-worker setup
 #beaker.cache.repo_cache_long.type = ext:memcached
 #beaker.cache.repo_cache_long.url = localhost:11211
 #beaker.cache.repo_cache_long.expire = 1209600
 #beaker.cache.repo_cache_long.key_length = 256
 
 ####################################
 ###       BEAKER SESSION        ####
 ####################################
 
 ## .session.type is type of storage options for the session, current allowed
-## types are file, ext:memcached, ext:database, and memory(default).
+## types are file, ext:memcached, ext:database, and memory (default).
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/data/sessions/data
 
 ## db based session, fast, and allows easy management over logged in users ##
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = production-rc-uytcxaz
-#beaker.session.lock_dir = %(here)s/data/sessions/lock
+beaker.session.lock_dir = %(here)s/data/sessions/lock
 
 ## Secure encrypted cookie. Requires AES and AES python libraries
 ## you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = <key_for_encryption>
 #beaker.session.validate_key = <validation_key>
 
 ## sets session as invalid(also logging out user) if it haven not been
 ## accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 #beaker.session.cookie_path = /<your-prefix>
 
 ## uncomment for https secure cookie
 beaker.session.secure = false
 
 ## auto save the session to not to use .save()
 beaker.session.auto = false
 
 ## default cookie expiration time in seconds, set to `true` to set expire
 ## at browser close
 #beaker.session.cookie_expires = 3600
 
 ###################################
 ## SEARCH INDEXING CONFIGURATION ##
 ###################################
+## Full text search indexer is available in rhodecode-tools under
+## `rhodecode-tools index` command
 
+# WHOOSH Backend, doesn't require additional services to run
+# it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
+
 ###################################
-## ERROR AND LOG HANDLING SYSTEM ##
+##       APPENLIGHT CONFIG       ##
 ###################################
 
 ## Appenlight is tailored to work with RhodeCode, see
 ## http://appenlight.com for details how to obtain an account
 
 ## appenlight integration enabled
 appenlight = false
 
 appenlight.server_url = https://api.appenlight.com
 appenlight.api_key = YOUR_API_KEY
-;appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
+#appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
 
 # used for JS client
 appenlight.api_public_key = YOUR_API_PUBLIC_KEY
 
 ## TWEAK AMOUNT OF INFO SENT HERE
 
 ## enables 404 error logging (default False)
 appenlight.report_404 = false
 
 ## time in seconds after request is considered being slow (default 1)
 appenlight.slow_request_time = 1
 
 ## record slow requests in application
 ## (needs to be enabled for slow datastore recording and time tracking)
 appenlight.slow_requests = true
 
 ## enable hooking to application loggers
 appenlight.logging = true
 
 ## minimum log level for log capture
 appenlight.logging.level = WARNING
 
 ## send logs only from erroneous/slow requests
 ## (saves API quota for intensive logging)
 appenlight.logging_on_error = false
 
 ## list of additonal keywords that should be grabbed from environ object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always send following info:
 ## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
 ## start with HTTP* this list be extended with additional keywords here
 appenlight.environ_keys_whitelist =
 
 ## list of keywords that should be blanked from request object
 ## can be string with comma separated list of words in lowercase
 ## (by default client will always blank keys that contain following words
 ## 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
 ## this list be extended with additional keywords set here
 appenlight.request_keys_blacklist =
 
 ## list of namespaces that should be ignores when gathering log entries
 ## can be string with comma separated list of namespaces
 ## (by default the client ignores own entries: appenlight_client.client)
 appenlight.log_namespace_blacklist =
 
 
 ################################################################################
 ## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 ## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 ## execute malicious code after an exception is raised.                       ##
 ################################################################################
 set debug = false
 
 
-##############
-## STYLING  ##
-##############
-debug_style = false
-
 #########################################################
 ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 #########################################################
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode
 
 # see sqlalchemy docs for other advanced settings
 
 ## print the sql statements to output
 sqlalchemy.db1.echo = false
 ## recycle the connections after this ammount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.db1.convert_unicode = true
 
 ## the number of connections to keep open inside the connection pool.
 ## 0 indicates no limit
 #sqlalchemy.db1.pool_size = 5
 
 ## the number of connections to allow in connection pool "overflow", that is
 ## connections that can be opened above and beyond the pool_size setting,
 ## which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 
 ##################
 ### VCS CONFIG ###
 ##################
 vcs.server.enable = true
 vcs.server = localhost:9900
-# Available protocols: pyro4, http
-vcs.server.protocol = pyro4
 
-# available impl:
-# vcsserver.scm_app (EE only, for testing),
-# rhodecode.lib.middleware.utils.scm_app_http
-# pyro4
+## Web server connectivity protocol, responsible for web based VCS operatations
+## Available protocols are:
+## `pyro4` - using pyro4 server
+## `http` - using http-rpc backend
+#vcs.server.protocol = http
+
+## Push/Pull operations protocol, available options are:
+## `pyro4` - using pyro4 server
+## `rhodecode.lib.middleware.utils.scm_app_http` - Http based, recommended
+## `vcsserver.scm_app` - internal app (EE only)
 #vcs.scm_app_implementation = rhodecode.lib.middleware.utils.scm_app_http
 
+## Push/Pull operations hooks protocol, available options are:
+## `pyro4` - using pyro4 server
+## `http` - using http-rpc backend
+#vcs.hooks.protocol = http
+
 vcs.server.log_level = info
+## Start VCSServer with this instance as a subprocess, usefull for development
 vcs.start_server = false
 vcs.backends = hg, git, svn
 vcs.connection_timeout = 3600
 ## Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ## Available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible
 #vcs.svn.compatible_version = pre-1.8-compatible
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, routes, rhodecode, sqlalchemy, beaker, pyro4, templates, whoosh_indexer
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_routes]
 level = DEBUG
 handlers =
 qualname = routes.middleware
 ## "level = DEBUG" logs the route matched and routing variables.
 propagate = 1
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_pyro4]
 level = DEBUG
 handlers =
 qualname = Pyro4
 propagate = 1
 
 [logger_templates]
 level = INFO
 handlers =
 qualname = pylons.templating
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_whoosh_indexer]
 level = DEBUG
 handlers =
 qualname = whoosh_indexer
 propagate = 1
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = INFO
 formatter = generic
 
 [handler_console_sql]
 class = StreamHandler
 args = (sys.stderr,)
 level = WARN
 formatter = generic
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.Pyro4AwareFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

default.nix

@@ -1,215 +1,219 b''
 # Nix environment for the community edition
 #
 # This shall be as lean as possible, just producing the Enterprise
 # derivation. For advanced tweaks to pimp up the development environment we use
 # "shell.nix" so that it does not have to clutter this file.
 
 { pkgs ? (import <nixpkgs> {})
 , pythonPackages ? "python27Packages"
 , pythonExternalOverrides ? self: super: {}
 , doCheck ? true
 }:
 
 let pkgs_ = pkgs; in
 
 let
   pkgs = pkgs_.overridePackages (self: super: {
     # Override subversion derivation to
     #  - activate python bindings
     #  - set version to 1.8
     subversion = super.subversion18.override {
        httpSupport = true;
        pythonBindings = true;
        python = self.python27Packages.python;
     };
   });
 
   inherit (pkgs.lib) fix extends;
 
   basePythonPackages = with builtins; if isAttrs pythonPackages
     then pythonPackages
     else getAttr pythonPackages pkgs;
 
   elem = builtins.elem;
   basename = path: with pkgs.lib; last (splitString "/" path);
   startsWith = prefix: full: let
     actualPrefix = builtins.substring 0 (builtins.stringLength prefix) full;
   in actualPrefix == prefix;
 
   src-filter = path: type: with pkgs.lib;
     let
       ext = last (splitString "." path);
     in
       !elem (basename path) [
         ".git" ".hg" "__pycache__" ".eggs" "node_modules"
         "build" "data" "tmp"] &&
       !elem ext ["egg-info" "pyc"] &&
       !startsWith "result" path;
 
   rhodecode-enterprise-ce-src = builtins.filterSource src-filter ./.;
 
   # Load the generated node packages
   nodePackages = pkgs.callPackage "${pkgs.path}/pkgs/top-level/node-packages.nix" rec {
     self = nodePackages;
     generated = pkgs.callPackage ./pkgs/node-packages.nix { inherit self; };
   };
 
   # TODO: Should be taken automatically out of the generates packages.
   # apps.nix has one solution for this, although I'd prefer to have the deps
   # from package.json mapped in here.
   nodeDependencies = with nodePackages; [
     grunt
     grunt-contrib-concat
     grunt-contrib-jshint
     grunt-contrib-less
     grunt-contrib-watch
     jshint
   ];
 
   pythonGeneratedPackages = self: basePythonPackages.override (a: {
     inherit self;
   })
   // (scopedImport {
     self = self;
     super = basePythonPackages;
     inherit pkgs;
     inherit (pkgs) fetchurl fetchgit;
   } ./pkgs/python-packages.nix);
 
   pythonOverrides = import ./pkgs/python-packages-overrides.nix {
     inherit
       basePythonPackages
       pkgs;
   };
 
   pythonLocalOverrides = self: super: {
     rhodecode-enterprise-ce =
       let
-        version = "${builtins.readFile ./rhodecode/VERSION}";
+        version = builtins.readFile ./rhodecode/VERSION;
         linkNodeModules = ''
           echo "Link node packages"
           # TODO: check if this adds stuff as a dependency, closure size
           rm -fr node_modules
           mkdir -p node_modules
           ${pkgs.lib.concatMapStrings (dep: ''
             ln -sfv ${dep}/lib/node_modules/${dep.pkgName} node_modules/
           '') nodeDependencies}
           echo "DONE: Link node packages"
         '';
       in super.rhodecode-enterprise-ce.override (attrs: {
 
       inherit doCheck;
       name = "rhodecode-enterprise-ce-${version}";
       version = version;
       src = rhodecode-enterprise-ce-src;
 
       buildInputs =
         attrs.buildInputs ++
         (with self; [
           pkgs.nodePackages.grunt-cli
           pkgs.subversion
           pytest-catchlog
           rc_testdata
         ]);
 
       propagatedBuildInputs = attrs.propagatedBuildInputs ++ (with self; [
         rhodecode-tools
       ]);
 
       # TODO: johbo: Make a nicer way to expose the parts. Maybe
       # pkgs/default.nix?
       passthru = {
-        inherit myPythonPackagesUnfix;
+        inherit
+          pythonLocalOverrides
+          myPythonPackagesUnfix;
         pythonPackages = self;
       };
 
       LC_ALL = "en_US.UTF-8";
       LOCALE_ARCHIVE =
         if pkgs.stdenv ? glibc
         then "${pkgs.glibcLocales}/lib/locale/locale-archive"
         else "";
 
       # Somewhat snappier setup of the development environment
       # TODO: move into shell.nix
       # TODO: think of supporting a stable path again, so that multiple shells
       #       can share it.
       shellHook = ''
         tmp_path=$(mktemp -d)
         export PATH="$tmp_path/bin:$PATH"
         export PYTHONPATH="$tmp_path/${self.python.sitePackages}:$PYTHONPATH"
         mkdir -p $tmp_path/${self.python.sitePackages}
         python setup.py develop --prefix $tmp_path --allow-hosts ""
       '' + linkNodeModules;
 
       preCheck = ''
         export PATH="$out/bin:$PATH"
       '';
 
       postCheck = ''
         rm -rf $out/lib/${self.python.libPrefix}/site-packages/pytest_pylons
         rm -rf $out/lib/${self.python.libPrefix}/site-packages/rhodecode/tests
       '';
 
       preBuild = linkNodeModules + ''
         grunt
         rm -fr node_modules
       '';
 
       postInstall = ''
         # python based programs need to be wrapped
         ln -s ${self.supervisor}/bin/supervisor* $out/bin/
         ln -s ${self.gunicorn}/bin/gunicorn $out/bin/
         ln -s ${self.PasteScript}/bin/paster $out/bin/
+        ln -s ${self.pyramid}/bin/* $out/bin/  #*/
 
         # rhodecode-tools
         # TODO: johbo: re-think this. Do the tools import anything from enterprise?
         ln -s ${self.rhodecode-tools}/bin/rhodecode-* $out/bin/
 
         # note that condition should be restricted when adding further tools
         for file in $out/bin/*; do  #*/
           wrapProgram $file \
               --prefix PYTHONPATH : $PYTHONPATH \
+              --prefix PATH : $PATH \
               --set PYTHONHASHSEED random
         done
 
         mkdir $out/etc
         cp configs/production.ini $out/etc
 
         echo "Writing meta information for rccontrol to nix-support/rccontrol"
         mkdir -p $out/nix-support/rccontrol
         cp -v rhodecode/VERSION $out/nix-support/rccontrol/version
         echo "DONE: Meta information for rccontrol written"
 
         # TODO: johbo: Make part of ac-tests
         if [ ! -f rhodecode/public/js/scripts.js ]; then
           echo "Missing scripts.js"
           exit 1
         fi
         if [ ! -f rhodecode/public/css/style.css ]; then
           echo "Missing style.css"
           exit 1
         fi
       '';
 
     });
 
     rc_testdata = self.buildPythonPackage rec {
       name = "rc_testdata-0.7.0";
       src = pkgs.fetchhg {
         url = "https://code.rhodecode.com/upstream/rc_testdata";
         rev = "v0.7.0";
         sha256 = "0w3z0zn8lagr707v67lgys23sl6pbi4xg7pfvdbw58h3q384h6rx";
       };
     };
 
   };
 
   # Apply all overrides and fix the final package set
   myPythonPackagesUnfix =
     (extends pythonExternalOverrides
     (extends pythonLocalOverrides
     (extends pythonOverrides
              pythonGeneratedPackages)));
   myPythonPackages = (fix myPythonPackagesUnfix);
 
 in myPythonPackages.rhodecode-enterprise-ce

docs/admin/apache-reverse-proxy.rst

@@ -1,33 +1,33 b''
 Apache Reverse Proxy
 ^^^^^^^^^^^^^^^^^^^^
 
 Here is a sample configuration file for using Apache as a reverse proxy.
 
 .. code-block:: apache
 
     <VirtualHost *:80>
             ServerName hg.myserver.com
             ServerAlias hg.myserver.com
 
-            ## uncomment root directive if you want to serve static files by nginx
-            ## requires static_files = false in .ini file
-            DocumentRoot /path/to/installation/rhodecode/public
+            ## uncomment root directive if you want to serve static files by
+            ## Apache requires static_files = false in .ini file
+            #DocumentRoot /path/to/rhodecode/installation/public
 
             <Proxy *>
               Order allow,deny
               Allow from all
             </Proxy>
 
-            #important !
-            #Directive to properly generate url (clone url) for pylons
+            ## Important !
+            ## Directive to properly generate url (clone url) for pylons
             ProxyPreserveHost On
 
-            #rhodecode instance
-            ProxyPass / http://127.0.0.1:5000/
-            ProxyPassReverse / http://127.0.0.1:5000/
+            ## RhodeCode instance running
+            ProxyPass / http://127.0.0.1:10002/
+            ProxyPassReverse / http://127.0.0.1:10002/
 
-            #to enable https use line below
+            ## to enable https use line below
             #SetEnvIf X-Url-Scheme https HTTPS=1
 
     </VirtualHost>
 

docs/admin/indexing.rst

@@ -1,235 +1,272 b''
 .. _indexing-ref:
 
 Full-text Search
 ----------------
 
-By default |RCM| uses `Whoosh`_ to index |repos| and provide full-text search.
+By default |RC| is configured to use `Whoosh`_ to index |repos| and
+provide full-text search.
+
+|RCE| also provides support for `Elasticsearch`_ as a backend for scalable
+search. See :ref:`enable-elasticsearch` for details.
+
+Indexing
+^^^^^^^^
+
 To run the indexer you need to use an |authtoken| with admin rights to all
 |repos|.
 
 To index new content added, you have the option to set the indexer up in a
 number of ways, for example:
 
 * Call the indexer via a cron job. We recommend running this nightly,
   unless you need everything indexed immediately.
 * Set the indexer to infinitely loop and reindex as soon as it has run its
   cycle.
 * Hook the indexer up with your CI server to reindex after each push.
 
 The indexer works by indexing new commits added since the last run. If you
 wish to build a brand new index from scratch each time,
 use the ``force`` option in the configuration file.
 
 .. important::
 
    You need to have |RCT| installed, see :ref:`install-tools`. Since |RCE|
    3.5.0 they are installed by default.
 
 To set up indexing, use the following steps:
 
 1. :ref:`config-rhoderc`, if running tools remotely.
 2. :ref:`run-index`
 3. :ref:`set-index`
 4. :ref:`advanced-indexing`
 
 .. _config-rhoderc:
 
 Configure the ``.rhoderc`` File
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 |RCT| uses the :file:`/home/{user}/.rhoderc` file for connection details
 to |RCM| instances. If this file is not automatically created,
 you can configure it using the following example. You need to configure the
 details for each instance you want to index.
 
 .. code-block:: bash
 
     # Check the instance details
     # of the instance you want to index
     $ rccontrol status
 
      - NAME: enterprise-1
      - STATUS: RUNNING
      - TYPE: Momentum
      - VERSION: 1.5.0
      - URL: http://127.0.0.1:10000
 
 To get your API Token, on the |RCM| interface go to
 :menuselection:`username --> My Account --> Auth tokens`
 
 .. code-block:: ini
 
     # Configure .rhoderc with matching details
     # This allows the indexer to connect to the instance
     [instance:enterprise-1]
     api_host = http://127.0.0.1:10000
     api_key = <auth token goes here>
     repo_dir = /home/<username>/repos
 
 .. _run-index:
 
 Run the Indexer
 ^^^^^^^^^^^^^^^
 
 Run the indexer using the following command, and specify the instance you
 want to index:
 
 .. code-block:: bash
 
    # From inside a virtualevv
    (venv)$ rhodecode-index --instance-name=enterprise-1
 
    # Using default installation
    $ /home/user/.rccontrol/enterprise-4/profile/bin/rhodecode-index \
        --instance-name=enterprise-4
 
    # Using a custom mapping file
    $ /home/user/.rccontrol/enterprise-4/profile/bin/rhodecode-index \
        --instance-name=enterprise-4 \
        --mapping=/home/user/.rccontrol/enterprise-4/mapping.ini
 
 .. note::
 
    |RCT| require |PY| 2.7 to run.
 
 .. _set-index:
 
 Schedule the Indexer
 ^^^^^^^^^^^^^^^^^^^^
 
 To schedule the indexer, configure the crontab file to run the indexer inside
 your |RCT| virtualenv using the following steps.
 
 1. Open the crontab file, using ``crontab -e``.
 2. Add the indexer to the crontab, and schedule it to run as regularly as you
    wish.
 3. Save the file.
 
 .. code-block:: bash
 
     $ crontab -e
 
     # The virtualenv can be called using its full path, so for example you can
     # put this example into the crontab
 
     # Run the indexer daily at 4am using the default mapping settings
     * 4 * * * /home/ubuntu/.virtualenv/rhodecode-venv/bin/rhodecode-index \
     --instance-name=enterprise-1
 
     # Run the indexer every Sunday at 3am using default mapping
     * 3 * * 0 /home/ubuntu/.virtualenv/rhodecode-venv/bin/rhodecode-index \
     --instance-name=enterprise-1
 
     # Run the indexer every 15 minutes
     # using a specially configured mapping file
     */15 * * * * ~/.rccontrol/enterprise-4/profile/bin/rhodecode-index \
        --instance-name=enterprise-4 \
        --mapping=/home/user/.rccontrol/enterprise-4/mapping.ini
 
 .. _advanced-indexing:
 
 Advanced Indexing
 ^^^^^^^^^^^^^^^^^
 
 |RCT| indexes based on the :file:`mapping.ini` file. To configure your index,
 you can specify different options in this file. The default location is:
 
 * :file:`/home/{user}/.rccontrol/{instance-id}/mapping.ini`, using default
   |RCT|.
 * :file:`~/venv/lib/python2.7/site-packages/rhodecode_tools/templates/mapping.ini`,
   when using ``virtualenv``.
 
 .. note::
 
     If you need to create the :file:`mapping.ini` file, use the |RCT|
     ``rhodecode-index --create-mapping path/to/file`` API call. For details,
     see the :ref:`tools-cli` section.
 
 The indexer runs in a random order to prevent a failing |repo| from stopping
 a build. To configure different indexing scenarios, set the following options
 inside the :file:`mapping.ini` and specify the altered file using the
 ``--mapping`` option.
 
 * ``index_files`` : Index the specified file types.
 * ``skip_files`` : Do not index the specified file types.
 * ``index_files_content`` : Index the content of the specified file types.
 * ``skip_files_content`` : Do not index the content of the specified files.
 * ``force`` : Create a fresh index on each run.
 * ``max_filesize`` : Files larger than the set size will not be indexed.
 * ``commit_parse_limit`` : Set the batch size when indexing commit messages.
   Set to a lower number to lessen memory load.
 * ``repo_limit`` : Set the maximum number or |repos| indexed per run.
 * ``[INCLUDE]`` : Set |repos| you want indexed. This takes precedent over
   ``[EXCLUDE]``.
 * ``[EXCLUDE]`` : Set |repos| you do not want indexed. Exclude can be used to
   not index branches, forks, or log |repos|.
 
 At the end of the file you can specify conditions for specific |repos| that
 will override the default values. To configure your indexer,
 use the following example :file:`mapping.ini` file.
 
 .. code-block:: ini
 
     [__DEFAULT__]
     # default patterns for indexing files and content of files.
     # Binary files are skipped by default.
 
     # Index python and markdown files
     index_files = *.py, *.md
 
     # Do not index these file types
     skip_files = *.svg, *.log, *.dump, *.txt
 
     # Index both file types and their content
     index_files_content = *.cpp, *.ini, *.py
 
     # Index file names, but not file content
     skip_files_content = *.svg,
 
     # Force rebuilding an index from scratch. Each repository will be rebuild
     # from scratch with a global flag. Use local flag to rebuild single repos
     force = false
 
     # Do not index files larger than 385KB
     max_filesize = 385KB
 
     # Limit commit indexing to 500 per batch
     commit_parse_limit = 500
 
     # Limit each index run to 25 repos
     repo_limit = 25
 
     # __INCLUDE__ is more important that __EXCLUDE__.
 
     [__INCLUDE__]
     # Include all repos with these names
 
     docs/* = 1
     lib/* = 1
 
     [__EXCLUDE__]
     # Do not include the following repo in index
 
     dev-docs/* = 1
     legacy-repos/* = 1
     *-dev/* = 1
 
     # Each repo that needs special indexing is a separate section below.
     # In each section set the options to override the global configuration
     # parameters above.
     # If special settings are not configured, the global configuration values
     # above are inherited. If no special repositories are
     # defined here RhodeCode will use the API to ask for all repositories
 
     # For this repo use different settings
     [special-repo]
     commit_parse_limit = 20,
     skip_files = *.idea, *.xml,
 
     # For another repo use different settings
     [another-special-repo]
     index_files = *,
     max_filesize = 800MB
     commit_parse_limit = 20000
 
+.. _enable-elasticsearch:
+
+Enabling Elasticsearch
+^^^^^^^^^^^^^^^^^^^^^^
+
+1. Open the :file:`rhodecode.ini` file for the instance you wish to edit. The
+   default location is
+   :file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
+2. Find the search configuration section:
+
+.. code-block:: ini
+
+    ###################################
+    ## SEARCH INDEXING CONFIGURATION ##
+    ###################################
+
+    search.module = rhodecode.lib.index.whoosh
+    search.location = %(here)s/data/index
+
+and change it to:
+
+.. code-block:: ini
+
+    search.module = rc_elasticsearch
+    search.location = http://localhost:9200/
+
+where ``search.location`` points to the elasticsearch server.
+
 .. _Whoosh: https://pypi.python.org/pypi/Whoosh/
+.. _Elasticsearch: https://www.elastic.co/
  No newline at end of file

docs/admin/nginx-config-example.rst

@@ -1,71 +1,72 b''
 Nginx Configuration Example
 ---------------------------
 
 Use the following example to configure Nginx as a your web server.
 
 .. code-block:: nginx
 
     upstream rc {
 
-        server 127.0.0.1:5000;
+        server 127.0.0.1:10002;
 
         # add more instances for load balancing
-        # server 127.0.0.1:5001;
-        # server 127.0.0.1:5002;
+        # server 127.0.0.1:10003;
+        # server 127.0.0.1:10004;
     }
 
     ## gist alias
 
     server {
         listen          443;
         server_name     gist.myserver.com;
         access_log      /var/log/nginx/gist.access.log;
         error_log       /var/log/nginx/gist.error.log;
 
         ssl on;
         ssl_certificate     gist.rhodecode.myserver.com.crt;
         ssl_certificate_key gist.rhodecode.myserver.com.key;
 
         ssl_session_timeout 5m;
 
         ssl_protocols SSLv3 TLSv1;
         ssl_ciphers DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:EDH-RSA-DES-CBC3-SHA:AES256-SHA:DES-CBC3-SHA:AES128-SHA:RC4-SHA:RC4-MD5;
         ssl_prefer_server_ciphers on;
         add_header Strict-Transport-Security "max-age=31536000; includeSubdomains;";
 
         # Diffie-Hellman parameter for DHE ciphersuites, recommended 2048 bits
         ssl_dhparam /etc/nginx/ssl/dhparam.pem;
 
         rewrite ^/(.+)$ https://rhodecode.myserver.com/_admin/gists/$1;
         rewrite (.*)    https://rhodecode.myserver.com/_admin/gists;
     }
 
     server {
         listen          443;
         server_name     rhodecode.myserver.com;
         access_log      /var/log/nginx/rhodecode.access.log;
         error_log       /var/log/nginx/rhodecode.error.log;
 
         ssl on;
         ssl_certificate     rhodecode.myserver.com.crt;
         ssl_certificate_key rhodecode.myserver.com.key;
 
         ssl_session_timeout 5m;
 
         ssl_protocols SSLv3 TLSv1;
         ssl_ciphers DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:EDH-RSA-DES-CBC3-SHA:AES256-SHA:DES-CBC3-SHA:AES128-SHA:RC4-SHA:RC4-MD5;
         ssl_prefer_server_ciphers on;
 
         ## uncomment root directive if you want to serve static files by nginx
         ## requires static_files = false in .ini file
-        # root /path/to/installation/rhodecode/public;
+        # root /path/to/rhodecode/installation/public;
 
         include         /etc/nginx/proxy.conf;
-            location / {
-                try_files $uri @rhode;
-            }
+
+        location / {
+            try_files $uri @rhode;
+        }
 
        location @rhode {
-            proxy_pass      http://rc;
-            }
+           proxy_pass      http://rc;
+       }
     }

docs/admin/system-overview.rst

@@ -1,163 +1,171 b''
 .. _system-overview-ref:
 
 System Overview
 ===============
 
 Latest Version
 --------------
 
 * |release| on Unix and Windows systems.
 
 System Architecture
 -------------------
 
 The following diagram shows a typical production architecture.
 
 .. image:: ../images/architecture-diagram.png
   :align: center
 
 Supported Operating Systems
 ---------------------------
 
 Linux
 ^^^^^
 
 * Ubuntu 14.04
 * CentOS 6.2 and 7
 * Debian 7.8
 * RedHat Fedora
 * Arch Linux
 * SUSE Linux
 
 Windows
 ^^^^^^^
 
 * Windows Vista Ultimate 64bit
 * Windows 7 Ultimate 64bit
 * Windows 8 Professional 64bit
 * Windows 8.1 Enterprise 64bit
 * Windows Server 2008 64bit
 * Windows Server 2008-R2 64bit
 * Windows Server 2012 64bit
 
 Supported Databases
 -------------------
 
 * SQLite
 * MySQL
 * MariaDB
 * PostgreSQL
 
 Supported Browsers
 ------------------
 
 * Chrome
 * Safari
 * Firefox
 * Internet Explorer 10 & 11
 
 System Requirements
 -------------------
 
 |RCM| performs best on machines with ultra-fast hard disks. Generally disk
 performance is more important than CPU performance. In a corporate production
 environment handling 1000s of users and |repos| you should deploy on a 12+
 core 64GB RAM server. In short, the more RAM the better.
 
+
+For example:
+
+ - for team of 1 - 5 active users you can run on 1GB RAM machine with 1CPU
+ - above 250 active users, |RCM| needs at least 8GB of memory.
+   Number of CPUs is less important, but recommended to have at least 2-3 CPUs
+
+
 .. _config-rce-files:
 
 Configuration Files
 -------------------
 
 * :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
 * :file:`/home/{user}/.rccontrol/{instance-id}/mapping.ini`
 * :file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.ini`
 * :file:`/home/{user}/.rccontrol/supervisor/supervisord.ini`
 * :file:`/home/{user}/.rccontrol.ini`
 * :file:`/home/{user}/.rhoderc`
 * :file:`/home/{user}/.rccontrol/cache/MANIFEST`
 
 For more information, see the :ref:`config-files` section.
 
 Log Files
 ---------
 
 * :file:`/home/{user}/.rccontrol/{instance-id}/enterprise.log`
 * :file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.log`
 * :file:`/home/{user}/.rccontrol/supervisor/supervisord.log`
 * :file:`/tmp/rccontrol.log`
 * :file:`/tmp/rhodecode_tools.log`
 
 Storage Files
 -------------
 
 * :file:`/home/{user}/.rccontrol/{instance-id}/data/index/{index-file.toc}`
 * :file:`/home/{user}/repos/.rc_gist_store`
 * :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.db`
 * :file:`/opt/rhodecode/store/{unique-hash}`
 
 Default Repositories Location
 -----------------------------
 
 * :file:`/home/{user}/repos`
 
 Connection Methods
 ------------------
 
 * HTTPS
 * SSH
 * |RCM| API
 
 Internationalization Support
 ----------------------------
 
 Currently available in the following languages, see `Transifex`_ for the
 latest details. If you want a new language added, please contact us. To
 configure your language settings, see the :ref:`set-lang` section.
 
 .. hlist::
 
   * Belorussian
   * Chinese
   * French
   * German
   * Italian
   * Japanese
   * Portuguese
   * Polish
   * Russian
   * Spanish
 
 Licencing Information
 ---------------------
 
 * See licencing information `here`_
 
 Peer-to-peer Failover Support
 -----------------------------
 
 * Yes
 
 Additional Binaries
 -------------------
 
 * Yes, see :ref:`rhodecode-nix-ref` for full details.
 
 Remote Connectivity
 -------------------
 
 * Available
 
 Executable Files
 ----------------
 
 Windows: :file:`RhodeCode-installer-{version}.exe`
 
 Deprecated Support
 ------------------
 
 - Internet Explorer 8 support deprecated since version 3.7.0.
 - Internet Explorer 9 support deprecated since version 3.8.0.
 
 .. _here: https://rhodecode.com/licenses/
 .. _Transifex: https://www.transifex.com/projects/p/RhodeCode/

docs/common.py

@@ -1,32 +1,34 b''
 # Try and keep this list alphabetical
 # ui is for user interface elements and messages
 # button - that's obvious
 
 rst_epilog = '''
 .. |AE| replace:: Appenlight
 .. |authtoken| replace:: Authentication Token
 .. |authtokens| replace:: **Auth Tokens**
 .. |git| replace:: Git
 .. |hg| replace:: Mercurial
 .. |svn| replace:: Subversion
 .. |LDAP| replace:: LDAP / Active Directory
 .. |os| replace:: operating system
 .. |OS| replace:: Operating System
 .. |PY| replace:: Python
 .. |pr| replace:: pull request
 .. |prs| replace:: pull requests
 .. |psf| replace:: Python Software Foundation
 .. |repo| replace:: repository
 .. |repos| replace:: repositories
 .. |RCI| replace:: RhodeCode Control
 .. |RCC| replace:: RhodeCode Control
 .. |RCV| replace:: RhodeCode Enterprise
 .. |RCM| replace:: RhodeCode Enterprise
 .. |RCE| replace:: RhodeCode Enterprise
+.. |RCCE| replace:: RhodeCode Community
+.. |RCEE| replace:: RhodeCode Enterprise
 .. |RCX| replace:: RhodeCode Extensions
 .. |RCT| replace:: RhodeCode Tools
 .. |RCEBOLD| replace:: **RhodeCode Enterprise**
 .. |RCEITALICS| replace:: `RhodeCode Enterprise`
 .. |RC| replace:: RhodeCode
 .. |RNS| replace:: Release Notes
 '''

docs/install/database-string.rst

@@ -1,38 +1,38 b''
 .. _config-database:
 
 Make Database Changes
 ---------------------
 
 .. important::
 
-   If you do change the |repo| database that |RCM| uses, then you will need to
+   If you do change the |repo| database that |RCEE| uses, then you will need to
    upgrade the database, and also remap and rescan the |repos|. More detailed
    information is available in the
    :ref:`Alternative upgrade documentation <control:install-port>`.
 
-If you need to change database connection details for a |RCM| instance,
+If you need to change database connection details for a |RCEE| instance,
 use the following steps:
 
 1. Open the :file:`rhodecode.ini` file for the instance you wish to edit. The
    default location is
    :file:`home/{user}/.rccontrol/{instance-id}/rhodecode.ini`
 2. When you open the file, find the database configuration section,
    and use the below example to change the
    connection details:
 
 .. code-block:: ini
 
     #########################################################
     ### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
     #########################################################
 
     # Default SQLite config
     sqlalchemy.db1.url = sqlite:////home/brian/.rccontrol/enterprise-1/rhodecode.db
 
     # Use this example for a PostgreSQL
     sqlalchemy.db1.url = postgresql://postgres:qwe@localhost/rhodecode
 
     # see sqlalchemy docs for other advanced settings
     sqlalchemy.db1.echo = false
     sqlalchemy.db1.pool_recycle = 3600
     sqlalchemy.db1.convert_unicode = true

docs/install/quick-start.rst

@@ -1,102 +1,109 b''
 .. _quick-start:
 
 Quick Start Guide
 =================
 
 .. important::
 
     These are quick start instructions. To optimize your |RCE|,
     |RCC|, and |RCT| usage, read the more detailed instructions in our guides.
     For detailed installation instructions, see
     :ref:`RhodeCode Control Documentation <control:rcc>`
 
 .. tip::
 
    If using a non-SQLite database, install and configure the database, create
    a new user, and grant permissions. You will be prompted for this user's
    credentials during |RCE| installation. See the relevant database
    documentation for more details.
 
-To get |RCM| up and running, run through the below steps:
+To get |RCE| up and running, run through the below steps:
 
 1. Download the latest |RCC| installer from your `rhodecode.com`_ profile
-   page. If you don't have an account, sign up at `rhodecode.com/register`_.
+   or main page.
+   If you don't have an account, sign up at `rhodecode.com/register`_.
+
 2. Run the |RCC| installer and accept the End User Licence using the
    following example:
 
 .. code-block:: bash
 
     $ chmod 755 RhodeCode-installer-linux-*
     $ ./RhodeCode-installer-linux-*
 
 3. Install a VCS Server, and configure it to start at boot.
 
 .. code-block:: bash
 
     $ rccontrol install VCSServer
 
     Agree to the licence agreement? [y/N]: y
     IP to start the server on [127.0.0.1]:
     Port for the server to start [10005]:
     Creating new instance: vcsserver-1
     Installing RhodeCode VCSServer
     Configuring RhodeCode VCS Server ...
     Supervisord state is: RUNNING
     Added process group vcsserver-1
 
 
-4. Install |RCE|. If using MySQL or PostgreSQL, during installation you'll be
-   asked for your database credentials, so have them at hand. You don't need
-   any for SQLite.
+4. Install |RCEE| or |RCCE|. If using MySQL or PostgreSQL, during
+   installation you'll be asked for your database credentials, so have them at hand.
+   Mysql or Postgres needs to be running and a new database needs to be created.
+   You don't need any credentials or to create a database for SQLite.
 
 .. code-block:: bash
    :emphasize-lines: 11-16
 
+    $ rccontrol install Community
+
+    or
+
     $ rccontrol install Enterprise
 
     Username [admin]: username
     Password (min 6 chars):
     Repeat for confirmation:
     Email: [e-mail]
     Respositories location [/home/brian/repos]:
     IP to start the Enterprise server on [127.0.0.1]:
     Port for the Enterprise server to use [10004]:
     Database type - [s]qlite, [m]ysql, [p]ostresql:
     PostgreSQL selected
     Database host [127.0.0.1]:
     Database port [5432]:
     Database username: db-user-name
     Database password: somepassword
     Database name: example-db-name
 
-5. Check the status of your installation. You |RCE| instance runs on the URL
-   displayed in the status message.
+5. Check the status of your installation. You |RCEE|/|RCCE| instance runs
+   on the URL displayed in the status message.
 
 .. code-block:: bash
 
     $ rccontrol status
 
     - NAME: enterprise-1
     - STATUS: RUNNING
     - TYPE: Enterprise
-    - VERSION: 3.3.0
+    - VERSION: 4.1.0
     - URL: http://127.0.0.1:10003
 
     - NAME: vcsserver-1
     - STATUS: RUNNING
     - TYPE: VCSServer
-    - VERSION: 3.3.0
+    - VERSION: 4.1.0
     - URL: http://127.0.0.1:10001
 
 .. note::
 
    Recommended post quick start install instructions:
 
    * Read the documentation
    * Carry out the :ref:`rhodecode-post-instal-ref`
    * Set up :ref:`indexing-ref`
    * Familiarise yourself with the :ref:`rhodecode-admin-ref` section.
 
 .. _rhodecode.com/download/: https://rhodecode.com/download/
 .. _rhodecode.com: https://rhodecode.com/
 .. _rhodecode.com/register: https://rhodecode.com/register/

docs/release-notes/release-notes-4.0.0.rst

@@ -1,58 +1,64 b''
 |RCE| 4.0.0 |RNS|
 -----------------
 
 Release Date
 ^^^^^^^^^^^^
 
 - 2016-05-24
 
 General
 ^^^^^^^
 
 - Introduced Pyramid as a Pylons framework replacement. (porting still ongoing).
   Added few of components as plugins. Exposed RhodeCode plugins API for 3rd
   parties to extend RhodeCode functionality with custom pyramid apps. Pyramid
   is also our route to python3 support.
 - Various UX/UI improvements.
   - new summary page
   - new file browser (more consistent)
   - re-done admin section and added Panels
   - various other tweaks and improvements
 - Alternative fast and scalable HTTP based communication backend for VCServer.
   It soon will replace Pyro4.
 - Rewrote few caching techniques used and simplified those
 
 
 New Features
 ^^^^^^^^^^^^
 
 - RhodeCode code-review live chat (EE only). A live communication
   tool built right into the code-review process to quickly
   collaborate on crucial parts of code.
 
 - Elastic Search backend (EE only). Alternative backend to existing
   Whoosh to handle, large amount of data for full text search.
 
 - Social Auth (EE only): added new social authentication backends including:
   Github, Twitter, Bitbucket and Google. It's possible now to use your
   Google account to log in to RhodeCode and take advantage of things like 2FA.
 
+- Search: full text search now properly orders commits by date, and shows line
+  numbers for file content search.
+
+
 Security
 ^^^^^^^^
 
 - Added new action loggers for actions like adding/revoking permissions.
 
 
 Performance
 ^^^^^^^^^^^
 
-- Optimized admin pannels to faster load large ammount of data
+- Optimized admin panels to faster load large amount of data
 - Improved file tree loading speed
+- New HTTP backend is ~10% faster, and doesn't require so many threads
+  for vcsserver
 
 
 Fixes
 ^^^^^
 
 - Fixed backreferences to user group when deleting users
 - Fixed LDAP group user-group matching
 - Improved SVN support for various commands (MKOL, etc)
  No newline at end of file

docs/release-notes/release-notes.rst

@@ -1,76 +1,80 b''
 .. _rhodecode-release-notes-ref:
 
 Release Notes
 =============
 
 |RCE| 4.x Versions
 ------------------
 
+.. toctree::
+   :maxdepth: 1
+
+   release-notes-4.1.0.rst
    release-notes-4.0.1.rst
    release-notes-4.0.0.rst
 
 |RCE| 3.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-3.8.4.rst
    release-notes-3.8.3.rst
    release-notes-3.8.2.rst
    release-notes-3.8.1.rst
    release-notes-3.8.0.rst
    release-notes-3.7.1.rst
    release-notes-3.7.0.rst
    release-notes-3.6.1.rst
    release-notes-3.6.0.rst
    release-notes-3.5.2.rst
    release-notes-3.5.1.rst
    release-notes-3.5.0.rst
    release-notes-3.4.1.rst
    release-notes-3.4.0.rst
    release-notes-3.3.4.rst
    release-notes-3.3.3.rst
    release-notes-3.3.2.rst
    release-notes-3.3.1.rst
    release-notes-3.3.0.rst
    release-notes-3.2.3.rst
    release-notes-3.2.2.rst
    release-notes-3.2.1.rst
    release-notes-3.2.0.rst
    release-notes-3.1.1.rst
    release-notes-3.1.0.rst
    release-notes-3.0.2.rst
    release-notes-3.0.1.rst
    release-notes-3.0.0.rst
 
 |RCE| 2.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-2.2.8.rst
    release-notes-2.2.7.rst
    release-notes-2.2.6.rst
    release-notes-2.2.5.rst
    release-notes-2.2.4.rst
    release-notes-2.2.3.rst
    release-notes-2.2.2.rst
    release-notes-2.2.1.rst
    release-notes-2.2.0.rst
    release-notes-2.1.0.rst
    release-notes-2.0.2.rst
    release-notes-2.0.1.rst
    release-notes-2.0.0.rst
 
 |RCE| 1.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-1.7.2.rst
    release-notes-1.7.1.rst
    release-notes-1.7.0.rst
    release-notes-1.6.0.rst

pkgs/patch-rhodecode-tools-setup.diff

@@ -1,13 +1,12 b''
 diff --git a/requirements.txt b/requirements.txt
 --- a/requirements.txt
 +++ b/requirements.txt
-@@ -1,8 +1,8 @@
- click==5.1
- future==0.14.3
+@@ -3,7 +3,7 @@future==0.14.3
  six==1.9.0
  mako==1.0.1
  markupsafe==0.23
 -requests==2.5.1
 +requests
+ #responses
  whoosh==2.7.0
- pyelasticsearch==1.4
+ elasticsearch==2.3.0
  No newline at end of file

pkgs/python-packages-overrides.nix

@@ -1,151 +1,165 b''
 # Overrides for the generated python-packages.nix
 #
 # This function is intended to be used as an extension to the generated file
 # python-packages.nix. The main objective is to add needed dependencies of C
 # libraries and tweak the build instructions where needed.
 
 { pkgs, basePythonPackages }:
 
 let
   sed = "sed -i";
 in
 
 self: super: {
 
   gnureadline = super.gnureadline.override (attrs: {
     buildInputs = attrs.buildInputs ++ [
       pkgs.ncurses
     ];
     patchPhase = ''
       substituteInPlace setup.py --replace "/bin/bash" "${pkgs.bash}/bin/bash"
     '';
   });
 
+  gunicorn = super.gunicorn.override (attrs: {
+    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+      # johbo: futures is needed as long as we are on Python 2, otherwise
+      # gunicorn explodes if used with multiple threads per worker.
+      self.futures
+    ];
+  });
+
+  ipython = super.ipython.override (attrs: {
+    propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
+      self.gnureadline
+    ];
+  });
+
   kombu = super.kombu.override (attrs: {
     # The current version of kombu needs some patching to work with the
     # other libs. Should be removed once we update celery and kombu.
     patches = [
       ./patch-kombu-py-2-7-11.diff
       ./patch-kombu-msgpack.diff
     ];
   });
 
   lxml = super.lxml.override (attrs: {
     buildInputs = with self; [
       pkgs.libxml2
       pkgs.libxslt
     ];
   });
 
   MySQL-python = super.MySQL-python.override (attrs: {
     buildInputs = attrs.buildInputs ++ [
       pkgs.openssl
     ];
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.mysql.lib
       pkgs.zlib
     ];
   });
 
   psutil = super.psutil.override (attrs: {
     buildInputs = attrs.buildInputs ++
       pkgs.lib.optional pkgs.stdenv.isDarwin pkgs.darwin.IOKit;
   });
 
   psycopg2 = super.psycopg2.override (attrs: {
     buildInputs = attrs.buildInputs ++
       pkgs.lib.optional pkgs.stdenv.isDarwin pkgs.openssl;
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.postgresql
     ];
   });
 
   pycurl = super.pycurl.override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.curl
       pkgs.openssl
     ];
     preConfigure = ''
       substituteInPlace setup.py --replace '--static-libs' '--libs'
       export PYCURL_SSL_LIBRARY=openssl
     '';
   });
 
   Pylons = super.Pylons.override (attrs: {
     name = "Pylons-1.0.1-patch1";
     src = pkgs.fetchgit {
       url = "https://code.rhodecode.com/upstream/pylons";
       rev = "707354ee4261b9c10450404fc9852ccea4fd667d";
       sha256 = "b2763274c2780523a335f83a1df65be22ebe4ff413a7bc9e9288d23c1f62032e";
     };
   });
 
   pyramid = super.pyramid.override (attrs: {
     postFixup = ''
       wrapPythonPrograms
       # TODO: johbo: "wrapPython" adds this magic line which
       # confuses pserve.
       ${sed} '/import sys; sys.argv/d' $out/bin/.pserve-wrapped
     '';
   });
 
   Pyro4 = super.Pyro4.override (attrs: {
     # TODO: Was not able to generate this version, needs further
     # investigation.
     name = "Pyro4-4.35";
     src = pkgs.fetchurl {
       url = "https://pypi.python.org/packages/source/P/Pyro4/Pyro4-4.35.src.tar.gz";
       md5 = "cbe6cb855f086a0f092ca075005855f3";
     };
   });
 
   pysqlite = super.pysqlite.override (attrs: {
     propagatedBuildInputs = [
       pkgs.sqlite
     ];
   });
 
   pytest-runner = super.pytest-runner.override (attrs: {
     propagatedBuildInputs = [
       self.setuptools-scm
     ];
   });
 
   python-ldap = super.python-ldap.override (attrs: {
     propagatedBuildInputs = attrs.propagatedBuildInputs ++ [
       pkgs.cyrus_sasl
       pkgs.openldap
       pkgs.openssl
     ];
     NIX_CFLAGS_COMPILE = "-I${pkgs.cyrus_sasl}/include/sasl";
   });
 
   python-pam = super.python-pam.override (attrs:
     let
       includeLibPam = pkgs.stdenv.isLinux;
     in {
       # TODO: johbo: Move the option up into the default.nix, we should
       # include python-pam only on supported platforms.
       propagatedBuildInputs = attrs.propagatedBuildInputs ++
         pkgs.lib.optional includeLibPam [
           pkgs.pam
         ];
       # TODO: johbo: Check if this can be avoided, or transform into
       # a real patch
       patchPhase = pkgs.lib.optionals includeLibPam ''
         substituteInPlace pam.py \
           --replace 'find_library("pam")' '"${pkgs.pam}/lib/libpam.so.0"'
       '';
     });
 
   rhodecode-tools = super.rhodecode-tools.override (attrs: {
     patches = [
       ./patch-rhodecode-tools-setup.diff
     ];
   });
 
   # Avoid that setuptools is replaced, this leads to trouble
   # with buildPythonPackage.
   setuptools = basePythonPackages.setuptools;
 
 }

pkgs/python-packages.nix

@@ -1,1273 +1,1263 b''
 {
   Babel = super.buildPythonPackage {
     name = "Babel-1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pytz];
     src = fetchurl {
       url = "https://pypi.python.org/packages/33/27/e3978243a03a76398c384c83f7ca879bc6e8f1511233a621fcada135606e/Babel-1.3.tar.gz";
       md5 = "5264ceb02717843cbc9ffce8e6e06bdb";
     };
   };
   Beaker = super.buildPythonPackage {
     name = "Beaker-1.7.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/97/8e/409d2e7c009b8aa803dc9e6f239f1db7c3cdf578249087a404e7c27a505d/Beaker-1.7.0.tar.gz";
       md5 = "386be3f7fe427358881eee4622b428b3";
     };
   };
   CProfileV = super.buildPythonPackage {
     name = "CProfileV-1.0.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [bottle];
     src = fetchurl {
       url = "https://pypi.python.org/packages/eb/df/983a0b6cfd3ac94abf023f5011cb04f33613ace196e33f53c86cf91850d5/CProfileV-1.0.6.tar.gz";
       md5 = "08c7c242b6e64237bc53c5d13537e03d";
     };
   };
   Fabric = super.buildPythonPackage {
     name = "Fabric-1.10.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [paramiko];
     src = fetchurl {
       url = "https://pypi.python.org/packages/e3/5f/b6ebdb5241d5ec9eab582a5c8a01255c1107da396f849e538801d2fe64a5/Fabric-1.10.0.tar.gz";
       md5 = "2cb96473387f0e7aa035210892352f4a";
     };
   };
   FormEncode = super.buildPythonPackage {
     name = "FormEncode-1.2.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/8e/59/0174271a6f004512e0201188593e6d319db139d14cb7490e488bbb078015/FormEncode-1.2.4.tar.gz";
       md5 = "6bc17fb9aed8aea198975e888e2077f4";
     };
   };
   Jinja2 = super.buildPythonPackage {
     name = "Jinja2-2.7.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [MarkupSafe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b0/73/eab0bca302d6d6a0b5c402f47ad1760dc9cb2dd14bbc1873ad48db258e4d/Jinja2-2.7.3.tar.gz";
       md5 = "b9dffd2f3b43d673802fe857c8445b1a";
     };
   };
   Mako = super.buildPythonPackage {
     name = "Mako-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [MarkupSafe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/8e/a4/aa56533ecaa5f22ca92428f74e074d0c9337282933c722391902c8f9e0f8/Mako-1.0.1.tar.gz";
       md5 = "9f0aafd177b039ef67b90ea350497a54";
     };
   };
   Markdown = super.buildPythonPackage {
     name = "Markdown-2.6.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/62/8b/83658b5f6c220d5fcde9f9852d46ea54765d734cfbc5a9f4c05bfc36db4d/Markdown-2.6.2.tar.gz";
       md5 = "256d19afcc564dc4ce4c229bb762f7ae";
     };
   };
   MarkupSafe = super.buildPythonPackage {
     name = "MarkupSafe-0.23";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c0/41/bae1254e0396c0cc8cf1751cb7d9afc90a602353695af5952530482c963f/MarkupSafe-0.23.tar.gz";
       md5 = "f5ab3deee4c37cd6a922fb81e730da6e";
     };
   };
   MySQL-python = super.buildPythonPackage {
     name = "MySQL-python-1.2.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/a5/e9/51b544da85a36a68debe7a7091f068d802fc515a3a202652828c73453cad/MySQL-python-1.2.5.zip";
       md5 = "654f75b302db6ed8dc5a898c625e030c";
     };
   };
   Paste = super.buildPythonPackage {
     name = "Paste-2.0.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six];
     src = fetchurl {
       url = "https://pypi.python.org/packages/d5/8d/0f8ac40687b97ff3e07ebd1369be20bdb3f93864d2dc3c2ff542edb4ce50/Paste-2.0.2.tar.gz";
       md5 = "4bfc8a7eaf858f6309d2ac0f40fc951c";
     };
   };
   PasteDeploy = super.buildPythonPackage {
     name = "PasteDeploy-1.5.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/0f/90/8e20cdae206c543ea10793cbf4136eb9a8b3f417e04e40a29d72d9922cbd/PasteDeploy-1.5.2.tar.gz";
       md5 = "352b7205c78c8de4987578d19431af3b";
     };
   };
   PasteScript = super.buildPythonPackage {
     name = "PasteScript-1.7.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [Paste PasteDeploy];
     src = fetchurl {
       url = "https://pypi.python.org/packages/a5/05/fc60efa7c2f17a1dbaeccb2a903a1e90902d92b9d00eebabe3095829d806/PasteScript-1.7.5.tar.gz";
       md5 = "4c72d78dcb6bb993f30536842c16af4d";
     };
   };
   Pygments = super.buildPythonPackage {
     name = "Pygments-2.0.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f4/c6/bdbc5a8a112256b2b6136af304dbae93d8b1ef8738ff2d12a51018800e46/Pygments-2.0.2.tar.gz";
       md5 = "238587a1370d62405edabd0794b3ec4a";
     };
   };
   Pylons = super.buildPythonPackage {
     name = "Pylons-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [Routes WebHelpers Beaker Paste PasteDeploy PasteScript FormEncode simplejson decorator nose Mako WebError WebTest Tempita MarkupSafe WebOb];
     src = fetchurl {
       url = "https://pypi.python.org/packages/a2/69/b835a6bad00acbfeed3f33c6e44fa3f936efc998c795bfb15c61a79ecf62/Pylons-1.0.1.tar.gz";
       md5 = "6cb880d75fa81213192142b07a6e4915";
     };
   };
   Pyro4 = super.buildPythonPackage {
     name = "Pyro4-4.41";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [serpent];
     src = fetchurl {
       url = "https://pypi.python.org/packages/56/2b/89b566b4bf3e7f8ba790db2d1223852f8cb454c52cab7693dd41f608ca2a/Pyro4-4.41.tar.gz";
       md5 = "ed69e9bfafa9c06c049a87cb0c4c2b6c";
     };
   };
   Routes = super.buildPythonPackage {
     name = "Routes-1.13";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [repoze.lru];
     src = fetchurl {
       url = "https://pypi.python.org/packages/88/d3/259c3b3cde8837eb9441ab5f574a660e8a4acea8f54a078441d4d2acac1c/Routes-1.13.tar.gz";
       md5 = "d527b0ab7dd9172b1275a41f97448783";
     };
   };
   SQLAlchemy = super.buildPythonPackage {
     name = "SQLAlchemy-0.9.9";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/28/f7/1bbfd0d8597e8c358d5e15a166a486ad82fc5579b4e67b6ef7c05b1d182b/SQLAlchemy-0.9.9.tar.gz";
       md5 = "8a10a9bd13ed3336ef7333ac2cc679ff";
     };
   };
   Sphinx = super.buildPythonPackage {
     name = "Sphinx-1.2.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [Pygments docutils Jinja2];
     src = fetchurl {
       url = "https://pypi.python.org/packages/0a/50/34017e6efcd372893a416aba14b84a1a149fc7074537b0e9cb6ca7b7abe9/Sphinx-1.2.2.tar.gz";
       md5 = "3dc73ccaa8d0bfb2d62fb671b1f7e8a4";
     };
   };
   Tempita = super.buildPythonPackage {
     name = "Tempita-0.5.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/56/c8/8ed6eee83dbddf7b0fc64dd5d4454bc05e6ccaafff47991f73f2894d9ff4/Tempita-0.5.2.tar.gz";
       md5 = "4c2f17bb9d481821c41b6fbee904cea1";
     };
   };
   URLObject = super.buildPythonPackage {
     name = "URLObject-2.4.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/cb/b6/e25e58500f9caef85d664bec71ec67c116897bfebf8622c32cb75d1ca199/URLObject-2.4.0.tar.gz";
       md5 = "2ed819738a9f0a3051f31dc9924e3065";
     };
   };
   WebError = super.buildPythonPackage {
     name = "WebError-0.10.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [WebOb Tempita Pygments Paste];
     src = fetchurl {
       url = "https://pypi.python.org/packages/35/76/e7e5c2ce7e9c7f31b54c1ff295a495886d1279a002557d74dd8957346a79/WebError-0.10.3.tar.gz";
       md5 = "84b9990b0baae6fd440b1e60cdd06f9a";
     };
   };
   WebHelpers = super.buildPythonPackage {
     name = "WebHelpers-1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [MarkupSafe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/ee/68/4d07672821d514184357f1552f2dad923324f597e722de3b016ca4f7844f/WebHelpers-1.3.tar.gz";
       md5 = "32749ffadfc40fea51075a7def32588b";
     };
   };
   WebHelpers2 = super.buildPythonPackage {
     name = "WebHelpers2-2.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [MarkupSafe six];
     src = fetchurl {
       url = "https://pypi.python.org/packages/ff/30/56342c6ea522439e3662427c8d7b5e5b390dff4ff2dc92d8afcb8ab68b75/WebHelpers2-2.0.tar.gz";
       md5 = "0f6b68d70c12ee0aed48c00b24da13d3";
     };
   };
   WebOb = super.buildPythonPackage {
     name = "WebOb-1.3.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/16/78/adfc0380b8a0d75b2d543fa7085ba98a573b1ae486d9def88d172b81b9fa/WebOb-1.3.1.tar.gz";
       md5 = "20918251c5726956ba8fef22d1556177";
     };
   };
   WebTest = super.buildPythonPackage {
     name = "WebTest-1.4.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [WebOb];
     src = fetchurl {
       url = "https://pypi.python.org/packages/51/3d/84fd0f628df10b30c7db87895f56d0158e5411206b721ca903cb51bfd948/WebTest-1.4.3.zip";
       md5 = "631ce728bed92c681a4020a36adbc353";
     };
   };
   Whoosh = super.buildPythonPackage {
     name = "Whoosh-2.7.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/1c/dc/2f0231ff3875ded36df8c1ab851451e51a237dc0e5a86d3d96036158da94/Whoosh-2.7.0.zip";
       md5 = "7abfd970f16fadc7311960f3fa0bc7a9";
     };
   };
   alembic = super.buildPythonPackage {
     name = "alembic-0.8.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [SQLAlchemy Mako python-editor];
     src = fetchurl {
       url = "https://pypi.python.org/packages/ca/7e/299b4499b5c75e5a38c5845145ad24755bebfb8eec07a2e1c366b7181eeb/alembic-0.8.4.tar.gz";
       md5 = "5f95d8ee62b443f9b37eb5bee76c582d";
     };
   };
   amqplib = super.buildPythonPackage {
     name = "amqplib-1.0.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/75/b7/8c2429bf8d92354a0118614f9a4d15e53bc69ebedce534284111de5a0102/amqplib-1.0.2.tgz";
       md5 = "5c92f17fbedd99b2b4a836d4352d1e2f";
     };
   };
   anyjson = super.buildPythonPackage {
     name = "anyjson-0.3.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c3/4d/d4089e1a3dd25b46bebdb55a992b0797cff657b4477bc32ce28038fdecbc/anyjson-0.3.3.tar.gz";
       md5 = "2ea28d6ec311aeeebaf993cb3008b27c";
     };
   };
   appenlight-client = super.buildPythonPackage {
     name = "appenlight-client-0.6.14";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [WebOb requests];
     src = fetchurl {
       url = "https://pypi.python.org/packages/4d/e0/23fee3ebada8143f707e65c06bcb82992040ee64ea8355e044ed55ebf0c1/appenlight_client-0.6.14.tar.gz";
       md5 = "578c69b09f4356d898fff1199b98a95c";
     };
   };
   authomatic = super.buildPythonPackage {
     name = "authomatic-0.1.0.post1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/08/1a/8a930461e604c2d5a7a871e1ac59fa82ccf994c32e807230c8d2fb07815a/Authomatic-0.1.0.post1.tar.gz";
       md5 = "be3f3ce08747d776aae6d6cc8dcb49a9";
     };
   };
   backport-ipaddress = super.buildPythonPackage {
     name = "backport-ipaddress-0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/d3/30/54c6dab05a4dec44db25ff309f1fbb6b7a8bde3f2bade38bb9da67bbab8f/backport_ipaddress-0.1.tar.gz";
       md5 = "9c1f45f4361f71b124d7293a60006c05";
     };
   };
   bottle = super.buildPythonPackage {
     name = "bottle-0.12.8";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/52/df/e4a408f3a7af396d186d4ecd3b389dd764f0f943b4fa8d257bfe7b49d343/bottle-0.12.8.tar.gz";
       md5 = "13132c0a8f607bf860810a6ee9064c5b";
     };
   };
   bumpversion = super.buildPythonPackage {
     name = "bumpversion-0.5.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/14/41/8c9da3549f8e00c84f0432c3a8cf8ed6898374714676aab91501d48760db/bumpversion-0.5.3.tar.gz";
       md5 = "c66a3492eafcf5ad4b024be9fca29820";
     };
   };
   celery = super.buildPythonPackage {
     name = "celery-2.2.10";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [python-dateutil anyjson kombu pyparsing];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b1/64/860fd50e45844c83442e7953effcddeff66b2851d90b2d784f7201c111b8/celery-2.2.10.tar.gz";
       md5 = "898bc87e54f278055b561316ba73e222";
     };
   };
-  certifi = super.buildPythonPackage {
-    name = "certifi-2016.2.28";
-    buildInputs = with self; [];
-    doCheck = false;
-    propagatedBuildInputs = with self; [];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/5c/f8/f6c54727c74579c6bbe5926f5deb9677c5810a33e11da58d1a4e2d09d041/certifi-2016.2.28.tar.gz";
-      md5 = "5d672aa766e1f773c75cfeccd02d3650";
-    };
-  };
   click = super.buildPythonPackage {
     name = "click-5.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b7/34/a496632c4fb6c1ee76efedf77bb8d28b29363d839953d95095b12defe791/click-5.1.tar.gz";
       md5 = "9c5323008cccfe232a8b161fc8196d41";
     };
   };
   colander = super.buildPythonPackage {
     name = "colander-1.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [translationstring iso8601];
     src = fetchurl {
       url = "https://pypi.python.org/packages/14/23/c9ceba07a6a1dc0eefbb215fc0dc64aabc2b22ee756bc0f0c13278fa0887/colander-1.2.tar.gz";
       md5 = "83db21b07936a0726e588dae1914b9ed";
     };
   };
   configobj = super.buildPythonPackage {
     name = "configobj-5.0.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six];
     src = fetchurl {
       url = "https://pypi.python.org/packages/64/61/079eb60459c44929e684fa7d9e2fdca403f67d64dd9dbac27296be2e0fab/configobj-5.0.6.tar.gz";
       md5 = "e472a3a1c2a67bb0ec9b5d54c13a47d6";
     };
   };
   cov-core = super.buildPythonPackage {
     name = "cov-core-1.15.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [coverage];
     src = fetchurl {
       url = "https://pypi.python.org/packages/4b/87/13e75a47b4ba1be06f29f6d807ca99638bedc6b57fa491cd3de891ca2923/cov-core-1.15.0.tar.gz";
       md5 = "f519d4cb4c4e52856afb14af52919fe6";
     };
   };
   coverage = super.buildPythonPackage {
     name = "coverage-3.7.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/09/4f/89b06c7fdc09687bca507dc411c342556ef9c5a3b26756137a4878ff19bf/coverage-3.7.1.tar.gz";
       md5 = "c47b36ceb17eaff3ecfab3bcd347d0df";
     };
   };
   cssselect = super.buildPythonPackage {
     name = "cssselect-0.9.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/aa/e5/9ee1460d485b94a6d55732eb7ad5b6c084caf73dd6f9cb0bb7d2a78fafe8/cssselect-0.9.1.tar.gz";
       md5 = "c74f45966277dc7a0f768b9b0f3522ac";
     };
   };
   decorator = super.buildPythonPackage {
     name = "decorator-3.4.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/35/3a/42566eb7a2cbac774399871af04e11d7ae3fc2579e7dae85213b8d1d1c57/decorator-3.4.2.tar.gz";
       md5 = "9e0536870d2b83ae27d58dbf22582f4d";
     };
   };
   docutils = super.buildPythonPackage {
     name = "docutils-0.12";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/37/38/ceda70135b9144d84884ae2fc5886c6baac4edea39550f28bcd144c1234d/docutils-0.12.tar.gz";
       md5 = "4622263b62c5c771c03502afa3157768";
     };
   };
   dogpile.cache = super.buildPythonPackage {
     name = "dogpile.cache-0.5.7";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [dogpile.core];
     src = fetchurl {
       url = "https://pypi.python.org/packages/07/74/2a83bedf758156d9c95d112691bbad870d3b77ccbcfb781b4ef836ea7d96/dogpile.cache-0.5.7.tar.gz";
       md5 = "3e58ce41af574aab41d78e9c4190f194";
     };
   };
   dogpile.core = super.buildPythonPackage {
     name = "dogpile.core-0.4.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/0e/77/e72abc04c22aedf874301861e5c1e761231c288b5de369c18be8f4b5c9bb/dogpile.core-0.4.1.tar.gz";
       md5 = "01cb19f52bba3e95c9b560f39341f045";
     };
   };
   dulwich = super.buildPythonPackage {
     name = "dulwich-0.12.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/6f/04/fbe561b6d45c0ec758330d5b7f5ba4b6cb4f1ca1ab49859d2fc16320da75/dulwich-0.12.0.tar.gz";
       md5 = "f3a8a12bd9f9dd8c233e18f3d49436fa";
     };
   };
   ecdsa = super.buildPythonPackage {
     name = "ecdsa-0.11";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/6c/3f/92fe5dcdcaa7bd117be21e5520c9a54375112b66ec000d209e9e9519fad1/ecdsa-0.11.tar.gz";
       md5 = "8ef586fe4dbb156697d756900cb41d7c";
     };
   };
   elasticsearch = super.buildPythonPackage {
-    name = "elasticsearch-1.9.0";
+    name = "elasticsearch-2.3.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [urllib3];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/13/9b/540e311b31a10c2a904acfb08030c656047e5c7ba479d35df2799e5dccfe/elasticsearch-1.9.0.tar.gz";
-      md5 = "3550390baea1639479f79758d66ab032";
+      url = "https://pypi.python.org/packages/10/35/5fd52c5f0b0ee405ed4b5195e8bce44c5e041787680dc7b94b8071cac600/elasticsearch-2.3.0.tar.gz";
+      md5 = "2550f3b51629cf1ef9636608af92c340";
+    };
+  };
+  elasticsearch-dsl = super.buildPythonPackage {
+    name = "elasticsearch-dsl-2.0.0";
+    buildInputs = with self; [];
+    doCheck = false;
+    propagatedBuildInputs = with self; [six python-dateutil elasticsearch];
+    src = fetchurl {
+      url = "https://pypi.python.org/packages/4e/5d/e788ae8dbe2ff4d13426db0a027533386a5c276c77a2654dc0e2007ce04a/elasticsearch-dsl-2.0.0.tar.gz";
+      md5 = "4cdfec81bb35383dd3b7d02d7dc5ee68";
     };
   };
   flake8 = super.buildPythonPackage {
     name = "flake8-2.4.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pyflakes pep8 mccabe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/8f/b5/9a73c66c7dba273bac8758398f060c008a25f3e84531063b42503b5d0a95/flake8-2.4.1.tar.gz";
       md5 = "ed45d3db81a3b7c88bd63c6e37ca1d65";
     };
   };
   future = super.buildPythonPackage {
     name = "future-0.14.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/83/80/8ef3a11a15f8eaafafa0937b20c1b3f73527e69ab6b3fa1cf94a5a96aabb/future-0.14.3.tar.gz";
       md5 = "e94079b0bd1fc054929e8769fc0f6083";
     };
   };
   futures = super.buildPythonPackage {
     name = "futures-3.0.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f8/e7/fc0fcbeb9193ba2d4de00b065e7fd5aecd0679e93ce95a07322b2b1434f4/futures-3.0.2.tar.gz";
       md5 = "42aaf1e4de48d6e871d77dc1f9d96d5a";
     };
   };
   gnureadline = super.buildPythonPackage {
     name = "gnureadline-6.3.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/3a/ee/2c3f568b0a74974791ac590ec742ef6133e2fbd287a074ba72a53fa5e97c/gnureadline-6.3.3.tar.gz";
       md5 = "c4af83c9a3fbeac8f2da9b5a7c60e51c";
     };
   };
   gprof2dot = super.buildPythonPackage {
-    name = "gprof2dot-2015.12.1";
+    name = "gprof2dot-2015.12.01";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b9/34/7bf93c1952d40fa5c95ad963f4d8344b61ef58558632402eca18e6c14127/gprof2dot-2015.12.1.tar.gz";
       md5 = "e23bf4e2f94db032750c193384b4165b";
     };
   };
   greenlet = super.buildPythonPackage {
-    name = "greenlet-0.4.7";
+    name = "greenlet-0.4.9";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/7a/9f/a1a0d9bdf3203ae1502c5a8434fe89d323599d78a106985bc327351a69d4/greenlet-0.4.7.zip";
-      md5 = "c2333a8ff30fa75c5d5ec0e67b461086";
+      url = "https://pypi.python.org/packages/4e/3d/9d421539b74e33608b245092870156b2e171fb49f2b51390aa4641eecb4a/greenlet-0.4.9.zip";
+      md5 = "c6659cdb2a5e591723e629d2eef22e82";
     };
   };
   gunicorn = super.buildPythonPackage {
     name = "gunicorn-19.6.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/84/ce/7ea5396efad1cef682bbc4068e72a0276341d9d9d0f501da609fab9fcb80/gunicorn-19.6.0.tar.gz";
       md5 = "338e5e8a83ea0f0625f768dba4597530";
     };
   };
   infrae.cache = super.buildPythonPackage {
     name = "infrae.cache-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [Beaker repoze.lru];
     src = fetchurl {
       url = "https://pypi.python.org/packages/bb/f0/e7d5e984cf6592fd2807dc7bc44a93f9d18e04e6a61f87fdfb2622422d74/infrae.cache-1.0.1.tar.gz";
       md5 = "b09076a766747e6ed2a755cc62088e32";
     };
   };
   invoke = super.buildPythonPackage {
     name = "invoke-0.11.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/d3/bb/36a5558ea19882073def7b0edeef4a0e6282056fed96506dd10b1d532bd4/invoke-0.11.1.tar.gz";
       md5 = "3d4ecbe26779ceef1046ecf702c9c4a8";
     };
   };
   ipdb = super.buildPythonPackage {
     name = "ipdb-0.8";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [ipython];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f0/25/d7dd430ced6cd8dc242a933c8682b5dbf32eb4011d82f87e34209e5ec845/ipdb-0.8.zip";
       md5 = "96dca0712efa01aa5eaf6b22071dd3ed";
     };
   };
   ipython = super.buildPythonPackage {
     name = "ipython-3.1.0";
     buildInputs = with self; [];
     doCheck = false;
-    propagatedBuildInputs = with self; [gnureadline];
+    propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/06/91/120c0835254c120af89f066afaabf81289bc2726c1fc3ca0555df6882f58/ipython-3.1.0.tar.gz";
       md5 = "a749d90c16068687b0ec45a27e72ef8f";
     };
   };
   iso8601 = super.buildPythonPackage {
     name = "iso8601-0.1.11";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c0/75/c9209ee4d1b5975eb8c2cba4428bde6b61bd55664a98290dd015cdb18e98/iso8601-0.1.11.tar.gz";
       md5 = "b06d11cd14a64096f907086044f0fe38";
     };
   };
   itsdangerous = super.buildPythonPackage {
     name = "itsdangerous-0.24";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/dc/b4/a60bcdba945c00f6d608d8975131ab3f25b22f2bcfe1dab221165194b2d4/itsdangerous-0.24.tar.gz";
       md5 = "a3d55aa79369aef5345c036a8a26307f";
     };
   };
   kombu = super.buildPythonPackage {
     name = "kombu-1.5.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [anyjson amqplib];
     src = fetchurl {
       url = "https://pypi.python.org/packages/19/53/74bf2a624644b45f0850a638752514fc10a8e1cbd738f10804951a6df3f5/kombu-1.5.1.tar.gz";
       md5 = "50662f3c7e9395b3d0721fb75d100b63";
     };
   };
   lxml = super.buildPythonPackage {
     name = "lxml-3.4.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/63/c7/4f2a2a4ad6c6fa99b14be6b3c1cece9142e2d915aa7c43c908677afc8fa4/lxml-3.4.4.tar.gz";
       md5 = "a9a65972afc173ec7a39c585f4eea69c";
     };
   };
   mccabe = super.buildPythonPackage {
     name = "mccabe-0.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c9/2e/75231479e11a906b64ac43bad9d0bb534d00080b18bdca8db9da46e1faf7/mccabe-0.3.tar.gz";
       md5 = "81640948ff226f8c12b3277059489157";
     };
   };
   meld3 = super.buildPythonPackage {
     name = "meld3-1.0.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/45/a0/317c6422b26c12fe0161e936fc35f36552069ba8e6f7ecbd99bbffe32a5f/meld3-1.0.2.tar.gz";
       md5 = "3ccc78cd79cffd63a751ad7684c02c91";
     };
   };
   mock = super.buildPythonPackage {
     name = "mock-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/15/45/30273ee91feb60dabb8fbb2da7868520525f02cf910279b3047182feed80/mock-1.0.1.zip";
       md5 = "869f08d003c289a97c1a6610faf5e913";
     };
   };
   msgpack-python = super.buildPythonPackage {
     name = "msgpack-python-0.4.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/15/ce/ff2840885789ef8035f66cd506ea05bdb228340307d5e71a7b1e3f82224c/msgpack-python-0.4.6.tar.gz";
       md5 = "8b317669314cf1bc881716cccdaccb30";
     };
   };
   nose = super.buildPythonPackage {
     name = "nose-1.3.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/70/c7/469e68148d17a0d3db5ed49150242fd70a74a8147b8f3f8b87776e028d99/nose-1.3.6.tar.gz";
       md5 = "0ca546d81ca8309080fc80cb389e7a16";
     };
   };
   objgraph = super.buildPythonPackage {
     name = "objgraph-2.0.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/d7/33/ace750b59247496ed769b170586c5def7202683f3d98e737b75b767ff29e/objgraph-2.0.0.tar.gz";
       md5 = "25b0d5e5adc74aa63ead15699614159c";
     };
   };
   packaging = super.buildPythonPackage {
     name = "packaging-15.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/24/c4/185da1304f07047dc9e0c46c31db75c0351bd73458ac3efad7da3dbcfbe1/packaging-15.2.tar.gz";
       md5 = "c16093476f6ced42128bf610e5db3784";
     };
   };
   paramiko = super.buildPythonPackage {
     name = "paramiko-1.15.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pycrypto ecdsa];
     src = fetchurl {
       url = "https://pypi.python.org/packages/04/2b/a22d2a560c1951abbbf95a0628e245945565f70dc082d9e784666887222c/paramiko-1.15.1.tar.gz";
       md5 = "48c274c3f9b1282932567b21f6acf3b5";
     };
   };
   pep8 = super.buildPythonPackage {
     name = "pep8-1.5.7";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/8b/de/259f5e735897ada1683489dd514b2a1c91aaa74e5e6b68f80acf128a6368/pep8-1.5.7.tar.gz";
       md5 = "f6adbdd69365ecca20513c709f9b7c93";
     };
   };
   psutil = super.buildPythonPackage {
     name = "psutil-2.2.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/df/47/ee54ef14dd40f8ce831a7581001a5096494dc99fe71586260ca6b531fe86/psutil-2.2.1.tar.gz";
       md5 = "1a2b58cd9e3a53528bb6148f0c4d5244";
     };
   };
   psycopg2 = super.buildPythonPackage {
     name = "psycopg2-2.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/dd/c7/9016ff8ff69da269b1848276eebfb264af5badf6b38caad805426771f04d/psycopg2-2.6.tar.gz";
       md5 = "fbbb039a8765d561a1c04969bbae7c74";
     };
   };
   py = super.buildPythonPackage {
     name = "py-1.4.29";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/2a/bc/a1a4a332ac10069b8e5e25136a35e08a03f01fd6ab03d819889d79a1fd65/py-1.4.29.tar.gz";
       md5 = "c28e0accba523a29b35a48bb703fb96c";
     };
   };
   py-bcrypt = super.buildPythonPackage {
     name = "py-bcrypt-0.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/68/b1/1c3068c5c4d2e35c48b38dcc865301ebfdf45f54507086ac65ced1fd3b3d/py-bcrypt-0.4.tar.gz";
       md5 = "dd8b367d6b716a2ea2e72392525f4e36";
     };
   };
   pycrypto = super.buildPythonPackage {
     name = "pycrypto-2.6.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/60/db/645aa9af249f059cc3a368b118de33889219e0362141e75d4eaf6f80f163/pycrypto-2.6.1.tar.gz";
       md5 = "55a61a054aa66812daf5161a0d5d7eda";
     };
   };
   pycurl = super.buildPythonPackage {
     name = "pycurl-7.19.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/6c/48/13bad289ef6f4869b1d8fc11ae54de8cfb3cc4a2eb9f7419c506f763be46/pycurl-7.19.5.tar.gz";
       md5 = "47b4eac84118e2606658122104e62072";
     };
   };
-  pyelasticsearch = super.buildPythonPackage {
-    name = "pyelasticsearch-1.4";
-    buildInputs = with self; [];
-    doCheck = false;
-    propagatedBuildInputs = with self; [certifi elasticsearch urllib3 simplejson six];
-    src = fetchurl {
-      url = "https://pypi.python.org/packages/2f/3a/7643cfcfc4cbdbb20ada800bbd54ac9705d0c047d7b8f8d5eeeb3047b4eb/pyelasticsearch-1.4.tar.gz";
-      md5 = "ed61ebb7b253364e55b4923d11e17049";
-    };
-  };
   pyflakes = super.buildPythonPackage {
     name = "pyflakes-0.8.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/75/22/a90ec0252f4f87f3ffb6336504de71fe16a49d69c4538dae2f12b9360a38/pyflakes-0.8.1.tar.gz";
       md5 = "905fe91ad14b912807e8fdc2ac2e2c23";
     };
   };
   pyparsing = super.buildPythonPackage {
     name = "pyparsing-1.5.7";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/2e/26/e8fb5b4256a5f5036be7ce115ef8db8d06bc537becfbdc46c6af008314ee/pyparsing-1.5.7.zip";
       md5 = "b86854857a368d6ccb4d5b6e76d0637f";
     };
   };
   pyramid = super.buildPythonPackage {
     name = "pyramid-1.6.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools WebOb repoze.lru zope.interface zope.deprecation venusian translationstring PasteDeploy];
     src = fetchurl {
       url = "https://pypi.python.org/packages/30/b3/fcc4a2a4800cbf21989e00454b5828cf1f7fe35c63e0810b350e56d4c475/pyramid-1.6.1.tar.gz";
       md5 = "b18688ff3cc33efdbb098a35b45dd122";
     };
   };
   pyramid-beaker = super.buildPythonPackage {
     name = "pyramid-beaker-0.8";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pyramid Beaker];
     src = fetchurl {
       url = "https://pypi.python.org/packages/d9/6e/b85426e00fd3d57f4545f74e1c3828552d8700f13ededeef9233f7bca8be/pyramid_beaker-0.8.tar.gz";
       md5 = "22f14be31b06549f80890e2c63a93834";
     };
   };
   pyramid-debugtoolbar = super.buildPythonPackage {
     name = "pyramid-debugtoolbar-2.4.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pyramid pyramid-mako repoze.lru Pygments];
     src = fetchurl {
       url = "https://pypi.python.org/packages/89/00/ed5426ee41ed747ba3ffd30e8230841a6878286ea67d480b1444d24f06a2/pyramid_debugtoolbar-2.4.2.tar.gz";
       md5 = "073ea67086cc4bd5decc3a000853642d";
     };
   };
   pyramid-jinja2 = super.buildPythonPackage {
     name = "pyramid-jinja2-2.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pyramid zope.deprecation Jinja2 MarkupSafe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/a1/80/595e26ffab7deba7208676b6936b7e5a721875710f982e59899013cae1ed/pyramid_jinja2-2.5.tar.gz";
       md5 = "07cb6547204ac5e6f0b22a954ccee928";
     };
   };
   pyramid-mako = super.buildPythonPackage {
     name = "pyramid-mako-1.0.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pyramid Mako];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f1/92/7e69bcf09676d286a71cb3bbb887b16595b96f9ba7adbdc239ffdd4b1eb9/pyramid_mako-1.0.2.tar.gz";
       md5 = "ee25343a97eb76bd90abdc2a774eb48a";
     };
   };
   pysqlite = super.buildPythonPackage {
     name = "pysqlite-2.6.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/5c/a6/1c429cd4c8069cf4bfbd0eb4d592b3f4042155a8202df83d7e9b93aa3dc2/pysqlite-2.6.3.tar.gz";
       md5 = "7ff1cedee74646b50117acff87aa1cfa";
     };
   };
   pytest = super.buildPythonPackage {
     name = "pytest-2.8.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [py];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b1/3d/d7ea9b0c51e0cacded856e49859f0a13452747491e842c236bbab3714afe/pytest-2.8.5.zip";
       md5 = "8493b06f700862f1294298d6c1b715a9";
     };
   };
   pytest-catchlog = super.buildPythonPackage {
     name = "pytest-catchlog-1.2.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [py pytest];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f2/2b/2faccdb1a978fab9dd0bf31cca9f6847fbe9184a0bdcc3011ac41dd44191/pytest-catchlog-1.2.2.zip";
       md5 = "09d890c54c7456c818102b7ff8c182c8";
     };
   };
   pytest-cov = super.buildPythonPackage {
     name = "pytest-cov-1.8.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [py pytest coverage cov-core];
     src = fetchurl {
       url = "https://pypi.python.org/packages/11/4b/b04646e97f1721878eb21e9f779102d84dd044d324382263b1770a3e4838/pytest-cov-1.8.1.tar.gz";
       md5 = "76c778afa2494088270348be42d759fc";
     };
   };
   pytest-profiling = super.buildPythonPackage {
     name = "pytest-profiling-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six pytest gprof2dot];
     src = fetchurl {
       url = "https://pypi.python.org/packages/d8/67/8ffab73406e22870e07fa4dc8dce1d7689b26dba8efd00161c9b6fc01ec0/pytest-profiling-1.0.1.tar.gz";
       md5 = "354404eb5b3fd4dc5eb7fffbb3d9b68b";
     };
   };
   pytest-runner = super.buildPythonPackage {
     name = "pytest-runner-2.7.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/99/6b/c4ff4418d3424d4475b7af60724fd4a5cdd91ed8e489dc9443281f0052bc/pytest-runner-2.7.1.tar.gz";
       md5 = "e56f0bc8d79a6bd91772b44ef4215c7e";
     };
   };
   pytest-timeout = super.buildPythonPackage {
     name = "pytest-timeout-0.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pytest];
     src = fetchurl {
       url = "https://pypi.python.org/packages/24/48/5f6bd4b8026a26e1dd427243d560a29a0f1b24a5c7cffca4bf049a7bb65b/pytest-timeout-0.4.tar.gz";
       md5 = "03b28aff69cbbfb959ed35ade5fde262";
     };
   };
   python-dateutil = super.buildPythonPackage {
     name = "python-dateutil-1.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b4/7c/df59c89a753eb33c7c44e1dd42de0e9bc2ccdd5a4d576e0bfad97cc280cb/python-dateutil-1.5.tar.gz";
       md5 = "0dcb1de5e5cad69490a3b6ab63f0cfa5";
     };
   };
   python-editor = super.buildPythonPackage {
     name = "python-editor-1.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f5/d9/01eb441489c8bd2adb33ee4f3aea299a3db531a584cb39c57a0ecf516d9c/python-editor-1.0.tar.gz";
       md5 = "a5ead611360b17b52507297d8590b4e8";
     };
   };
   python-ldap = super.buildPythonPackage {
     name = "python-ldap-2.4.19";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/42/81/1b64838c82e64f14d4e246ff00b52e650a35c012551b891ada2b85d40737/python-ldap-2.4.19.tar.gz";
       md5 = "b941bf31d09739492aa19ef679e94ae3";
     };
   };
   python-memcached = super.buildPythonPackage {
     name = "python-memcached-1.57";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six];
     src = fetchurl {
       url = "https://pypi.python.org/packages/52/9d/eebc0dcbc5c7c66840ad207dfc1baa376dadb74912484bff73819cce01e6/python-memcached-1.57.tar.gz";
       md5 = "de21f64b42b2d961f3d4ad7beb5468a1";
     };
   };
   python-pam = super.buildPythonPackage {
     name = "python-pam-1.8.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/de/8c/f8f5d38b4f26893af267ea0b39023d4951705ab0413a39e0cf7cf4900505/python-pam-1.8.2.tar.gz";
       md5 = "db71b6b999246fb05d78ecfbe166629d";
     };
   };
   pytz = super.buildPythonPackage {
     name = "pytz-2015.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/7e/1a/f43b5c92df7b156822030fed151327ea096bcf417e45acc23bd1df43472f/pytz-2015.4.zip";
       md5 = "233f2a2b370d03f9b5911700cc9ebf3c";
     };
   };
   pyzmq = super.buildPythonPackage {
     name = "pyzmq-14.6.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/8a/3b/5463d5a9d712cd8bbdac335daece0d69f6a6792da4e3dd89956c0db4e4e6/pyzmq-14.6.0.tar.gz";
       md5 = "395b5de95a931afa5b14c9349a5b8024";
     };
   };
   recaptcha-client = super.buildPythonPackage {
     name = "recaptcha-client-1.0.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/0a/ea/5f2fbbfd894bdac1c68ef8d92019066cfcf9fbff5fe3d728d2b5c25c8db4/recaptcha-client-1.0.6.tar.gz";
       md5 = "74228180f7e1fb76c4d7089160b0d919";
     };
   };
   repoze.lru = super.buildPythonPackage {
     name = "repoze.lru-0.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/6e/1e/aa15cc90217e086dc8769872c8778b409812ff036bf021b15795638939e4/repoze.lru-0.6.tar.gz";
       md5 = "2c3b64b17a8e18b405f55d46173e14dd";
     };
   };
   requests = super.buildPythonPackage {
     name = "requests-2.9.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f9/6d/07c44fb1ebe04d069459a189e7dab9e4abfe9432adcd4477367c25332748/requests-2.9.1.tar.gz";
       md5 = "0b7f480d19012ec52bab78292efd976d";
     };
   };
   rhodecode-enterprise-ce = super.buildPythonPackage {
-    name = "rhodecode-enterprise-ce-4.0.1";
+    name = "rhodecode-enterprise-ce-4.1.0";
     buildInputs = with self; [WebTest configobj cssselect flake8 lxml mock pytest pytest-cov pytest-runner];
     doCheck = true;
     propagatedBuildInputs = with self; [Babel Beaker FormEncode Mako Markdown MarkupSafe MySQL-python Paste PasteDeploy PasteScript Pygments Pylons Pyro4 Routes SQLAlchemy Tempita URLObject WebError WebHelpers WebHelpers2 WebOb WebTest Whoosh alembic amqplib anyjson appenlight-client authomatic backport-ipaddress celery colander decorator docutils gunicorn infrae.cache ipython iso8601 kombu msgpack-python packaging psycopg2 pycrypto pycurl pyparsing pyramid pyramid-debugtoolbar pyramid-mako pyramid-beaker pysqlite python-dateutil python-ldap python-memcached python-pam recaptcha-client repoze.lru requests simplejson waitress zope.cachedescriptors psutil py-bcrypt];
     src = ./.;
   };
   rhodecode-tools = super.buildPythonPackage {
-    name = "rhodecode-tools-0.7.1";
+    name = "rhodecode-tools-0.8.3";
     buildInputs = with self; [];
     doCheck = false;
-    propagatedBuildInputs = with self; [click future six Mako MarkupSafe requests Whoosh pyelasticsearch];
+    propagatedBuildInputs = with self; [click future six Mako MarkupSafe requests Whoosh elasticsearch elasticsearch-dsl];
     src = fetchurl {
-      url = "https://code.rhodecode.com/rhodecode-tools-ce/archive/v0.7.1.zip";
-      md5 = "91daea803aaa264ce7a8213bc2220d4c";
+      url = "https://code.rhodecode.com/rhodecode-tools-ce/archive/v0.8.3.zip";
+      md5 = "9acdfd71b8ddf4056057065f37ab9ccb";
     };
   };
   serpent = super.buildPythonPackage {
     name = "serpent-1.12";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/3b/19/1e0e83b47c09edaef8398655088036e7e67386b5c48770218ebb339fbbd5/serpent-1.12.tar.gz";
       md5 = "05869ac7b062828b34f8f927f0457b65";
     };
   };
   setproctitle = super.buildPythonPackage {
     name = "setproctitle-1.1.8";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/33/c3/ad367a4f4f1ca90468863ae727ac62f6edb558fc09a003d344a02cfc6ea6/setproctitle-1.1.8.tar.gz";
       md5 = "728f4c8c6031bbe56083a48594027edd";
     };
   };
   setuptools = super.buildPythonPackage {
     name = "setuptools-20.8.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c4/19/c1bdc88b53da654df43770f941079dbab4e4788c2dcb5658fb86259894c7/setuptools-20.8.1.zip";
       md5 = "fe58a5cac0df20bb83942b252a4b0543";
     };
   };
   setuptools-scm = super.buildPythonPackage {
     name = "setuptools-scm-1.11.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/cd/5f/e3a038292358058d83d764a47d09114aa5a8003ed4529518f9e580f1a94f/setuptools_scm-1.11.0.tar.gz";
       md5 = "4c5c896ba52e134bbc3507bac6400087";
     };
   };
   simplejson = super.buildPythonPackage {
     name = "simplejson-3.7.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/6d/89/7f13f099344eea9d6722779a1f165087cb559598107844b1ac5dbd831fb1/simplejson-3.7.2.tar.gz";
       md5 = "a5fc7d05d4cb38492285553def5d4b46";
     };
   };
   six = super.buildPythonPackage {
     name = "six-1.9.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/16/64/1dc5e5976b17466fd7d712e59cbe9fb1e18bec153109e5ba3ed6c9102f1a/six-1.9.0.tar.gz";
       md5 = "476881ef4012262dfc8adc645ee786c4";
     };
   };
   subprocess32 = super.buildPythonPackage {
     name = "subprocess32-3.2.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/28/8d/33ccbff51053f59ae6c357310cac0e79246bbed1d345ecc6188b176d72c3/subprocess32-3.2.6.tar.gz";
       md5 = "754c5ab9f533e764f931136974b618f1";
     };
   };
   supervisor = super.buildPythonPackage {
     name = "supervisor-3.1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [meld3];
     src = fetchurl {
       url = "https://pypi.python.org/packages/a6/41/65ad5bd66230b173eb4d0b8810230f3a9c59ef52ae066e540b6b99895db7/supervisor-3.1.3.tar.gz";
       md5 = "aad263c4fbc070de63dd354864d5e552";
     };
   };
   transifex-client = super.buildPythonPackage {
     name = "transifex-client-0.10";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f3/4e/7b925192aee656fb3e04fa6381c8b3dc40198047c3b4a356f6cfd642c809/transifex-client-0.10.tar.gz";
       md5 = "5549538d84b8eede6b254cd81ae024fa";
     };
   };
   translationstring = super.buildPythonPackage {
     name = "translationstring-1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/5e/eb/bee578cc150b44c653b63f5ebe258b5d0d812ddac12497e5f80fcad5d0b4/translationstring-1.3.tar.gz";
       md5 = "a4b62e0f3c189c783a1685b3027f7c90";
     };
   };
   trollius = super.buildPythonPackage {
     name = "trollius-1.0.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [futures];
     src = fetchurl {
       url = "https://pypi.python.org/packages/aa/e6/4141db437f55e6ee7a3fb69663239e3fde7841a811b4bef293145ad6c836/trollius-1.0.4.tar.gz";
       md5 = "3631a464d49d0cbfd30ab2918ef2b783";
     };
   };
   uWSGI = super.buildPythonPackage {
     name = "uWSGI-2.0.11.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/9b/78/918db0cfab0546afa580c1e565209c49aaf1476bbfe491314eadbe47c556/uwsgi-2.0.11.2.tar.gz";
       md5 = "1f02dcbee7f6f61de4b1fd68350cf16f";
     };
   };
   urllib3 = super.buildPythonPackage {
     name = "urllib3-1.15.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/49/26/a7d12ea00cb4b9fa1e13b5980e5a04a1fe7c477eb8f657ce0b757a7a497d/urllib3-1.15.1.tar.gz";
       md5 = "5be254b0dbb55d1307ede99e1895c8dd";
     };
   };
   venusian = super.buildPythonPackage {
     name = "venusian-1.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/86/20/1948e0dfc4930ddde3da8c33612f6a5717c0b4bc28f591a5c5cf014dd390/venusian-1.0.tar.gz";
       md5 = "dccf2eafb7113759d60c86faf5538756";
     };
   };
   waitress = super.buildPythonPackage {
     name = "waitress-0.8.9";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/ee/65/fc9dee74a909a1187ca51e4f15ad9c4d35476e4ab5813f73421505c48053/waitress-0.8.9.tar.gz";
       md5 = "da3f2e62b3676be5dd630703a68e2a04";
     };
   };
   wsgiref = super.buildPythonPackage {
     name = "wsgiref-0.1.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/41/9e/309259ce8dff8c596e8c26df86dbc4e848b9249fd36797fd60be456f03fc/wsgiref-0.1.2.zip";
       md5 = "29b146e6ebd0f9fb119fe321f7bcf6cb";
     };
   };
   zope.cachedescriptors = super.buildPythonPackage {
     name = "zope.cachedescriptors-4.0.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/40/33/694b6644c37f28553f4b9f20b3c3a20fb709a22574dff20b5bdffb09ecd5/zope.cachedescriptors-4.0.0.tar.gz";
       md5 = "8d308de8c936792c8e758058fcb7d0f0";
     };
   };
   zope.deprecation = super.buildPythonPackage {
     name = "zope.deprecation-4.1.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c1/d3/3919492d5e57d8dd01b36f30b34fc8404a30577392b1eb817c303499ad20/zope.deprecation-4.1.2.tar.gz";
       md5 = "e9a663ded58f4f9f7881beb56cae2782";
     };
   };
   zope.event = super.buildPythonPackage {
     name = "zope.event-4.0.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c1/29/91ba884d7d6d96691df592e9e9c2bfa57a47040ec1ff47eff18c85137152/zope.event-4.0.3.tar.gz";
       md5 = "9a3780916332b18b8b85f522bcc3e249";
     };
   };
   zope.interface = super.buildPythonPackage {
     name = "zope.interface-4.1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/9d/81/2509ca3c6f59080123c1a8a97125eb48414022618cec0e64eb1313727bfe/zope.interface-4.1.3.tar.gz";
       md5 = "9ae3d24c0c7415deb249dd1a132f0f79";
     };
   };
 
 ### Test requirements
 
   
 }

requirements.txt

@@ -1,151 +1,151 b''
 Babel==1.3
 Beaker==1.7.0
 CProfileV==1.0.6
 Fabric==1.10.0
 FormEncode==1.2.4
 Jinja2==2.7.3
 Mako==1.0.1
 Markdown==2.6.2
 MarkupSafe==0.23
 MySQL-python==1.2.5
 Paste==2.0.2
 PasteDeploy==1.5.2
 PasteScript==1.7.5
-pyelasticsearch==1.4
 Pygments==2.0.2
 
 # TODO: This version is not available on PyPI
 # Pylons==1.0.2.dev20160108
 Pylons==1.0.1
 
 # TODO: This version is not available, but newer ones are
 # Pyro4==4.35
 Pyro4==4.41
 
 # TODO: This should probably not be in here
 # -e hg+https://[e-mail]/johbo/rhodecode-fork@3a454bd1f17c0b2b2a951cf2b111e0320d7942a9#egg=RhodeCodeEnterprise-dev
 
 # TODO: This is not really a dependency, we should add it only
 # into the development environment, since there it is useful.
 # RhodeCodeVCSServer==3.9.0
 
 Routes==1.13
 SQLAlchemy==0.9.9
 Sphinx==1.2.2
 Tempita==0.5.2
 URLObject==2.4.0
 WebError==0.10.3
 
 # TODO: This is modified by us, needs a better integration. For now
 # using the latest version before.
 # WebHelpers==1.3.dev20150807
 WebHelpers==1.3
 
 WebHelpers2==2.0
 WebOb==1.3.1
 WebTest==1.4.3
 Whoosh==2.7.0
 alembic==0.8.4
 amqplib==1.0.2
 anyjson==0.3.3
 appenlight-client==0.6.14
 authomatic==0.1.0.post1;
 backport-ipaddress==0.1
 bottle==0.12.8
 bumpversion==0.5.3
 celery==2.2.10
 click==5.1
 colander==1.2
 configobj==5.0.6
 cov-core==1.15.0
 coverage==3.7.1
 cssselect==0.9.1
 decorator==3.4.2
 docutils==0.12
 dogpile.cache==0.5.7
 dogpile.core==0.4.1
 dulwich==0.12.0
 ecdsa==0.11
 flake8==2.4.1
 future==0.14.3
 futures==3.0.2
 gprof2dot==2015.12.1
-greenlet==0.4.7
+greenlet==0.4.9
 gunicorn==19.6.0
 
 # TODO: Needs subvertpy and blows up without Subversion headers,
 # actually we should not need this for Enterprise at all.
 # hgsubversion==1.8.2
 
+gnureadline==6.3.3
 infrae.cache==1.0.1
 invoke==0.11.1
 ipdb==0.8
 ipython==3.1.0
 iso8601==0.1.11
 itsdangerous==0.24
 kombu==1.5.1
 lxml==3.4.4
 mccabe==0.3
 meld3==1.0.2
 mock==1.0.1
 msgpack-python==0.4.6
 nose==1.3.6
 objgraph==2.0.0
 packaging==15.2
 paramiko==1.15.1
 pep8==1.5.7
 psutil==2.2.1
 psycopg2==2.6
 py==1.4.29
 py-bcrypt==0.4
 pycrypto==2.6.1
 pycurl==7.19.5
 pyflakes==0.8.1
 pyparsing==1.5.7
 pyramid==1.6.1
 pyramid-beaker==0.8
 pyramid-debugtoolbar==2.4.2
 pyramid-jinja2==2.5
 pyramid-mako==1.0.2
 pysqlite==2.6.3
 pytest==2.8.5
 pytest-runner==2.7.1
 pytest-catchlog==1.2.2
 pytest-cov==1.8.1
 pytest-profiling==1.0.1
 pytest-timeout==0.4
 python-dateutil==1.5
 python-ldap==2.4.19
 python-memcached==1.57
 python-pam==1.8.2
 pytz==2015.4
 pyzmq==14.6.0
 
 # TODO: This is not available in public
 # rc-testdata==0.2.0
 
-https://code.rhodecode.com/rhodecode-tools-ce/archive/v0.7.1.zip#md5=91daea803aaa264ce7a8213bc2220d4c
+https://code.rhodecode.com/rhodecode-tools-ce/archive/v0.8.3.zip#md5=9acdfd71b8ddf4056057065f37ab9ccb
 
 
 recaptcha-client==1.0.6
 repoze.lru==0.6
 requests==2.9.1
 serpent==1.12
 setproctitle==1.1.8
 setuptools==20.8.1
 setuptools-scm==1.11.0
 simplejson==3.7.2
 six==1.9.0
 subprocess32==3.2.6
 supervisor==3.1.3
 transifex-client==0.10
 translationstring==1.3
 trollius==1.0.4
 uWSGI==2.0.11.2
 venusian==1.0
 waitress==0.8.9
 wsgiref==0.1.2
 zope.cachedescriptors==4.0.0
 zope.deprecation==4.1.2
 zope.event==4.0.3
 zope.interface==4.1.3

rhodecode/VERSION

@@ -1,1 +1,1 b''
-4.0.1
  No newline at end of file
+4.1.0
  No newline at end of file

rhodecode/__init__.py

@@ -1,58 +1,58 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 
 RhodeCode, a web based repository management software
 versioning implementation: http://www.python.org/dev/peps/pep-0386/
 """
 
 import os
 import sys
 import platform
 
 VERSION = tuple(open(os.path.join(
     os.path.dirname(__file__), 'VERSION')).read().split('.'))
 
 BACKENDS = {
     'hg': 'Mercurial repository',
     'git': 'Git repository',
     'svn': 'Subversion repository',
 }
 
 CELERY_ENABLED = False
 CELERY_EAGER = False
 
 # link to config for pylons
 CONFIG = {}
 
 # Linked module for extensions
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 51  # defines current db version for migrations
+__dbversion__ = 54  # defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'
 __url__ = 'http://rhodecode.com'
 
 is_windows = __platform__ in ['Windows']
 is_unix = not is_windows
 is_test = False

rhodecode/api/tests/test_update_repo.py

@@ -1,144 +1,158 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import mock
 import pytest
 
 from rhodecode.model.repo import RepoModel
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_LOGIN
 from rhodecode.api.tests.utils import (
-    build_data, api_call, assert_error, assert_ok, crash)
+    build_data, api_call, assert_error, assert_ok, crash, jsonify)
 from rhodecode.tests.fixture import Fixture
 
 
 fixture = Fixture()
 
+UPDATE_REPO_NAME = 'api_update_me'
+
+class SAME_AS_UPDATES(object): """ Constant used for tests below """
 
 @pytest.mark.usefixtures("testuser_api", "app")
 class TestApiUpdateRepo(object):
-    @pytest.mark.parametrize("changing_attr, updates", [
-        ('owner', {'owner': TEST_USER_REGULAR_LOGIN}),
-        ('description', {'description': 'new description'}),
-        ('active', {'active': True}),
-        ('active', {'active': False}),
-        ('clone_uri', {'clone_uri': 'http://foo.com/repo'}),
-        ('clone_uri', {'clone_uri': None}),
-        ('landing_rev', {'landing_rev': 'branch:master'}),
-        ('enable_statistics', {'enable_statistics': True}),
-        ('enable_locking', {'enable_locking': True}),
-        ('enable_downloads', {'enable_downloads': True}),
-        ('name', {'name': 'new_repo_name'}),
-        ('repo_group', {'group': 'test_group_for_update'}),
+
+    @pytest.mark.parametrize("updates, expected", [
+        ({'owner': TEST_USER_REGULAR_LOGIN}, SAME_AS_UPDATES),
+        ({'description': 'new description'}, SAME_AS_UPDATES),
+        ({'clone_uri': 'http://foo.com/repo'}, SAME_AS_UPDATES),
+        ({'clone_uri': None}, {'clone_uri': ''}),
+        ({'clone_uri': ''}, {'clone_uri': ''}),
+        ({'landing_rev': 'branch:master'}, {'landing_rev': ['branch','master']}),
+        ({'enable_statistics': True}, SAME_AS_UPDATES),
+        ({'enable_locking': True}, SAME_AS_UPDATES),
+        ({'enable_downloads': True}, SAME_AS_UPDATES),
+        ({'name': 'new_repo_name'}, {'repo_name': 'new_repo_name'}),
+        ({'group': 'test_group_for_update'},
+            {'repo_name': 'test_group_for_update/%s' % UPDATE_REPO_NAME}),
     ])
-    def test_api_update_repo(self, changing_attr, updates, backend):
-        repo_name = 'api_update_me'
+    def test_api_update_repo(self, updates, expected, backend):
+        repo_name = UPDATE_REPO_NAME
         repo = fixture.create_repo(repo_name, repo_type=backend.alias)
-        if changing_attr == 'repo_group':
+        if updates.get('group'):
             fixture.create_repo_group(updates['group'])
 
+        expected_api_data = repo.get_api_data(include_secrets=True)
+        if expected is SAME_AS_UPDATES:
+            expected_api_data.update(updates)
+        else:
+            expected_api_data.update(expected)
+
+
         id_, params = build_data(
             self.apikey, 'update_repo', repoid=repo_name, **updates)
         response = api_call(self.app, params)
-        if changing_attr == 'name':
+
+        if updates.get('name'):
             repo_name = updates['name']
-        if changing_attr == 'repo_group':
+        if updates.get('group'):
             repo_name = '/'.join([updates['group'], repo_name])
+
         try:
             expected = {
                 'msg': 'updated repo ID:%s %s' % (repo.repo_id, repo_name),
-                'repository': repo.get_api_data(include_secrets=True)
+                'repository': jsonify(expected_api_data)
             }
             assert_ok(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
-            if changing_attr == 'repo_group':
-
+            if updates.get('group'):
                 fixture.destroy_repo_group(updates['group'])
 
     def test_api_update_repo_fork_of_field(self, backend):
         master_repo = backend.create_repo()
         repo = backend.create_repo()
-
         updates = {
             'fork_of': master_repo.repo_name
         }
+        expected_api_data = repo.get_api_data(include_secrets=True)
+        expected_api_data.update(updates)
+
         id_, params = build_data(
             self.apikey, 'update_repo', repoid=repo.repo_name, **updates)
         response = api_call(self.app, params)
         expected = {
             'msg': 'updated repo ID:%s %s' % (repo.repo_id, repo.repo_name),
-            'repository': repo.get_api_data(include_secrets=True)
+            'repository': jsonify(expected_api_data)
         }
         assert_ok(id_, expected, given=response.body)
         result = response.json['result']['repository']
         assert result['fork_of'] == master_repo.repo_name
 
     def test_api_update_repo_fork_of_not_found(self, backend):
         master_repo_name = 'fake-parent-repo'
         repo = backend.create_repo()
         updates = {
             'fork_of': master_repo_name
         }
         id_, params = build_data(
             self.apikey, 'update_repo', repoid=repo.repo_name, **updates)
         response = api_call(self.app, params)
         expected = 'repository `{}` does not exist'.format(master_repo_name)
         assert_error(id_, expected, given=response.body)
 
     def test_api_update_repo_with_repo_group_not_existing(self):
         repo_name = 'admin_owned'
         fixture.create_repo(repo_name)
         updates = {'group': 'test_group_for_update'}
         id_, params = build_data(
             self.apikey, 'update_repo', repoid=repo_name, **updates)
         response = api_call(self.app, params)
         try:
             expected = 'repository group `%s` does not exist' % (
                 updates['group'],)
             assert_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
 
     def test_api_update_repo_regular_user_not_allowed(self):
         repo_name = 'admin_owned'
         fixture.create_repo(repo_name)
         updates = {'active': False}
         id_, params = build_data(
             self.apikey_regular, 'update_repo', repoid=repo_name, **updates)
         response = api_call(self.app, params)
         try:
             expected = 'repository `%s` does not exist' % (repo_name,)
             assert_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)
 
     @mock.patch.object(RepoModel, 'update', crash)
     def test_api_update_repo_exception_occurred(self, backend):
-        repo_name = 'api_update_me'
+        repo_name = UPDATE_REPO_NAME
         fixture.create_repo(repo_name, repo_type=backend.alias)
         id_, params = build_data(
             self.apikey, 'update_repo', repoid=repo_name,
             owner=TEST_USER_ADMIN_LOGIN,)
         response = api_call(self.app, params)
         try:
             expected = 'failed to update repo `%s`' % (repo_name,)
             assert_error(id_, expected, given=response.body)
         finally:
             fixture.destroy_repo(repo_name)

rhodecode/api/tests/test_update_user_group.py

@@ -1,100 +1,108 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import mock
 import pytest
 
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.tests import TEST_USER_REGULAR_LOGIN
 from rhodecode.api.tests.utils import (
-    build_data, api_call, assert_error, assert_ok, crash)
+    build_data, api_call, assert_error, assert_ok, crash, jsonify)
 
 
 @pytest.mark.usefixtures("testuser_api", "app")
 class TestUpdateUserGroup(object):
     @pytest.mark.parametrize("changing_attr, updates", [
         ('group_name', {'group_name': 'new_group_name'}),
         ('group_name', {'group_name': 'test_group_for_update'}),
         ('owner', {'owner': TEST_USER_REGULAR_LOGIN}),
         ('active', {'active': False}),
         ('active', {'active': True})
     ])
     def test_api_update_user_group(self, changing_attr, updates, user_util):
         user_group = user_util.create_user_group()
         group_name = user_group.users_group_name
+        expected_api_data = user_group.get_api_data()
+        expected_api_data.update(updates)
+
         id_, params = build_data(
             self.apikey, 'update_user_group', usergroupid=group_name,
             **updates)
         response = api_call(self.app, params)
+
         expected = {
             'msg': 'updated user group ID:%s %s' % (
                 user_group.users_group_id, user_group.users_group_name),
-            'user_group': user_group.get_api_data()
+            'user_group': jsonify(expected_api_data)
         }
         assert_ok(id_, expected, given=response.body)
 
     @pytest.mark.parametrize("changing_attr, updates", [
         # TODO: mikhail: decide if we need to test against the commented params
         # ('group_name', {'group_name': 'new_group_name'}),
         # ('group_name', {'group_name': 'test_group_for_update'}),
         ('owner', {'owner': TEST_USER_REGULAR_LOGIN}),
         ('active', {'active': False}),
         ('active', {'active': True})
     ])
     def test_api_update_user_group_regular_user(
             self, changing_attr, updates, user_util):
         user_group = user_util.create_user_group()
         group_name = user_group.users_group_name
+        expected_api_data = user_group.get_api_data()
+        expected_api_data.update(updates)
+
+
         # grant permission to this user
         user = UserModel().get_by_username(self.TEST_USER_LOGIN)
 
         user_util.grant_user_permission_to_user_group(
             user_group, user, 'usergroup.admin')
         id_, params = build_data(
             self.apikey_regular, 'update_user_group',
             usergroupid=group_name, **updates)
         response = api_call(self.app, params)
         expected = {
             'msg': 'updated user group ID:%s %s' % (
                 user_group.users_group_id, user_group.users_group_name),
-            'user_group': user_group.get_api_data()
+            'user_group': jsonify(expected_api_data)
         }
         assert_ok(id_, expected, given=response.body)
 
     def test_api_update_user_group_regular_user_no_permission(self, user_util):
         user_group = user_util.create_user_group()
         group_name = user_group.users_group_name
         id_, params = build_data(
             self.apikey_regular, 'update_user_group', usergroupid=group_name)
         response = api_call(self.app, params)
 
         expected = 'user group `%s` does not exist' % (group_name)
         assert_error(id_, expected, given=response.body)
 
     @mock.patch.object(UserGroupModel, 'update', crash)
     def test_api_update_user_group_exception_occurred(self, user_util):
         user_group = user_util.create_user_group()
         group_name = user_group.users_group_name
         id_, params = build_data(
             self.apikey, 'update_user_group', usergroupid=group_name)
         response = api_call(self.app, params)
         expected = 'failed to update user group `%s`' % (group_name,)
         assert_error(id_, expected, given=response.body)

rhodecode/api/views/repo_api.py

@@ -1,1774 +1,1777 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import time
 
 import colander
 
 from rhodecode import BACKENDS
 from rhodecode.api import jsonrpc_method, JSONRPCError, JSONRPCForbidden, json
 from rhodecode.api.utils import (
     has_superadmin_permission, Optional, OAttr, get_repo_or_error,
     get_user_group_or_error, get_user_or_error, has_repo_permissions,
     get_perm_or_error, store_update, get_repo_group_or_error, parse_args,
     get_origin, build_commit_data)
 from rhodecode.lib.auth import (
     HasPermissionAnyApi, HasRepoGroupPermissionAnyApi,
     HasUserGroupPermissionAnyApi)
 from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError
 from rhodecode.lib.utils import map_groups
 from rhodecode.lib.utils2 import str2bool, time_to_datetime
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import ChangesetCommentsModel
 from rhodecode.model.db import (
     Session, ChangesetStatus, RepositoryField, Repository)
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.scm import ScmModel, RepoList
 from rhodecode.model.settings import SettingsModel
 from rhodecode.model.validation_schema import RepoSchema
 
 log = logging.getLogger(__name__)
 
 
 @jsonrpc_method()
 def get_repo(request, apiuser, repoid, cache=Optional(True)):
     """
     Gets an existing repository by its name or repository_id.
 
     The members section so the output returns users groups or users
     associated with that repository.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository id.
     :type repoid: str or int
     :param cache: use the cached value for last changeset
     :type: cache: Optional(bool)
 
     Example output:
 
     .. code-block:: bash
 
         {
           "error": null,
           "id": <repo_id>,
           "result": {
             "clone_uri": null,
             "created_on": "timestamp",
             "description": "repo description",
             "enable_downloads": false,
             "enable_locking": false,
             "enable_statistics": false,
             "followers": [
               {
                 "active": true,
                 "admin": false,
                 "api_key": "****************************************",
                 "api_keys": [
                   "****************************************"
                 ],
                 "email": "[e-mail]",
                 "emails": [
                   "[e-mail]"
                 ],
                 "extern_name": "rhodecode",
                 "extern_type": "rhodecode",
                 "firstname": "username",
                 "ip_addresses": [],
                 "language": null,
                 "last_login": "2015-09-16T17:16:35.854",
                 "lastname": "surname",
                 "user_id": <user_id>,
                 "username": "name"
               }
             ],
             "fork_of": "parent-repo",
             "landing_rev": [
               "rev",
               "tip"
             ],
             "last_changeset": {
               "author": "User <[e-mail]>",
               "branch": "default",
               "date": "timestamp",
               "message": "last commit message",
               "parents": [
                 {
                   "raw_id": "commit-id"
                 }
               ],
               "raw_id": "commit-id",
               "revision": <revision number>,
               "short_id": "short id"
             },
             "lock_reason": null,
             "locked_by": null,
             "locked_date": null,
             "members": [
               {
                 "name": "super-admin-name",
                 "origin": "super-admin",
                 "permission": "repository.admin",
                 "type": "user"
               },
               {
                 "name": "owner-name",
                 "origin": "owner",
                 "permission": "repository.admin",
                 "type": "user"
               },
               {
                 "name": "user-group-name",
                 "origin": "permission",
                 "permission": "repository.write",
                 "type": "user_group"
               }
             ],
             "owner": "owner-name",
             "permissions": [
               {
                 "name": "super-admin-name",
                 "origin": "super-admin",
                 "permission": "repository.admin",
                 "type": "user"
               },
               {
                 "name": "owner-name",
                 "origin": "owner",
                 "permission": "repository.admin",
                 "type": "user"
               },
               {
                 "name": "user-group-name",
                 "origin": "permission",
                 "permission": "repository.write",
                 "type": "user_group"
               }
             ],
             "private": true,
             "repo_id": 676,
             "repo_name": "user-group/repo-name",
             "repo_type": "hg"
           }
         }
     """
 
     repo = get_repo_or_error(repoid)
     cache = Optional.extract(cache)
     include_secrets = False
     if has_superadmin_permission(apiuser):
         include_secrets = True
     else:
         # check if we have at least read permission for this repo !
         _perms = (
             'repository.admin', 'repository.write', 'repository.read',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     permissions = []
     for _user in repo.permissions():
         user_data = {
             'name': _user.username,
             'permission': _user.permission,
             'origin': get_origin(_user),
             'type': "user",
         }
         permissions.append(user_data)
 
     for _user_group in repo.permission_user_groups():
         user_group_data = {
             'name': _user_group.users_group_name,
             'permission': _user_group.permission,
             'origin': get_origin(_user_group),
             'type': "user_group",
         }
         permissions.append(user_group_data)
 
     following_users = [
         user.user.get_api_data(include_secrets=include_secrets)
         for user in repo.followers]
 
     if not cache:
         repo.update_commit_cache()
     data = repo.get_api_data(include_secrets=include_secrets)
     data['members'] = permissions  # TODO: this should be deprecated soon
     data['permissions'] = permissions
     data['followers'] = following_users
     return data
 
 
 @jsonrpc_method()
 def get_repos(request, apiuser):
     """
     Lists all existing repositories.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: [
                   {
                     "repo_id" :          "<repo_id>",
                     "repo_name" :        "<reponame>"
                     "repo_type" :        "<repo_type>",
                     "clone_uri" :        "<clone_uri>",
                     "private": :         "<bool>",
                     "created_on" :       "<datetimecreated>",
                     "description" :      "<description>",
                     "landing_rev":       "<landing_rev>",
                     "owner":             "<repo_owner>",
                     "fork_of":           "<name_of_fork_parent>",
                     "enable_downloads":  "<bool>",
                     "enable_locking":    "<bool>",
                     "enable_statistics": "<bool>",
                   },
                   ...
                 ]
         error:  null
     """
 
     include_secrets = has_superadmin_permission(apiuser)
     _perms = ('repository.read', 'repository.write', 'repository.admin',)
     extras = {'user': apiuser}
 
     repo_list = RepoList(
         RepoModel().get_all(), perm_set=_perms, extra_kwargs=extras)
     return [repo.get_api_data(include_secrets=include_secrets)
             for repo in repo_list]
 
 
 @jsonrpc_method()
 def get_repo_changeset(request, apiuser, repoid, revision,
                        details=Optional('basic')):
     """
     Returns information about a changeset.
 
     Additionally parameters define the amount of details returned by
     this function.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository id
     :type repoid: str or int
     :param revision: revision for which listing should be done
     :type revision: str
     :param details: details can be 'basic|extended|full' full gives diff
         info details like the diff itself, and number of changed files etc.
     :type details: Optional(str)
 
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = (
             'repository.admin', 'repository.write', 'repository.read',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     changes_details = Optional.extract(details)
     _changes_details_types = ['basic', 'extended', 'full']
     if changes_details not in _changes_details_types:
         raise JSONRPCError(
             'ret_type must be one of %s' % (
                 ','.join(_changes_details_types)))
 
     pre_load = ['author', 'branch', 'date', 'message', 'parents',
                 'status', '_commit', '_file_paths']
 
     try:
         cs = repo.get_commit(commit_id=revision, pre_load=pre_load)
     except TypeError as e:
         raise JSONRPCError(e.message)
     _cs_json = cs.__json__()
     _cs_json['diff'] = build_commit_data(cs, changes_details)
     if changes_details == 'full':
         _cs_json['refs'] = {
             'branches': [cs.branch],
             'bookmarks': getattr(cs, 'bookmarks', []),
             'tags': cs.tags
         }
     return _cs_json
 
 
 @jsonrpc_method()
 def get_repo_changesets(request, apiuser, repoid, start_rev, limit,
                         details=Optional('basic')):
     """
-    Returns a set of changesets limited by the number of commits starting
+    Returns a set of commits limited by the number starting
     from the `start_rev` option.
 
     Additional parameters define the amount of details returned by this
     function.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param start_rev: The starting revision from where to get changesets.
     :type start_rev: str
-    :param limit: Limit the number of changesets to this amount
+    :param limit: Limit the number of commits to this amount
     :type limit: str or int
     :param details: Set the level of detail returned. Valid option are:
         ``basic``, ``extended`` and ``full``.
     :type details: Optional(str)
 
     .. note::
 
        Setting the parameter `details` to the value ``full`` is extensive
        and returns details like the diff itself, and the number
        of changed files.
 
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = (
             'repository.admin', 'repository.write', 'repository.read',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     changes_details = Optional.extract(details)
     _changes_details_types = ['basic', 'extended', 'full']
     if changes_details not in _changes_details_types:
         raise JSONRPCError(
             'ret_type must be one of %s' % (
                 ','.join(_changes_details_types)))
 
     limit = int(limit)
     pre_load = ['author', 'branch', 'date', 'message', 'parents',
                 'status', '_commit', '_file_paths']
 
     vcs_repo = repo.scm_instance()
     # SVN needs a special case to distinguish its index and commit id
-    if vcs_repo.alias == 'svn' and (start_rev == '0'):
+    if vcs_repo and vcs_repo.alias == 'svn' and (start_rev == '0'):
         start_rev = vcs_repo.commit_ids[0]
 
     try:
-        commits = repo.scm_instance().get_commits(
+        commits = vcs_repo.get_commits(
             start_id=start_rev, pre_load=pre_load)
     except TypeError as e:
         raise JSONRPCError(e.message)
+    except Exception:
+        log.exception('Fetching of commits failed')
+        raise JSONRPCError('Error occurred during commit fetching')
 
     ret = []
     for cnt, commit in enumerate(commits):
         if cnt >= limit != -1:
             break
         _cs_json = commit.__json__()
         _cs_json['diff'] = build_commit_data(commit, changes_details)
         if changes_details == 'full':
             _cs_json['refs'] = {
                 'branches': [commit.branch],
                 'bookmarks': getattr(commit, 'bookmarks', []),
                 'tags': commit.tags
             }
         ret.append(_cs_json)
     return ret
 
 
 @jsonrpc_method()
 def get_repo_nodes(request, apiuser, repoid, revision, root_path,
                    ret_type=Optional('all'), details=Optional('basic')):
     """
     Returns a list of nodes and children in a flat list for a given
     path at given revision.
 
     It's possible to specify ret_type to show only `files` or `dirs`.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param revision: The revision for which listing should be done.
     :type revision: str
     :param root_path: The path from which to start displaying.
     :type root_path: str
     :param ret_type: Set the return type. Valid options are
         ``all`` (default), ``files`` and ``dirs``.
     :type ret_type: Optional(str)
     :param details: Returns extended information about nodes, such as
         md5, binary, and or content.  The valid options are ``basic`` and
         ``full``.
     :type details: Optional(str)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: [
                   {
                     "name" : "<name>"
                     "type" : "<type>",
                     "binary": "<true|false>" (only in extended mode)
                     "md5"  : "<md5 of file content>" (only in extended mode)
                   },
                   ...
                 ]
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = (
             'repository.admin', 'repository.write', 'repository.read',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     ret_type = Optional.extract(ret_type)
     details = Optional.extract(details)
     _extended_types = ['basic', 'full']
     if details not in _extended_types:
         raise JSONRPCError(
             'ret_type must be one of %s' % (','.join(_extended_types)))
     extended_info = False
     content = False
     if details == 'basic':
         extended_info = True
 
     if details == 'full':
         extended_info = content = True
 
     _map = {}
     try:
         # check if repo is not empty by any chance, skip quicker if it is.
         _scm = repo.scm_instance()
         if _scm.is_empty():
             return []
 
         _d, _f = ScmModel().get_nodes(
             repo, revision, root_path, flat=False,
             extended_info=extended_info, content=content)
         _map = {
             'all': _d + _f,
             'files': _f,
             'dirs': _d,
         }
         return _map[ret_type]
     except KeyError:
         raise JSONRPCError(
             'ret_type must be one of %s' % (','.join(sorted(_map.keys()))))
     except Exception:
         log.exception("Exception occurred while trying to get repo nodes")
         raise JSONRPCError(
             'failed to get repo: `%s` nodes' % repo.repo_name
         )
 
 
 @jsonrpc_method()
 def get_repo_refs(request, apiuser, repoid):
     """
     Returns a dictionary of current references. It returns
     bookmarks, branches, closed_branches, and tags for given repository
 
     It's possible to specify ret_type to show only `files` or `dirs`.
 
     This command can only be run using an |authtoken| with admin rights,
     or users with at least read rights to |repos|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: [
                   TODO...
                 ]
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         # check if repo is not empty by any chance, skip quicker if it is.
         vcs_instance = repo.scm_instance()
         refs = vcs_instance.refs()
         return refs
     except Exception:
         log.exception("Exception occurred while trying to get repo refs")
         raise JSONRPCError(
             'failed to get repo: `%s` references' % repo.repo_name
         )
 
 
 @jsonrpc_method()
 def create_repo(request, apiuser, repo_name, repo_type,
                 owner=Optional(OAttr('apiuser')), description=Optional(''),
                 private=Optional(False), clone_uri=Optional(None),
                 landing_rev=Optional('rev:tip'),
                 enable_statistics=Optional(False),
                 enable_locking=Optional(False),
                 enable_downloads=Optional(False),
                 copy_permissions=Optional(False)):
     """
     Creates a repository.
 
     * If the repository name contains "/", all the required repository
       groups will be created.
 
       For example "foo/bar/baz" will create |repo| groups "foo" and "bar"
       (with "foo" as parent). It will also create the "baz" repository
       with "bar" as |repo| group.
 
     This command can only be run using an |authtoken| with at least
     write permissions to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repo_name: Set the repository name.
     :type repo_name: str
     :param repo_type: Set the repository type; 'hg','git', or 'svn'.
     :type repo_type: str
     :param owner: user_id or username
     :type owner: Optional(str)
     :param description: Set the repository description.
     :type description: Optional(str)
     :param private:
     :type private: bool
     :param clone_uri:
     :type clone_uri: str
     :param landing_rev: <rev_type>:<rev>
     :type landing_rev: str
     :param enable_locking:
     :type enable_locking: bool
     :param enable_downloads:
     :type enable_downloads: bool
     :param enable_statistics:
     :type enable_statistics: bool
     :param copy_permissions: Copy permission from group in which the
         repository is being created.
     :type copy_permissions: bool
 
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg": "Created new repository `<reponame>`",
                   "success": true,
                   "task": "<celery task id or None if done sync>"
                 }
         error:  null
 
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
          'failed to create repository `<repo_name>`
       }
 
     """
     schema = RepoSchema()
     try:
         data = schema.deserialize({
             'repo_name': repo_name
         })
     except colander.Invalid as e:
         raise JSONRPCError("Validation failed: %s" % (e.asdict(),))
     repo_name = data['repo_name']
 
     (repo_name_cleaned,
      parent_group_name) = RepoGroupModel()._get_group_name_and_parent(
         repo_name)
 
     if not HasPermissionAnyApi(
             'hg.admin', 'hg.create.repository')(user=apiuser):
         # check if we have admin permission for this repo group if given !
 
         if parent_group_name:
             repogroupid = parent_group_name
             repo_group = get_repo_group_or_error(parent_group_name)
 
             _perms = ('group.admin',)
             if not HasRepoGroupPermissionAnyApi(*_perms)(
                     user=apiuser, group_name=repo_group.group_name):
                 raise JSONRPCError(
                     'repository group `%s` does not exist' % (
                         repogroupid,))
         else:
             raise JSONRPCForbidden()
 
     if not has_superadmin_permission(apiuser):
         if not isinstance(owner, Optional):
             # forbid setting owner for non-admins
             raise JSONRPCError(
                 'Only RhodeCode admin can specify `owner` param')
 
     if isinstance(owner, Optional):
         owner = apiuser.user_id
 
     owner = get_user_or_error(owner)
 
     if RepoModel().get_by_repo_name(repo_name):
         raise JSONRPCError("repo `%s` already exist" % repo_name)
 
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     if isinstance(private, Optional):
         private = defs.get('repo_private') or Optional.extract(private)
     if isinstance(repo_type, Optional):
         repo_type = defs.get('repo_type')
     if isinstance(enable_statistics, Optional):
         enable_statistics = defs.get('repo_enable_statistics')
     if isinstance(enable_locking, Optional):
         enable_locking = defs.get('repo_enable_locking')
     if isinstance(enable_downloads, Optional):
         enable_downloads = defs.get('repo_enable_downloads')
 
     clone_uri = Optional.extract(clone_uri)
     description = Optional.extract(description)
     landing_rev = Optional.extract(landing_rev)
     copy_permissions = Optional.extract(copy_permissions)
 
     try:
         # create structure of groups and return the last group
         repo_group = map_groups(repo_name)
         data = {
             'repo_name': repo_name_cleaned,
             'repo_name_full': repo_name,
             'repo_type': repo_type,
             'repo_description': description,
             'owner': owner,
             'repo_private': private,
             'clone_uri': clone_uri,
             'repo_group': repo_group.group_id if repo_group else None,
             'repo_landing_rev': landing_rev,
             'enable_statistics': enable_statistics,
             'enable_locking': enable_locking,
             'enable_downloads': enable_downloads,
             'repo_copy_permissions': copy_permissions,
         }
 
         if repo_type not in BACKENDS.keys():
             raise Exception("Invalid backend type %s" % repo_type)
         task = RepoModel().create(form_data=data, cur_user=owner)
         from celery.result import BaseAsyncResult
         task_id = None
         if isinstance(task, BaseAsyncResult):
             task_id = task.task_id
         # no commit, it's done in RepoModel, or async via celery
         return {
             'msg': "Created new repository `%s`" % (repo_name,),
             'success': True,  # cannot return the repo data here since fork
             # cann be done async
             'task': task_id
         }
     except Exception:
         log.exception(
             u"Exception while trying to create the repository %s",
             repo_name)
         raise JSONRPCError(
             'failed to create repository `%s`' % (repo_name,))
 
 
 @jsonrpc_method()
 def add_field_to_repo(request, apiuser, repoid, key, label=Optional(''),
                       description=Optional('')):
     """
     Adds an extra field to a repository.
 
     This command can only be run using an |authtoken| with at least
     write permissions to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository id.
     :type repoid: str or int
     :param key: Create a unique field key for this repository.
     :type key: str
     :param label:
     :type label: Optional(str)
     :param description:
     :type description: Optional(str)
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     label = Optional.extract(label) or key
     description = Optional.extract(description)
 
     field = RepositoryField.get_by_key_name(key, repo)
     if field:
         raise JSONRPCError('Field with key '
                            '`%s` exists for repo `%s`' % (key, repoid))
 
     try:
         RepoModel().add_repo_field(repo, key, field_label=label,
                                    field_desc=description)
         Session().commit()
         return {
             'msg': "Added new repository field `%s`" % (key,),
             'success': True,
         }
     except Exception:
         log.exception("Exception occurred while trying to add field to repo")
         raise JSONRPCError(
             'failed to create new field for repository `%s`' % (repoid,))
 
 
 @jsonrpc_method()
 def remove_field_from_repo(request, apiuser, repoid, key):
     """
     Removes an extra field from a repository.
 
     This command can only be run using an |authtoken| with at least
     write permissions to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param key: Set the unique field key for this repository.
     :type key: str
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     field = RepositoryField.get_by_key_name(key, repo)
     if not field:
         raise JSONRPCError('Field with key `%s` does not '
                            'exists for repo `%s`' % (key, repoid))
 
     try:
         RepoModel().delete_repo_field(repo, field_key=key)
         Session().commit()
         return {
             'msg': "Deleted repository field `%s`" % (key,),
             'success': True,
         }
     except Exception:
         log.exception(
             "Exception occurred while trying to delete field from repo")
         raise JSONRPCError(
             'failed to delete field for repository `%s`' % (repoid,))
 
 
 @jsonrpc_method()
 def update_repo(request, apiuser, repoid, name=Optional(None),
                 owner=Optional(OAttr('apiuser')),
                 group=Optional(None),
                 fork_of=Optional(None),
                 description=Optional(''), private=Optional(False),
                 clone_uri=Optional(None), landing_rev=Optional('rev:tip'),
                 enable_statistics=Optional(False),
                 enable_locking=Optional(False),
                 enable_downloads=Optional(False),
                 fields=Optional('')):
     """
     Updates a repository with the given information.
 
     This command can only be run using an |authtoken| with at least
     write permissions to the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: repository name or repository ID.
     :type repoid: str or int
     :param name: Update the |repo| name.
     :type name: str
     :param owner: Set the |repo| owner.
     :type owner: str
     :param group: Set the |repo| group the |repo| belongs to.
     :type group: str
     :param fork_of: Set the master |repo| name.
     :type fork_of: str
     :param description: Update the |repo| description.
     :type description: str
     :param private: Set the |repo| as private. (True | False)
     :type private: bool
     :param clone_uri: Update the |repo| clone URI.
     :type clone_uri: str
     :param landing_rev: Set the |repo| landing revision. Default is
         ``tip``.
     :type landing_rev: str
     :param enable_statistics: Enable statistics on the |repo|,
         (True | False).
     :type enable_statistics: bool
     :param enable_locking: Enable |repo| locking.
     :type enable_locking: bool
     :param enable_downloads: Enable downloads from the |repo|,
         (True | False).
     :type enable_downloads: bool
     :param fields: Add extra fields to the |repo|. Use the following
         example format: ``field_key=field_val,field_key2=fieldval2``.
         Escape ', ' with \,
     :type fields: str
     """
     repo = get_repo_or_error(repoid)
     include_secrets = False
     if has_superadmin_permission(apiuser):
         include_secrets = True
     else:
         _perms = ('repository.admin',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     updates = {
         # update function requires this.
         'repo_name': repo.just_name
     }
     repo_group = group
     if not isinstance(repo_group, Optional):
         repo_group = get_repo_group_or_error(repo_group)
         repo_group = repo_group.group_id
 
     repo_fork_of = fork_of
     if not isinstance(repo_fork_of, Optional):
         repo_fork_of = get_repo_or_error(repo_fork_of)
         repo_fork_of = repo_fork_of.repo_id
 
     try:
         store_update(updates, name, 'repo_name')
         store_update(updates, repo_group, 'repo_group')
         store_update(updates, repo_fork_of, 'fork_id')
         store_update(updates, owner, 'user')
         store_update(updates, description, 'repo_description')
         store_update(updates, private, 'repo_private')
         store_update(updates, clone_uri, 'clone_uri')
         store_update(updates, landing_rev, 'repo_landing_rev')
         store_update(updates, enable_statistics, 'repo_enable_statistics')
         store_update(updates, enable_locking, 'repo_enable_locking')
         store_update(updates, enable_downloads, 'repo_enable_downloads')
 
         # extra fields
         fields = parse_args(Optional.extract(fields), key_prefix='ex_')
         if fields:
             updates.update(fields)
 
         RepoModel().update(repo, **updates)
         Session().commit()
         return {
             'msg': 'updated repo ID:%s %s' % (
                 repo.repo_id, repo.repo_name),
             'repository': repo.get_api_data(
                 include_secrets=include_secrets)
         }
     except Exception:
         log.exception(
             u"Exception while trying to update the repository %s",
             repoid)
         raise JSONRPCError('failed to update repo `%s`' % repoid)
 
 
 @jsonrpc_method()
 def fork_repo(request, apiuser, repoid, fork_name,
               owner=Optional(OAttr('apiuser')),
               description=Optional(''), copy_permissions=Optional(False),
               private=Optional(False), landing_rev=Optional('rev:tip')):
     """
     Creates a fork of the specified |repo|.
 
     * If using |RCE| with Celery this will immediately return a success
       message, even though the fork will be created asynchronously.
 
     This command can only be run using an |authtoken| with fork
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set repository name or repository ID.
     :type repoid: str or int
     :param fork_name: Set the fork name.
     :type fork_name: str
     :param owner: Set the fork owner.
     :type owner: str
     :param description: Set the fork descripton.
     :type description: str
     :param copy_permissions: Copy permissions from parent |repo|. The
         default is False.
     :type copy_permissions: bool
     :param private: Make the fork private. The default is False.
     :type private: bool
     :param landing_rev: Set the landing revision. The default is tip.
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_for_response>
         api_key : "<api_key>"
         args:     {
                     "repoid" :          "<reponame or repo_id>",
                     "fork_name":        "<forkname>",
                     "owner":            "<username or user_id = Optional(=apiuser)>",
                     "description":      "<description>",
                     "copy_permissions": "<bool>",
                     "private":          "<bool>",
                     "landing_rev":      "<landing_rev>"
                   }
 
     Example error output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg": "Created fork of `<reponame>` as `<forkname>`",
                   "success": true,
                   "task": "<celery task id or None if done sync>"
                 }
         error:  null
 
     """
     if not has_superadmin_permission(apiuser):
         if not HasPermissionAnyApi('hg.fork.repository')(user=apiuser):
             raise JSONRPCForbidden()
 
     repo = get_repo_or_error(repoid)
     repo_name = repo.repo_name
 
     (fork_name_cleaned,
      parent_group_name) = RepoGroupModel()._get_group_name_and_parent(
         fork_name)
 
     if not has_superadmin_permission(apiuser):
         # check if we have at least read permission for
         # this repo that we fork !
         _perms = (
             'repository.admin', 'repository.write', 'repository.read')
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
         if not isinstance(owner, Optional):
             # forbid setting owner for non super admins
             raise JSONRPCError(
                 'Only RhodeCode admin can specify `owner` param'
             )
         # check if we have a create.repo permission if not maybe the parent
         # group permission
         if not HasPermissionAnyApi('hg.create.repository')(user=apiuser):
             if parent_group_name:
                 repogroupid = parent_group_name
                 repo_group = get_repo_group_or_error(parent_group_name)
 
                 _perms = ('group.admin',)
                 if not HasRepoGroupPermissionAnyApi(*_perms)(
                         user=apiuser, group_name=repo_group.group_name):
                     raise JSONRPCError(
                         'repository group `%s` does not exist' % (
                             repogroupid,))
             else:
                 raise JSONRPCForbidden()
 
     _repo = RepoModel().get_by_repo_name(fork_name)
     if _repo:
         type_ = 'fork' if _repo.fork else 'repo'
         raise JSONRPCError("%s `%s` already exist" % (type_, fork_name))
 
     if isinstance(owner, Optional):
         owner = apiuser.user_id
 
     owner = get_user_or_error(owner)
 
     try:
         # create structure of groups and return the last group
         repo_group = map_groups(fork_name)
         form_data = {
             'repo_name': fork_name_cleaned,
             'repo_name_full': fork_name,
             'repo_group': repo_group.group_id if repo_group else None,
             'repo_type': repo.repo_type,
             'description': Optional.extract(description),
             'private': Optional.extract(private),
             'copy_permissions': Optional.extract(copy_permissions),
             'landing_rev': Optional.extract(landing_rev),
             'fork_parent_id': repo.repo_id,
         }
 
         task = RepoModel().create_fork(form_data, cur_user=owner)
         # no commit, it's done in RepoModel, or async via celery
         from celery.result import BaseAsyncResult
         task_id = None
         if isinstance(task, BaseAsyncResult):
             task_id = task.task_id
         return {
             'msg': 'Created fork of `%s` as `%s`' % (
                 repo.repo_name, fork_name),
             'success': True,  # cannot return the repo data here since fork
             # can be done async
             'task': task_id
         }
     except Exception:
         log.exception("Exception occurred while trying to fork a repo")
         raise JSONRPCError(
             'failed to fork repository `%s` as `%s`' % (
                 repo_name, fork_name))
 
 
 @jsonrpc_method()
 def delete_repo(request, apiuser, repoid, forks=Optional('')):
     """
     Deletes a repository.
 
     * When the `forks` parameter is set it's possible to detach or delete
       forks of deleted repository.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param forks: Set to `detach` or `delete` forks from the |repo|.
     :type forks: Optional(str)
 
     Example error output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg": "Deleted repository `<reponame>`",
                   "success": true
                 }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         handle_forks = Optional.extract(forks)
         _forks_msg = ''
         _forks = [f for f in repo.forks]
         if handle_forks == 'detach':
             _forks_msg = ' ' + 'Detached %s forks' % len(_forks)
         elif handle_forks == 'delete':
             _forks_msg = ' ' + 'Deleted %s forks' % len(_forks)
         elif _forks:
             raise JSONRPCError(
                 'Cannot delete `%s` it still contains attached forks' %
                 (repo.repo_name,)
             )
 
         RepoModel().delete(repo, forks=forks)
         Session().commit()
         return {
             'msg': 'Deleted repository `%s`%s' % (
                 repo.repo_name, _forks_msg),
             'success': True
         }
     except Exception:
         log.exception("Exception occurred while trying to delete repo")
         raise JSONRPCError(
             'failed to delete repository `%s`' % (repo.repo_name,)
         )
 
 
 #TODO: marcink, change name ?
 @jsonrpc_method()
 def invalidate_cache(request, apiuser, repoid, delete_keys=Optional(False)):
     """
     Invalidates the cache for the specified repository.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Sets the repository name or repository ID.
     :type repoid: str or int
     :param delete_keys: This deletes the invalidated keys instead of
         just flagging them.
     :type delete_keys: Optional(``True`` | ``False``)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'msg': Cache for repository `<repository name>` was invalidated,
         'repository': <repository name>
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error : {
          'Error occurred during cache invalidation action'
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin', 'repository.write',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     delete = Optional.extract(delete_keys)
     try:
         ScmModel().mark_for_invalidation(repo.repo_name, delete=delete)
         return {
             'msg': 'Cache for repository `%s` was invalidated' % (repoid,),
             'repository': repo.repo_name
         }
     except Exception:
         log.exception(
             "Exception occurred while trying to invalidate repo cache")
         raise JSONRPCError(
             'Error occurred during cache invalidation action'
         )
 
 
 #TODO: marcink, change name ?
 @jsonrpc_method()
 def lock(request, apiuser, repoid, locked=Optional(None),
          userid=Optional(OAttr('apiuser'))):
     """
     Sets the lock state of the specified |repo| by the given user.
     From more information, see :ref:`repo-locking`.
 
     * If the ``userid`` option is not set, the repository is locked to the
       user who called the method.
     * If the ``locked`` parameter is not set, the current lock state of the
       repository is displayed.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Sets the repository name or repository ID.
     :type repoid: str or int
     :param locked: Sets the lock state.
     :type locked: Optional(``True`` | ``False``)
     :param userid: Set the repository lock to this user.
     :type userid: Optional(str or int)
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'repo': '<reponame>',
         'locked': <bool: lock state>,
         'locked_since': <int: lock timestamp>,
         'locked_by': <username of person who made the lock>,
         'lock_reason': <str: reason for locking>,
         'lock_state_changed': <bool: True if lock state has been changed in this request>,
         'msg': 'Repo `<reponame>` locked by `<username>` on <timestamp>.'
         or
         'msg': 'Repo `<repository name>` not locked.'
         or
         'msg': 'User `<user name>` set lock state for repo `<repository name>` to `<new lock state>`'
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred locking repository `<reponame>`
       }
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         # check if we have at least write permission for this repo !
         _perms = ('repository.admin', 'repository.write',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
         # make sure normal user does not pass someone else userid,
         # he is not allowed to do that
         if not isinstance(userid, Optional) and userid != apiuser.user_id:
             raise JSONRPCError('userid is not the same as your user')
 
     if isinstance(userid, Optional):
         userid = apiuser.user_id
 
     user = get_user_or_error(userid)
 
     if isinstance(locked, Optional):
         lockobj = repo.locked
 
         if lockobj[0] is None:
             _d = {
                 'repo': repo.repo_name,
                 'locked': False,
                 'locked_since': None,
                 'locked_by': None,
                 'lock_reason': None,
                 'lock_state_changed': False,
                 'msg': 'Repo `%s` not locked.' % repo.repo_name
             }
             return _d
         else:
             _user_id, _time, _reason = lockobj
             lock_user = get_user_or_error(userid)
             _d = {
                 'repo': repo.repo_name,
                 'locked': True,
                 'locked_since': _time,
                 'locked_by': lock_user.username,
                 'lock_reason': _reason,
                 'lock_state_changed': False,
                 'msg': ('Repo `%s` locked by `%s` on `%s`.'
                         % (repo.repo_name, lock_user.username,
                            json.dumps(time_to_datetime(_time))))
             }
             return _d
 
     # force locked state through a flag
     else:
         locked = str2bool(locked)
         lock_reason = Repository.LOCK_API
         try:
             if locked:
                 lock_time = time.time()
                 Repository.lock(repo, user.user_id, lock_time, lock_reason)
             else:
                 lock_time = None
                 Repository.unlock(repo)
             _d = {
                 'repo': repo.repo_name,
                 'locked': locked,
                 'locked_since': lock_time,
                 'locked_by': user.username,
                 'lock_reason': lock_reason,
                 'lock_state_changed': True,
                 'msg': ('User `%s` set lock state for repo `%s` to `%s`'
                         % (user.username, repo.repo_name, locked))
             }
             return _d
         except Exception:
             log.exception(
                 "Exception occurred while trying to lock repository")
             raise JSONRPCError(
                 'Error occurred locking repository `%s`' % repo.repo_name
             )
 
 
 @jsonrpc_method()
 def comment_commit(
         request, apiuser, repoid, commit_id, message,
         userid=Optional(OAttr('apiuser')), status=Optional(None)):
     """
     Set a commit comment, and optionally change the status of the commit.
     This command can be executed only using api_key belonging to user
     with admin rights, or repository administrator.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param commit_id: Specify the commit_id for which to set a comment.
     :type commit_id: str
     :param message: The comment text.
     :type message: str
     :param userid: Set the user name of the comment creator.
     :type userid: Optional(str or int)
     :param status: status, one of 'not_reviewed', 'approved', 'rejected',
        'under_review'
     :type status: str
 
     Example error output:
 
     .. code-block:: json
 
         {
             "id" : <id_given_in_input>,
             "result" : {
                 "msg": "Commented on commit `<commit_id>` for repository `<repoid>`",
                 "status_change": null or <status>,
                 "success": true
             },
             "error" :  null
         }
 
     """
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     if isinstance(userid, Optional):
         userid = apiuser.user_id
 
     user = get_user_or_error(userid)
     status = Optional.extract(status)
 
     allowed_statuses = [x[0] for x in ChangesetStatus.STATUSES]
     if status and status not in allowed_statuses:
         raise JSONRPCError('Bad status, must be on '
                            'of %s got %s' % (allowed_statuses, status,))
 
     try:
         rc_config = SettingsModel().get_all_settings()
         renderer = rc_config.get('rhodecode_markup_renderer', 'rst')
 
         comm = ChangesetCommentsModel().create(
             message, repo, user, revision=commit_id, status_change=status,
             renderer=renderer)
         if status:
             # also do a status change
             try:
                 ChangesetStatusModel().set_status(
                     repo, status, user, comm, revision=commit_id,
                     dont_allow_on_closed_pull_request=True
                 )
             except StatusChangeOnClosedPullRequestError:
                 log.exception(
                     "Exception occurred while trying to change repo commit status")
                 msg = ('Changing status on a changeset associated with '
                        'a closed pull request is not allowed')
                 raise JSONRPCError(msg)
 
         Session().commit()
         return {
             'msg': (
                 'Commented on commit `%s` for repository `%s`' % (
                     comm.revision, repo.repo_name)),
             'status_change': status,
             'success': True,
         }
     except JSONRPCError:
         # catch any inside errors, and re-raise them to prevent from
         # below global catch to silence them
         raise
     except Exception:
         log.exception("Exception occurred while trying to comment on commit")
         raise JSONRPCError(
             'failed to set comment on repository `%s`' % (repo.repo_name,)
         )
 
 
 @jsonrpc_method()
 def grant_user_permission(request, apiuser, repoid, userid, perm):
     """
     Grant permissions for the specified user on the given repository,
     or update existing permissions if found.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param userid: Set the user name.
     :type userid: str
     :param perm: Set the user permissions, using the following format
         ``(repository.(none|read|write|admin))``
     :type perm: str
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg" : "Granted perm: `<perm>` for user: `<username>` in repo: `<reponame>`",
                   "success": true
                 }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     user = get_user_or_error(userid)
     perm = get_perm_or_error(perm)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
 
         RepoModel().grant_user_permission(repo=repo, user=user, perm=perm)
 
         Session().commit()
         return {
             'msg': 'Granted perm: `%s` for user: `%s` in repo: `%s`' % (
                 perm.permission_name, user.username, repo.repo_name
             ),
             'success': True
         }
     except Exception:
         log.exception(
             "Exception occurred while trying edit permissions for repo")
         raise JSONRPCError(
             'failed to edit permission for user: `%s` in repo: `%s`' % (
                 userid, repoid
             )
         )
 
 
 @jsonrpc_method()
 def revoke_user_permission(request, apiuser, repoid, userid):
     """
     Revoke permission for a user on the specified repository.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param userid: Set the user name of revoked user.
     :type userid: str or int
 
     Example error output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg" : "Revoked perm for user: `<username>` in repo: `<reponame>`",
                   "success": true
                 }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     user = get_user_or_error(userid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         RepoModel().revoke_user_permission(repo=repo, user=user)
         Session().commit()
         return {
             'msg': 'Revoked perm for user: `%s` in repo: `%s`' % (
                 user.username, repo.repo_name
             ),
             'success': True
         }
     except Exception:
         log.exception(
             "Exception occurred while trying revoke permissions to repo")
         raise JSONRPCError(
             'failed to edit permission for user: `%s` in repo: `%s`' % (
                 userid, repoid
             )
         )
 
 
 @jsonrpc_method()
 def grant_user_group_permission(request, apiuser, repoid, usergroupid, perm):
     """
     Grant permission for a user group on the specified repository,
     or update existing permissions.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param usergroupid: Specify the ID of the user group.
     :type usergroupid: str or int
     :param perm: Set the user group permissions using the following
         format: (repository.(none|read|write|admin))
     :type perm: str
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         "msg" : "Granted perm: `<perm>` for group: `<usersgroupname>` in repo: `<reponame>`",
         "success": true
 
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "failed to edit permission for user group: `<usergroup>` in repo `<repo>`'
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     perm = get_perm_or_error(perm)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     user_group = get_user_group_or_error(usergroupid)
     if not has_superadmin_permission(apiuser):
         # check if we have at least read permission for this user group !
         _perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin',)
         if not HasUserGroupPermissionAnyApi(*_perms)(
                 user=apiuser, user_group_name=user_group.users_group_name):
             raise JSONRPCError(
                 'user group `%s` does not exist' % (usergroupid,))
 
     try:
         RepoModel().grant_user_group_permission(
             repo=repo, group_name=user_group, perm=perm)
 
         Session().commit()
         return {
             'msg': 'Granted perm: `%s` for user group: `%s` in '
                    'repo: `%s`' % (
                        perm.permission_name, user_group.users_group_name,
                        repo.repo_name
                    ),
             'success': True
         }
     except Exception:
         log.exception(
             "Exception occurred while trying change permission on repo")
         raise JSONRPCError(
             'failed to edit permission for user group: `%s` in '
             'repo: `%s`' % (
                 usergroupid, repo.repo_name
             )
         )
 
 
 @jsonrpc_method()
 def revoke_user_group_permission(request, apiuser, repoid, usergroupid):
     """
     Revoke the permissions of a user group on a given repository.
 
     This command can only be run using an |authtoken| with admin
     permissions on the |repo|.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: Set the repository name or repository ID.
     :type repoid: str or int
     :param usergroupid: Specify the user group ID.
     :type usergroupid: str or int
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result: {
                   "msg" : "Revoked perm for group: `<usersgroupname>` in repo: `<reponame>`",
                   "success": true
                 }
         error:  null
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     user_group = get_user_group_or_error(usergroupid)
     if not has_superadmin_permission(apiuser):
         # check if we have at least read permission for this user group !
         _perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin',)
         if not HasUserGroupPermissionAnyApi(*_perms)(
                 user=apiuser, user_group_name=user_group.users_group_name):
             raise JSONRPCError(
                 'user group `%s` does not exist' % (usergroupid,))
 
     try:
         RepoModel().revoke_user_group_permission(
             repo=repo, group_name=user_group)
 
         Session().commit()
         return {
             'msg': 'Revoked perm for user group: `%s` in repo: `%s`' % (
                 user_group.users_group_name, repo.repo_name
             ),
             'success': True
         }
     except Exception:
         log.exception("Exception occurred while trying revoke "
                       "user group permission on repo")
         raise JSONRPCError(
             'failed to edit permission for user group: `%s` in '
             'repo: `%s`' % (
                 user_group.users_group_name, repo.repo_name
             )
         )
 
 
 @jsonrpc_method()
 def pull(request, apiuser, repoid):
     """
     Triggers a pull on the given repository from a remote location. You
     can use this to keep remote repositories up-to-date.
 
     This command can only be run using an |authtoken| with admin
     rights to the specified repository. For more information,
     see :ref:`config-token-ref`.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         "msg": "Pulled from `<repository name>`"
         "repository": "<repository name>"
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "Unable to pull changes from `<reponame>`"
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         ScmModel().pull_changes(repo.repo_name, apiuser.username)
         return {
             'msg': 'Pulled from `%s`' % repo.repo_name,
             'repository': repo.repo_name
         }
     except Exception:
         log.exception("Exception occurred while trying to "
                       "pull changes from remote location")
         raise JSONRPCError(
             'Unable to pull changes from `%s`' % repo.repo_name
         )
 
 
 @jsonrpc_method()
 def strip(request, apiuser, repoid, revision, branch):
     """
     Strips the given revision from the specified repository.
 
     * This will remove the revision and all of its decendants.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param repoid: The repository name or repository ID.
     :type repoid: str or int
     :param revision: The revision you wish to strip.
     :type revision: str
     :param branch: The branch from which to strip the revision.
     :type branch: str
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         "msg": "'Stripped commit <commit_hash> from repo `<repository name>`'"
         "repository": "<repository name>"
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         "Unable to strip commit <commit_hash> from repo `<repository name>`"
       }
 
     """
 
     repo = get_repo_or_error(repoid)
     if not has_superadmin_permission(apiuser):
         _perms = ('repository.admin',)
         has_repo_permissions(apiuser, repoid, repo, _perms)
 
     try:
         ScmModel().strip(repo, revision, branch)
         return {
             'msg': 'Stripped commit %s from repo `%s`' % (
                 revision, repo.repo_name),
             'repository': repo.repo_name
         }
     except Exception:
         log.exception("Exception while trying to strip")
         raise JSONRPCError(
             'Unable to strip commit %s from repo `%s`' % (
                 revision, repo.repo_name)
         )

rhodecode/authentication/__init__.py

@@ -1,85 +1,122 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
+import importlib
 
 from pkg_resources import iter_entry_points
 from pyramid.authentication import SessionAuthenticationPolicy
 
 from rhodecode.authentication.registry import AuthenticationPluginRegistry
 from rhodecode.authentication.routes import root_factory
 from rhodecode.authentication.routes import AuthnRootResource
 from rhodecode.config.routing import ADMIN_PREFIX
+from rhodecode.model.settings import SettingsModel
+
 
 log = logging.getLogger(__name__)
 
+# Plugin ID prefixes to distinct between normal and legacy plugins.
+plugin_prefix = 'egg:'
+legacy_plugin_prefix = 'py:'
+
 
 # TODO: Currently this is only used to discover the authentication plugins.
 # Later on this may be used in a generic way to look up and include all kinds
 # of supported enterprise plugins. Therefore this has to be moved and
 # refactored to a real 'plugin look up' machinery.
 # TODO: When refactoring this think about splitting it up into distinct
 # discover, load and include phases.
 def _discover_plugins(config, entry_point='enterprise.plugins1'):
-    _discovered_plugins = {}
-
     for ep in iter_entry_points(entry_point):
-        plugin_id = 'egg:{}#{}'.format(ep.dist.project_name, ep.name)
+        plugin_id = '{}{}#{}'.format(
+            plugin_prefix, ep.dist.project_name, ep.name)
         log.debug('Plugin discovered: "%s"', plugin_id)
-        module = ep.load()
-        plugin = module(plugin_id=plugin_id)
-        config.include(plugin.includeme)
+        try:
+            module = ep.load()
+            plugin = module(plugin_id=plugin_id)
+            config.include(plugin.includeme)
+        except Exception as e:
+            log.exception(
+                'Exception while loading authentication plugin '
+                '"{}": {}'.format(plugin_id, e.message))
+
+
+def _import_legacy_plugin(plugin_id):
+    module_name = plugin_id.split(legacy_plugin_prefix, 1)[-1]
+    module = importlib.import_module(module_name)
+    return module.plugin_factory(plugin_id=plugin_id)
+
 
-    return _discovered_plugins
+def _discover_legacy_plugins(config, prefix=legacy_plugin_prefix):
+    """
+    Function that imports the legacy plugins stored in the 'auth_plugins'
+    setting in database which are using the specified prefix. Normally 'py:' is
+    used for the legacy plugins.
+    """
+    auth_plugins = SettingsModel().get_setting_by_name('auth_plugins')
+    enabled_plugins = auth_plugins.app_settings_value
+    legacy_plugins = [id_ for id_ in enabled_plugins if id_.startswith(prefix)]
+
+    for plugin_id in legacy_plugins:
+        log.debug('Legacy plugin discovered: "%s"', plugin_id)
+        try:
+            plugin = _import_legacy_plugin(plugin_id)
+            config.include(plugin.includeme)
+        except Exception as e:
+            log.exception(
+                'Exception while loading legacy authentication plugin '
+                '"{}": {}'.format(plugin_id, e.message))
 
 
 def includeme(config):
     # Set authentication policy.
     authn_policy = SessionAuthenticationPolicy()
     config.set_authentication_policy(authn_policy)
 
     # Create authentication plugin registry and add it to the pyramid registry.
-    authn_registry = AuthenticationPluginRegistry()
+    authn_registry = AuthenticationPluginRegistry(config.get_settings())
     config.add_directive('add_authn_plugin', authn_registry.add_authn_plugin)
     config.registry.registerUtility(authn_registry)
 
     # Create authentication traversal root resource.
     authn_root_resource = root_factory()
     config.add_directive('add_authn_resource',
                          authn_root_resource.add_authn_resource)
 
     # Add the authentication traversal route.
     config.add_route('auth_home',
                      ADMIN_PREFIX + '/auth*traverse',
                      factory=root_factory)
     # Add the authentication settings root views.
     config.add_view('rhodecode.authentication.views.AuthSettingsView',
                     attr='index',
                     request_method='GET',
                     route_name='auth_home',
                     context=AuthnRootResource)
     config.add_view('rhodecode.authentication.views.AuthSettingsView',
                     attr='auth_settings',
                     request_method='POST',
                     route_name='auth_home',
                     context=AuthnRootResource)
 
     # Auto discover authentication plugins and include their configuration.
     _discover_plugins(config)
+    _discover_legacy_plugins(config)

rhodecode/authentication/base.py

@@ -1,739 +1,609 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Authentication modules
 """
 
 import logging
 import time
 import traceback
+import warnings
 
-from authomatic import Authomatic
-from authomatic.adapters import WebObAdapter
-from authomatic.providers import oauth2, oauth1
-from pylons import url
-from pylons.controllers.util import Response
-from pylons.i18n.translation import _
 from pyramid.threadlocal import get_current_registry
 from sqlalchemy.ext.hybrid import hybrid_property
 
-import rhodecode.lib.helpers as h
 from rhodecode.authentication.interface import IAuthnPluginRegistry
 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
 from rhodecode.lib import caches
 from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
 from rhodecode.lib.utils2 import md5_safe, safe_int
 from rhodecode.lib.utils2 import safe_str
-from rhodecode.model.db import User, ExternalIdentity
+from rhodecode.model.db import User
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import SettingsModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 
 
 log = logging.getLogger(__name__)
 
 # auth types that authenticate() function can receive
 VCS_TYPE = 'vcs'
 HTTP_TYPE = 'http'
 
 
 class LazyFormencode(object):
     def __init__(self, formencode_obj, *args, **kwargs):
         self.formencode_obj = formencode_obj
         self.args = args
         self.kwargs = kwargs
 
     def __call__(self, *args, **kwargs):
         from inspect import isfunction
         formencode_obj = self.formencode_obj
         if isfunction(formencode_obj):
             # case we wrap validators into functions
             formencode_obj = self.formencode_obj(*args, **kwargs)
         return formencode_obj(*self.args, **self.kwargs)
 
 
 class RhodeCodeAuthPluginBase(object):
     # cache the authentication request for N amount of seconds. Some kind
     # of authentication methods are very heavy and it's very efficient to cache
     # the result of a call. If it's set to None (default) cache is off
     AUTH_CACHE_TTL = None
     AUTH_CACHE = {}
 
     auth_func_attrs = {
         "username": "unique username",
         "firstname": "first name",
         "lastname": "last name",
         "email": "email address",
         "groups": '["list", "of", "groups"]',
         "extern_name": "name in external source of record",
         "extern_type": "type of external source of record",
         "admin": 'True|False defines if user should be RhodeCode super admin',
         "active":
             'True|False defines active state of user internally for RhodeCode',
         "active_from_extern":
             "True|False\None, active state from the external auth, "
             "None means use definition from RhodeCode extern_type active value"
     }
     # set on authenticate() method and via set_auth_type func.
     auth_type = None
 
     # List of setting names to store encrypted. Plugins may override this list
     # to store settings encrypted.
     _settings_encrypted = []
 
     # Mapping of python to DB settings model types. Plugins may override or
     # extend this mapping.
     _settings_type_map = {
         str: 'str',
         int: 'int',
         unicode: 'unicode',
         bool: 'bool',
         list: 'list',
     }
 
     def __init__(self, plugin_id):
         self._plugin_id = plugin_id
 
     def _get_setting_full_name(self, name):
         """
         Return the full setting name used for storing values in the database.
         """
         # TODO: johbo: Using the name here is problematic. It would be good to
         # introduce either new models in the database to hold Plugin and
         # PluginSetting or to use the plugin id here.
         return 'auth_{}_{}'.format(self.name, name)
 
     def _get_setting_type(self, name, value):
         """
         Get the type as used by the SettingsModel accordingly to type of passed
         value. Optionally the suffix `.encrypted` is appended to instruct
         SettingsModel to store it encrypted.
         """
         type_ = self._settings_type_map.get(type(value), 'unicode')
         if name in self._settings_encrypted:
             type_ = '{}.encrypted'.format(type_)
         return type_
 
     def is_enabled(self):
         """
         Returns true if this plugin is enabled. An enabled plugin can be
         configured in the admin interface but it is not consulted during
         authentication.
         """
         auth_plugins = SettingsModel().get_auth_plugins()
         return self.get_id() in auth_plugins
 
     def is_active(self):
         """
         Returns true if the plugin is activated. An activated plugin is
         consulted during authentication, assumed it is also enabled.
         """
         return self.get_setting_by_name('enabled')
 
     def get_id(self):
         """
         Returns the plugin id.
         """
         return self._plugin_id
 
     def get_display_name(self):
         """
         Returns a translation string for displaying purposes.
         """
         raise NotImplementedError('Not implemented in base class')
 
     def get_settings_schema(self):
         """
         Returns a colander schema, representing the plugin settings.
         """
         return AuthnPluginSettingsSchemaBase()
 
     def get_setting_by_name(self, name):
         """
         Returns a plugin setting by name.
         """
         full_name = self._get_setting_full_name(name)
         db_setting = SettingsModel().get_setting_by_name(full_name)
         return db_setting.app_settings_value if db_setting else None
 
     def create_or_update_setting(self, name, value):
         """
         Create or update a setting for this plugin in the persistent storage.
         """
         full_name = self._get_setting_full_name(name)
         type_ = self._get_setting_type(name, value)
         db_setting = SettingsModel().create_or_update_setting(
             full_name, value, type_)
         return db_setting.app_settings_value
 
     def get_settings(self):
         """
         Returns the plugin settings as dictionary.
         """
         settings = {}
         for node in self.get_settings_schema():
             settings[node.name] = self.get_setting_by_name(node.name)
         return settings
 
     @property
     def validators(self):
         """
         Exposes RhodeCode validators modules
         """
         # this is a hack to overcome issues with pylons threadlocals and
         # translator object _() not beein registered properly.
         class LazyCaller(object):
             def __init__(self, name):
                 self.validator_name = name
 
             def __call__(self, *args, **kwargs):
                 from rhodecode.model import validators as v
                 obj = getattr(v, self.validator_name)
                 # log.debug('Initializing lazy formencode object: %s', obj)
                 return LazyFormencode(obj, *args, **kwargs)
 
         class ProxyGet(object):
             def __getattribute__(self, name):
                 return LazyCaller(name)
 
         return ProxyGet()
 
     @hybrid_property
     def name(self):
         """
         Returns the name of this authentication plugin.
 
         :returns: string
         """
         raise NotImplementedError("Not implemented in base class")
 
+    @property
+    def is_headers_auth(self):
+        """
+        Returns True if this authentication plugin uses HTTP headers as
+        authentication method.
+        """
+        return False
+
     @hybrid_property
     def is_container_auth(self):
         """
-        Returns bool if this module uses container auth.
-
-        This property will trigger an automatic call to authenticate on
-        a visit to the website or during a push/pull.
-
-        :returns: bool
+        Deprecated method that indicates if this authentication plugin uses
+        HTTP headers as authentication method.
         """
-        return False
+        warnings.warn(
+            'Use is_headers_auth instead.', category=DeprecationWarning)
+        return self.is_headers_auth
 
     @hybrid_property
     def allows_creating_users(self):
         """
         Defines if Plugin allows users to be created on-the-fly when
         authentication is called. Controls how external plugins should behave
         in terms if they are allowed to create new users, or not. Base plugins
         should not be allowed to, but External ones should be !
 
         :return: bool
         """
         return False
 
     def set_auth_type(self, auth_type):
         self.auth_type = auth_type
 
     def allows_authentication_from(
             self, user, allows_non_existing_user=True,
             allowed_auth_plugins=None, allowed_auth_sources=None):
         """
         Checks if this authentication module should accept a request for
         the current user.
 
         :param user: user object fetched using plugin's get_user() method.
         :param allows_non_existing_user: if True, don't allow the
             user to be empty, meaning not existing in our database
         :param allowed_auth_plugins: if provided, users extern_type will be
             checked against a list of provided extern types, which are plugin
             auth_names in the end
         :param allowed_auth_sources: authentication type allowed,
             `http` or `vcs` default is both.
             defines if plugin will accept only http authentication vcs
             authentication(git/hg) or both
         :returns: boolean
         """
         if not user and not allows_non_existing_user:
             log.debug('User is empty but plugin does not allow empty users,'
                       'not allowed to authenticate')
             return False
 
         expected_auth_plugins = allowed_auth_plugins or [self.name]
         if user and (user.extern_type and
                      user.extern_type not in expected_auth_plugins):
             log.debug(
                 'User `%s` is bound to `%s` auth type. Plugin allows only '
                 '%s, skipping', user, user.extern_type, expected_auth_plugins)
 
             return False
 
         # by default accept both
         expected_auth_from = allowed_auth_sources or [HTTP_TYPE, VCS_TYPE]
         if self.auth_type not in expected_auth_from:
             log.debug('Current auth source is %s but plugin only allows %s',
                       self.auth_type, expected_auth_from)
             return False
 
         return True
 
     def get_user(self, username=None, **kwargs):
         """
         Helper method for user fetching in plugins, by default it's using
         simple fetch by username, but this method can be custimized in plugins
-        eg. container auth plugin to fetch user by environ params
+        eg. headers auth plugin to fetch user by environ params
 
         :param username: username if given to fetch from database
         :param kwargs: extra arguments needed for user fetching.
         """
         user = None
         log.debug(
             'Trying to fetch user `%s` from RhodeCode database', username)
         if username:
             user = User.get_by_username(username)
             if not user:
                 log.debug('User not found, fallback to fetch user in '
                           'case insensitive mode')
                 user = User.get_by_username(username, case_insensitive=True)
         else:
             log.debug('provided username:`%s` is empty skipping...', username)
         if not user:
             log.debug('User `%s` not found in database', username)
         return user
 
     def user_activation_state(self):
         """
         Defines user activation state when creating new users
 
         :returns: boolean
         """
         raise NotImplementedError("Not implemented in base class")
 
     def auth(self, userobj, username, passwd, settings, **kwargs):
         """
         Given a user object (which may be null), username, a plaintext
         password, and a settings object (containing all the keys needed as
         listed in settings()), authenticate this user's login attempt.
 
         Return None on failure. On success, return a dictionary of the form:
 
             see: RhodeCodeAuthPluginBase.auth_func_attrs
         This is later validated for correctness
         """
         raise NotImplementedError("not implemented in base class")
 
     def _authenticate(self, userobj, username, passwd, settings, **kwargs):
         """
         Wrapper to call self.auth() that validates call on it
 
         :param userobj: userobj
         :param username: username
         :param passwd: plaintext password
         :param settings: plugin settings
         """
         auth = self.auth(userobj, username, passwd, settings, **kwargs)
         if auth:
             # check if hash should be migrated ?
             new_hash = auth.get('_hash_migrate')
             if new_hash:
                 self._migrate_hash_to_bcrypt(username, passwd, new_hash)
             return self._validate_auth_return(auth)
         return auth
 
     def _migrate_hash_to_bcrypt(self, username, password, new_hash):
         new_hash_cypher = _RhodeCodeCryptoBCrypt()
         # extra checks, so make sure new hash is correct.
         password_encoded = safe_str(password)
         if new_hash and new_hash_cypher.hash_check(
                 password_encoded, new_hash):
             cur_user = User.get_by_username(username)
             cur_user.password = new_hash
             Session().add(cur_user)
             Session().flush()
             log.info('Migrated user %s hash to bcrypt', cur_user)
 
     def _validate_auth_return(self, ret):
         if not isinstance(ret, dict):
             raise Exception('returned value from auth must be a dict')
         for k in self.auth_func_attrs:
             if k not in ret:
                 raise Exception('Missing %s attribute from returned data' % k)
         return ret
 
 
 class RhodeCodeExternalAuthPlugin(RhodeCodeAuthPluginBase):
 
     @hybrid_property
     def allows_creating_users(self):
         return True
 
     def use_fake_password(self):
         """
         Return a boolean that indicates whether or not we should set the user's
         password to a random value when it is authenticated by this plugin.
         If your plugin provides authentication, then you will generally
         want this.
 
         :returns: boolean
         """
         raise NotImplementedError("Not implemented in base class")
 
     def _authenticate(self, userobj, username, passwd, settings, **kwargs):
         # at this point _authenticate calls plugin's `auth()` function
         auth = super(RhodeCodeExternalAuthPlugin, self)._authenticate(
             userobj, username, passwd, settings, **kwargs)
         if auth:
             # maybe plugin will clean the username ?
             # we should use the return value
             username = auth['username']
 
             # if external source tells us that user is not active, we should
             # skip rest of the process. This can prevent from creating users in
             # RhodeCode when using external authentication, but if it's
             # inactive user we shouldn't create that user anyway
             if auth['active_from_extern'] is False:
                 log.warning(
                     "User %s authenticated against %s, but is inactive",
                     username, self.__module__)
                 return None
 
             cur_user = User.get_by_username(username, case_insensitive=True)
             is_user_existing = cur_user is not None
 
             if is_user_existing:
                 log.debug('Syncing user `%s` from '
                           '`%s` plugin', username, self.name)
             else:
                 log.debug('Creating non existing user `%s` from '
                           '`%s` plugin', username, self.name)
 
             if self.allows_creating_users:
                 log.debug('Plugin `%s` allows to '
                           'create new users', self.name)
             else:
                 log.debug('Plugin `%s` does not allow to '
                           'create new users', self.name)
 
             user_parameters = {
                 'username': username,
                 'email': auth["email"],
                 'firstname': auth["firstname"],
                 'lastname': auth["lastname"],
                 'active': auth["active"],
                 'admin': auth["admin"],
                 'extern_name': auth["extern_name"],
                 'extern_type': self.name,
                 'plugin': self,
                 'allow_to_create_user': self.allows_creating_users,
             }
 
             if not is_user_existing:
                 if self.use_fake_password():
                     # Randomize the PW because we don't need it, but don't want
                     # them blank either
                     passwd = PasswordGenerator().gen_password(length=16)
                 user_parameters['password'] = passwd
             else:
                 # Since the password is required by create_or_update method of
                 # UserModel, we need to set it explicitly.
                 # The create_or_update method is smart and recognises the
                 # password hashes as well.
                 user_parameters['password'] = cur_user.password
 
             # we either create or update users, we also pass the flag
             # that controls if this method can actually do that.
             # raises NotAllowedToCreateUserError if it cannot, and we try to.
             user = UserModel().create_or_update(**user_parameters)
             Session().flush()
             # enforce user is just in given groups, all of them has to be ones
             # created from plugins. We store this info in _group_data JSON
             # field
             try:
                 groups = auth['groups'] or []
                 UserGroupModel().enforce_groups(user, groups, self.name)
             except Exception:
                 # for any reason group syncing fails, we should
                 # proceed with login
                 log.error(traceback.format_exc())
             Session().commit()
         return auth
 
 
-class AuthomaticBase(RhodeCodeExternalAuthPlugin):
-
-    # TODO: Think about how to create and store this secret string.
-    # We need the secret for the authomatic library. It needs to be the same
-    # across requests.
-    def _get_authomatic_secret(self, length=40):
-        secret = self.get_setting_by_name('secret')
-        if secret is None or secret == 'None' or secret == '':
-            from Crypto import Random, Hash
-            secret_bytes = Random.new().read(length)
-            secret_hash = Hash.SHA256.new()
-            secret_hash.update(secret_bytes)
-            secret = secret_hash.hexdigest()
-            self.create_or_update_setting('secret', secret)
-            Session.commit()
-            secret = self.get_setting_by_name('secret')
-        return secret
-
-    def get_authomatic(self):
-        scope = []
-        if self.name == 'bitbucket':
-            provider_class = oauth1.Bitbucket
-            scope = ['account', 'email', 'repository', 'issue', 'issue:write']
-        elif self.name == 'github':
-            provider_class = oauth2.GitHub
-            scope = ['repo', 'public_repo', 'user:email']
-        elif self.name == 'google':
-            provider_class = oauth2.Google
-            scope = ['profile', 'email']
-        elif self.name == 'twitter':
-            provider_class = oauth1.Twitter
-
-        authomatic_conf = {
-            self.name: {
-                'class_': provider_class,
-                'consumer_key': self.get_setting_by_name('consumer_key'),
-                'consumer_secret': self.get_setting_by_name('consumer_secret'),
-                'scope': scope,
-                'access_headers': {'User-Agent': 'TestAppAgent'},
-            }
-        }
-        secret = self._get_authomatic_secret()
-        return Authomatic(config=authomatic_conf,
-                          secret=secret)
-
-    def get_provider_result(self, request):
-        """
-        Provides `authomatic.core.LoginResult` for provider and request
-
-        :param provider_name:
-        :param request:
-        :param config:
-        :return:
-        """
-        response = Response()
-        adapter = WebObAdapter(request, response)
-        authomatic_inst = self.get_authomatic()
-        return authomatic_inst.login(adapter, self.name), response
-
-    def handle_social_data(self, session, user_id, social_data):
-        """
-        Updates user tokens in database whenever necessary
-        :param request:
-        :param user:
-        :param social_data:
-        :return:
-        """
-        if not self.is_active():
-            h.flash(_('This provider is currently disabled'),
-                    category='warning')
-            return False
-
-        social_data = social_data
-        update_identity = False
-
-        existing_row = ExternalIdentity.by_external_id_and_provider(
-            social_data['user']['id'],
-            social_data['credentials.provider']
-        )
-
-        if existing_row:
-            Session().delete(existing_row)
-            update_identity = True
-
-        if not existing_row or update_identity:
-            if not update_identity:
-                h.flash(_('Your external identity is now '
-                          'connected with your account'), category='success')
-
-            if not social_data['user']['id']:
-                h.flash(_('No external user id found? Perhaps permissions'
-                          'for authentication are set incorrectly'),
-                        category='error')
-                return False
-
-            ex_identity = ExternalIdentity()
-            ex_identity.external_id = social_data['user']['id']
-            ex_identity.external_username = social_data['user']['user_name']
-            ex_identity.provider_name = social_data['credentials.provider']
-            ex_identity.access_token = social_data['credentials.token']
-            ex_identity.token_secret = social_data['credentials.token_secret']
-            ex_identity.alt_token = social_data['credentials.refresh_token']
-            ex_identity.local_user_id = user_id
-            Session().add(ex_identity)
-            session.pop('rhodecode.social_auth', None)
-            return ex_identity
-
-    def callback_url(self):
-        try:
-            return url('social_auth', provider_name=self.name, qualified=True)
-        except TypeError:
-            pass
-        return ''
-
-
 def loadplugin(plugin_id):
     """
     Loads and returns an instantiated authentication plugin.
     Returns the RhodeCodeAuthPluginBase subclass on success,
-    raises exceptions on failure.
-
-    raises:
-        KeyError -- if no plugin available with given name
-        TypeError -- if the RhodeCodeAuthPlugin is not a subclass of
-                     ours RhodeCodeAuthPluginBase
+    or None on failure.
     """
     # TODO: Disusing pyramids thread locals to retrieve the registry.
     authn_registry = get_current_registry().getUtility(IAuthnPluginRegistry)
     plugin = authn_registry.get_plugin(plugin_id)
     if plugin is None:
         log.error('Authentication plugin not found: "%s"', plugin_id)
     return plugin
 
 
 def get_auth_cache_manager(custom_ttl=None):
     return caches.get_cache_manager(
         'auth_plugins', 'rhodecode.authentication', custom_ttl)
 
 
 def authenticate(username, password, environ=None, auth_type=None,
                  skip_missing=False):
     """
     Authentication function used for access control,
     It tries to authenticate based on enabled authentication modules.
 
-    :param username: username can be empty for container auth
-    :param password: password can be empty for container auth
-    :param environ: environ headers passed for container auth
+    :param username: username can be empty for headers auth
+    :param password: password can be empty for headers auth
+    :param environ: environ headers passed for headers auth
     :param auth_type: type of authentication, either `HTTP_TYPE` or `VCS_TYPE`
     :param skip_missing: ignores plugins that are in db but not in environment
     :returns: None if auth failed, plugin_user dict if auth is correct
     """
     if not auth_type or auth_type not in [HTTP_TYPE, VCS_TYPE]:
         raise ValueError('auth type must be on of http, vcs got "%s" instead'
                          % auth_type)
-    container_only = environ and not (username and password)
-    auth_plugins = SettingsModel().get_auth_plugins()
-    for plugin_id in auth_plugins:
-        plugin = loadplugin(plugin_id)
+    headers_only = environ and not (username and password)
 
-        if plugin is None:
-            log.warning('Authentication plugin missing: "{}"'.format(
-                plugin_id))
-            continue
-
-        if not plugin.is_active():
-            log.info('Authentication plugin is inactive: "{}"'.format(
-                plugin_id))
-            continue
-
+    authn_registry = get_current_registry().getUtility(IAuthnPluginRegistry)
+    for plugin in authn_registry.get_plugins_for_authentication():
         plugin.set_auth_type(auth_type)
         user = plugin.get_user(username)
         display_user = user.username if user else username
 
-        if container_only and not plugin.is_container_auth:
-            log.debug('Auth type is for container only and plugin `%s` is not '
-                      'container plugin, skipping...', plugin_id)
+        if headers_only and not plugin.is_headers_auth:
+            log.debug('Auth type is for headers only and plugin `%s` is not '
+                      'headers plugin, skipping...', plugin.get_id())
             continue
 
         # load plugin settings from RhodeCode database
         plugin_settings = plugin.get_settings()
         log.debug('Plugin settings:%s', plugin_settings)
 
-        log.debug('Trying authentication using ** %s **', plugin_id)
+        log.debug('Trying authentication using ** %s **', plugin.get_id())
         # use plugin's method of user extraction.
         user = plugin.get_user(username, environ=environ,
                                settings=plugin_settings)
         display_user = user.username if user else username
-        log.debug('Plugin %s extracted user is `%s`', plugin_id, display_user)
+        log.debug(
+            'Plugin %s extracted user is `%s`', plugin.get_id(), display_user)
 
         if not plugin.allows_authentication_from(user):
             log.debug('Plugin %s does not accept user `%s` for authentication',
-                      plugin_id, display_user)
+                      plugin.get_id(), display_user)
             continue
         else:
             log.debug('Plugin %s accepted user `%s` for authentication',
-                      plugin_id, display_user)
+                      plugin.get_id(), display_user)
 
         log.info('Authenticating user `%s` using %s plugin',
-                 display_user, plugin_id)
+                 display_user, plugin.get_id())
 
         _cache_ttl = 0
 
         if isinstance(plugin.AUTH_CACHE_TTL, (int, long)):
             # plugin cache set inside is more important than the settings value
             _cache_ttl = plugin.AUTH_CACHE_TTL
         elif plugin_settings.get('auth_cache_ttl'):
             _cache_ttl = safe_int(plugin_settings.get('auth_cache_ttl'), 0)
 
         plugin_cache_active = bool(_cache_ttl and _cache_ttl > 0)
 
         # get instance of cache manager configured for a namespace
         cache_manager = get_auth_cache_manager(custom_ttl=_cache_ttl)
 
-        log.debug('Cache for plugin `%s` active: %s', plugin_id,
+        log.debug('Cache for plugin `%s` active: %s', plugin.get_id(),
                   plugin_cache_active)
 
         # for environ based password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
         _password_hash = md5_safe(plugin.name + username + (password or ''))
 
         # _authenticate is a wrapper for .auth() method of plugin.
         # it checks if .auth() sends proper data.
         # For RhodeCodeExternalAuthPlugin it also maps users to
         # Database and maps the attributes returned from .auth()
         # to RhodeCode database. If this function returns data
         # then auth is correct.
         start = time.time()
         log.debug('Running plugin `%s` _authenticate method',
-                  plugin_id)
+                  plugin.get_id())
 
         def auth_func():
             """
             This function is used internally in Cache of Beaker to calculate
             Results
             """
             return plugin._authenticate(
                 user, username, password, plugin_settings,
                 environ=environ or {})
 
         if plugin_cache_active:
             plugin_user = cache_manager.get(
                 _password_hash, createfunc=auth_func)
         else:
             plugin_user = auth_func()
 
         auth_time = time.time() - start
         log.debug('Authentication for plugin `%s` completed in %.3fs, '
                   'expiration time of fetched cache %.1fs.',
-                  plugin_id, auth_time, _cache_ttl)
+                  plugin.get_id(), auth_time, _cache_ttl)
 
         log.debug('PLUGIN USER DATA: %s', plugin_user)
 
         if plugin_user:
             log.debug('Plugin returned proper authentication data')
             return plugin_user
         # we failed to Auth because .auth() method didn't return proper user
         log.debug("User `%s` failed to authenticate against %s",
-                  display_user, plugin_id)
+                  display_user, plugin.get_id())
     return None

rhodecode/authentication/plugins/auth_crowd.py

@@ -1,276 +1,284 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode authentication plugin for Atlassian CROWD
 """
 
 
 import colander
 import base64
 import logging
 import urllib2
 
 from pylons.i18n.translation import lazy_ugettext as _
 from sqlalchemy.ext.hybrid import hybrid_property
 
 from rhodecode.authentication.base import RhodeCodeExternalAuthPlugin
 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
 from rhodecode.authentication.routes import AuthnPluginResourceBase
+from rhodecode.lib.colander_utils import strip_whitespace
 from rhodecode.lib.ext_json import json, formatted_json
 from rhodecode.model.db import User
 
 log = logging.getLogger(__name__)
 
 
 def plugin_factory(plugin_id, *args, **kwds):
     """
     Factory function that is called during plugin discovery.
     It returns the plugin instance.
     """
     plugin = RhodeCodeAuthPlugin(plugin_id)
     return plugin
 
 
 class CrowdAuthnResource(AuthnPluginResourceBase):
     pass
 
 
 class CrowdSettingsSchema(AuthnPluginSettingsSchemaBase):
     host = colander.SchemaNode(
         colander.String(),
         default='127.0.0.1',
         description=_('The FQDN or IP of the Atlassian CROWD Server'),
+        preparer=strip_whitespace,
         title=_('Host'),
         widget='string')
     port = colander.SchemaNode(
         colander.Int(),
         default=8095,
         description=_('The Port in use by the Atlassian CROWD Server'),
+        preparer=strip_whitespace,
         title=_('Port'),
         validator=colander.Range(min=0, max=65536),
         widget='int')
     app_name = colander.SchemaNode(
         colander.String(),
         default='',
         description=_('The Application Name to authenticate to CROWD'),
+        preparer=strip_whitespace,
         title=_('Application Name'),
         widget='string')
     app_password = colander.SchemaNode(
         colander.String(),
         default='',
         description=_('The password to authenticate to CROWD'),
+        preparer=strip_whitespace,
         title=_('Application Password'),
         widget='password')
     admin_groups = colander.SchemaNode(
         colander.String(),
         default='',
         description=_('A comma separated list of group names that identify '
                       'users as RhodeCode Administrators'),
         missing='',
+        preparer=strip_whitespace,
         title=_('Admin Groups'),
         widget='string')
 
 
 class CrowdServer(object):
     def __init__(self, *args, **kwargs):
         """
         Create a new CrowdServer object that points to IP/Address 'host',
         on the given port, and using the given method (https/http). user and
         passwd can be set here or with set_credentials. If unspecified,
         "version" defaults to "latest".
 
         example::
 
             cserver = CrowdServer(host="127.0.0.1",
                                   port="8095",
                                   user="some_app",
                                   passwd="some_passwd",
                                   version="1")
         """
         if not "port" in kwargs:
             kwargs["port"] = "8095"
         self._logger = kwargs.get("logger", logging.getLogger(__name__))
         self._uri = "%s://%s:%s/crowd" % (kwargs.get("method", "http"),
                                     kwargs.get("host", "127.0.0.1"),
                                     kwargs.get("port", "8095"))
         self.set_credentials(kwargs.get("user", ""),
                              kwargs.get("passwd", ""))
         self._version = kwargs.get("version", "latest")
         self._url_list = None
         self._appname = "crowd"
 
     def set_credentials(self, user, passwd):
         self.user = user
         self.passwd = passwd
         self._make_opener()
 
     def _make_opener(self):
         mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
         mgr.add_password(None, self._uri, self.user, self.passwd)
         handler = urllib2.HTTPBasicAuthHandler(mgr)
         self.opener = urllib2.build_opener(handler)
 
     def _request(self, url, body=None, headers=None,
                  method=None, noformat=False,
                  empty_response_ok=False):
         _headers = {"Content-type": "application/json",
                     "Accept": "application/json"}
         if self.user and self.passwd:
             authstring = base64.b64encode("%s:%s" % (self.user, self.passwd))
             _headers["Authorization"] = "Basic %s" % authstring
         if headers:
             _headers.update(headers)
         log.debug("Sent crowd: \n%s"
                   % (formatted_json({"url": url, "body": body,
                                            "headers": _headers})))
         request = urllib2.Request(url, body, _headers)
         if method:
             request.get_method = lambda: method
 
         global msg
         msg = ""
         try:
             rdoc = self.opener.open(request)
             msg = "".join(rdoc.readlines())
             if not msg and empty_response_ok:
                 rval = {}
                 rval["status"] = True
                 rval["error"] = "Response body was empty"
             elif not noformat:
                 rval = json.loads(msg)
                 rval["status"] = True
             else:
                 rval = "".join(rdoc.readlines())
         except Exception as e:
             if not noformat:
                 rval = {"status": False,
                         "body": body,
                         "error": str(e) + "\n" + msg}
             else:
                 rval = None
         return rval
 
     def user_auth(self, username, password):
         """Authenticate a user against crowd. Returns brief information about
         the user."""
         url = ("%s/rest/usermanagement/%s/authentication?username=%s"
                % (self._uri, self._version, username))
         body = json.dumps({"value": password})
         return self._request(url, body)
 
     def user_groups(self, username):
         """Retrieve a list of groups to which this user belongs."""
         url = ("%s/rest/usermanagement/%s/user/group/nested?username=%s"
                % (self._uri, self._version, username))
         return self._request(url)
 
 
 class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
 
     def includeme(self, config):
         config.add_authn_plugin(self)
         config.add_authn_resource(self.get_id(), CrowdAuthnResource(self))
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_get',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
             request_method='GET',
             route_name='auth_home',
             context=CrowdAuthnResource)
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_post',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
             request_method='POST',
             route_name='auth_home',
             context=CrowdAuthnResource)
 
     def get_settings_schema(self):
         return CrowdSettingsSchema()
 
     def get_display_name(self):
         return _('CROWD')
 
     @hybrid_property
     def name(self):
         return "crowd"
 
     def use_fake_password(self):
         return True
 
     def user_activation_state(self):
         def_user_perms = User.get_default_user().AuthUser.permissions['global']
         return 'hg.extern_activate.auto' in def_user_perms
 
     def auth(self, userobj, username, password, settings, **kwargs):
         """
         Given a user object (which may be null), username, a plaintext password,
         and a settings object (containing all the keys needed as listed in settings()),
         authenticate this user's login attempt.
 
         Return None on failure. On success, return a dictionary of the form:
 
             see: RhodeCodeAuthPluginBase.auth_func_attrs
         This is later validated for correctness
         """
         if not username or not password:
             log.debug('Empty username or password skipping...')
             return None
 
         log.debug("Crowd settings: \n%s" % (formatted_json(settings)))
         server = CrowdServer(**settings)
         server.set_credentials(settings["app_name"], settings["app_password"])
         crowd_user = server.user_auth(username, password)
         log.debug("Crowd returned: \n%s" % (formatted_json(crowd_user)))
         if not crowd_user["status"]:
             return None
 
         res = server.user_groups(crowd_user["name"])
         log.debug("Crowd groups: \n%s" % (formatted_json(res)))
         crowd_user["groups"] = [x["name"] for x in res["groups"]]
 
         # old attrs fetched from RhodeCode database
         admin = getattr(userobj, 'admin', False)
         active = getattr(userobj, 'active', True)
         email = getattr(userobj, 'email', '')
         username = getattr(userobj, 'username', username)
         firstname = getattr(userobj, 'firstname', '')
         lastname = getattr(userobj, 'lastname', '')
         extern_type = getattr(userobj, 'extern_type', '')
 
         user_attrs = {
             'username': username,
             'firstname': crowd_user["first-name"] or firstname,
             'lastname': crowd_user["last-name"] or lastname,
             'groups': crowd_user["groups"],
             'email': crowd_user["email"] or email,
             'admin': admin,
             'active': active,
             'active_from_extern': crowd_user.get('active'),
             'extern_name': crowd_user["name"],
             'extern_type': extern_type,
         }
 
         # set an admin if we're in admin_groups of crowd
         for group in settings["admin_groups"]:
             if group in user_attrs["groups"]:
                 user_attrs["admin"] = True
         log.debug("Final crowd user object: \n%s" % (formatted_json(user_attrs)))
         log.info('user %s authenticated correctly' % user_attrs['username'])
         return user_attrs

rhodecode/authentication/plugins/auth_jasig_cas.py

@@ -1,163 +1,167 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode authentication plugin for Jasig CAS
 http://www.jasig.org/cas
 """
 
 
 import colander
 import logging
 import rhodecode
 import urllib
 import urllib2
 
 from pylons.i18n.translation import lazy_ugettext as _
 from sqlalchemy.ext.hybrid import hybrid_property
 
 from rhodecode.authentication.base import RhodeCodeExternalAuthPlugin
 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
 from rhodecode.authentication.routes import AuthnPluginResourceBase
+from rhodecode.lib.colander_utils import strip_whitespace
 from rhodecode.lib.utils2 import safe_unicode
 from rhodecode.model.db import User
 
 log = logging.getLogger(__name__)
 
 
 def plugin_factory(plugin_id, *args, **kwds):
     """
     Factory function that is called during plugin discovery.
     It returns the plugin instance.
     """
     plugin = RhodeCodeAuthPlugin(plugin_id)
     return plugin
 
 
 class JasigCasAuthnResource(AuthnPluginResourceBase):
     pass
 
 
 class JasigCasSettingsSchema(AuthnPluginSettingsSchemaBase):
     service_url = colander.SchemaNode(
         colander.String(),
         default='https://domain.com/cas/v1/tickets',
         description=_('The url of the Jasig CAS REST service'),
+        preparer=strip_whitespace,
         title=_('URL'),
         widget='string')
 
 
 class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
 
     def includeme(self, config):
         config.add_authn_plugin(self)
         config.add_authn_resource(self.get_id(), JasigCasAuthnResource(self))
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_get',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
             request_method='GET',
             route_name='auth_home',
             context=JasigCasAuthnResource)
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_post',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
             request_method='POST',
             route_name='auth_home',
             context=JasigCasAuthnResource)
 
     def get_settings_schema(self):
         return JasigCasSettingsSchema()
 
     def get_display_name(self):
         return _('Jasig-CAS')
 
     @hybrid_property
     def name(self):
         return "jasig-cas"
 
-    @hybrid_property
-    def is_container_auth(self):
+    @property
+    def is_headers_auth(self):
         return True
 
     def use_fake_password(self):
         return True
 
     def user_activation_state(self):
         def_user_perms = User.get_default_user().AuthUser.permissions['global']
         return 'hg.extern_activate.auto' in def_user_perms
 
     def auth(self, userobj, username, password, settings, **kwargs):
         """
         Given a user object (which may be null), username, a plaintext password,
         and a settings object (containing all the keys needed as listed in settings()),
         authenticate this user's login attempt.
 
         Return None on failure. On success, return a dictionary of the form:
 
             see: RhodeCodeAuthPluginBase.auth_func_attrs
         This is later validated for correctness
         """
         if not username or not password:
             log.debug('Empty username or password skipping...')
             return None
 
         log.debug("Jasig CAS settings: %s", settings)
         params = urllib.urlencode({'username': username, 'password': password})
         headers = {"Content-type": "application/x-www-form-urlencoded",
                    "Accept": "text/plain",
                    "User-Agent": "RhodeCode-auth-%s" % rhodecode.__version__}
         url = settings["service_url"]
 
         log.debug("Sent Jasig CAS: \n%s",
                   {"url": url, "body": params, "headers": headers})
         request = urllib2.Request(url, params, headers)
         try:
             response = urllib2.urlopen(request)
         except urllib2.HTTPError as e:
             log.debug("HTTPError when requesting Jasig CAS (status code: %d)" % e.code)
             return None
         except urllib2.URLError as e:
             log.debug("URLError when requesting Jasig CAS url: %s " % url)
             return None
 
         # old attrs fetched from RhodeCode database
         admin = getattr(userobj, 'admin', False)
         active = getattr(userobj, 'active', True)
         email = getattr(userobj, 'email', '')
         username = getattr(userobj, 'username', username)
         firstname = getattr(userobj, 'firstname', '')
         lastname = getattr(userobj, 'lastname', '')
         extern_type = getattr(userobj, 'extern_type', '')
 
         user_attrs = {
             'username': username,
             'firstname': safe_unicode(firstname or username),
             'lastname': safe_unicode(lastname or ''),
             'groups': [],
             'email': email or '',
             'admin': admin or False,
             'active': active,
             'active_from_extern': True,
             'extern_name': username,
             'extern_type': extern_type,
         }
 
         log.info('user %s authenticated correctly' % user_attrs['username'])
         return user_attrs

rhodecode/authentication/plugins/auth_ldap.py

@@ -1,447 +1,461 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode authentication plugin for LDAP
 """
 
 
 import colander
 import logging
 import traceback
 
 from pylons.i18n.translation import lazy_ugettext as _
 from sqlalchemy.ext.hybrid import hybrid_property
 
 from rhodecode.authentication.base import RhodeCodeExternalAuthPlugin
 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
 from rhodecode.authentication.routes import AuthnPluginResourceBase
+from rhodecode.lib.colander_utils import strip_whitespace
 from rhodecode.lib.exceptions import (
     LdapConnectionError, LdapUsernameError, LdapPasswordError, LdapImportError
 )
 from rhodecode.lib.utils2 import safe_unicode, safe_str
 from rhodecode.model.db import User
 from rhodecode.model.validators import Missing
 
 log = logging.getLogger(__name__)
 
 try:
     import ldap
 except ImportError:
-    # means that python-ldap is not installed
-    ldap = Missing()
+    # means that python-ldap is not installed, we use Missing object to mark
+    # ldap lib is Missing
+    ldap = Missing
 
 
 def plugin_factory(plugin_id, *args, **kwds):
     """
     Factory function that is called during plugin discovery.
     It returns the plugin instance.
     """
     plugin = RhodeCodeAuthPlugin(plugin_id)
     return plugin
 
 
 class LdapAuthnResource(AuthnPluginResourceBase):
     pass
 
 
 class LdapSettingsSchema(AuthnPluginSettingsSchemaBase):
     tls_kind_choices = ['PLAIN', 'LDAPS', 'START_TLS']
     tls_reqcert_choices = ['NEVER', 'ALLOW', 'TRY', 'DEMAND', 'HARD']
     search_scope_choices = ['BASE', 'ONELEVEL', 'SUBTREE']
 
     host = colander.SchemaNode(
         colander.String(),
         default='',
         description=_('Host of the LDAP Server'),
+        preparer=strip_whitespace,
         title=_('LDAP Host'),
         widget='string')
     port = colander.SchemaNode(
         colander.Int(),
         default=389,
         description=_('Port that the LDAP server is listening on'),
+        preparer=strip_whitespace,
         title=_('Port'),
         validator=colander.Range(min=0, max=65536),
         widget='int')
     dn_user = colander.SchemaNode(
         colander.String(),
         default='',
         description=_('User to connect to LDAP'),
         missing='',
+        preparer=strip_whitespace,
         title=_('Account'),
         widget='string')
     dn_pass = colander.SchemaNode(
         colander.String(),
         default='',
         description=_('Password to connect to LDAP'),
         missing='',
+        preparer=strip_whitespace,
         title=_('Password'),
         widget='password')
     tls_kind = colander.SchemaNode(
         colander.String(),
         default=tls_kind_choices[0],
         description=_('TLS Type'),
         title=_('Connection Security'),
         validator=colander.OneOf(tls_kind_choices),
         widget='select')
     tls_reqcert = colander.SchemaNode(
         colander.String(),
         default=tls_reqcert_choices[0],
         description=_('Require Cert over TLS?'),
         title=_('Certificate Checks'),
         validator=colander.OneOf(tls_reqcert_choices),
         widget='select')
     base_dn = colander.SchemaNode(
         colander.String(),
         default='',
         description=_('Base DN to search (e.g., dc=mydomain,dc=com)'),
         missing='',
+        preparer=strip_whitespace,
         title=_('Base DN'),
         widget='string')
     filter = colander.SchemaNode(
         colander.String(),
         default='',
         description=_('Filter to narrow results (e.g., ou=Users, etc)'),
         missing='',
+        preparer=strip_whitespace,
         title=_('LDAP Search Filter'),
         widget='string')
     search_scope = colander.SchemaNode(
         colander.String(),
         default=search_scope_choices[0],
         description=_('How deep to search LDAP'),
         title=_('LDAP Search Scope'),
         validator=colander.OneOf(search_scope_choices),
         widget='select')
     attr_login = colander.SchemaNode(
         colander.String(),
         default='',
         description=_('LDAP Attribute to map to user name'),
+        missing_msg=_('The LDAP Login attribute of the CN must be specified'),
+        preparer=strip_whitespace,
         title=_('Login Attribute'),
-        missing_msg=_('The LDAP Login attribute of the CN must be specified'),
         widget='string')
     attr_firstname = colander.SchemaNode(
         colander.String(),
         default='',
         description=_('LDAP Attribute to map to first name'),
         missing='',
+        preparer=strip_whitespace,
         title=_('First Name Attribute'),
         widget='string')
     attr_lastname = colander.SchemaNode(
         colander.String(),
         default='',
         description=_('LDAP Attribute to map to last name'),
         missing='',
+        preparer=strip_whitespace,
         title=_('Last Name Attribute'),
         widget='string')
     attr_email = colander.SchemaNode(
         colander.String(),
         default='',
         description=_('LDAP Attribute to map to email address'),
         missing='',
+        preparer=strip_whitespace,
         title=_('Email Attribute'),
         widget='string')
 
 
 class AuthLdap(object):
 
     def _build_servers(self):
         return ', '.join(
             ["{}://{}:{}".format(
                 self.ldap_server_type, host.strip(), self.LDAP_SERVER_PORT)
              for host in self.SERVER_ADDRESSES])
 
     def __init__(self, server, base_dn, port=389, bind_dn='', bind_pass='',
                  tls_kind='PLAIN', tls_reqcert='DEMAND', ldap_version=3,
                  search_scope='SUBTREE', attr_login='uid',
                  ldap_filter='(&(objectClass=user)(!(objectClass=computer)))'):
-        if isinstance(ldap, Missing):
+        if ldap == Missing:
             raise LdapImportError("Missing or incompatible ldap library")
 
         self.ldap_version = ldap_version
         self.ldap_server_type = 'ldap'
 
         self.TLS_KIND = tls_kind
 
         if self.TLS_KIND == 'LDAPS':
             port = port or 689
             self.ldap_server_type += 's'
 
         OPT_X_TLS_DEMAND = 2
         self.TLS_REQCERT = getattr(ldap, 'OPT_X_TLS_%s' % tls_reqcert,
                                    OPT_X_TLS_DEMAND)
         # split server into list
         self.SERVER_ADDRESSES = server.split(',')
         self.LDAP_SERVER_PORT = port
 
         # USE FOR READ ONLY BIND TO LDAP SERVER
         self.attr_login = attr_login
 
         self.LDAP_BIND_DN = safe_str(bind_dn)
         self.LDAP_BIND_PASS = safe_str(bind_pass)
         self.LDAP_SERVER = self._build_servers()
         self.SEARCH_SCOPE = getattr(ldap, 'SCOPE_%s' % search_scope)
         self.BASE_DN = safe_str(base_dn)
         self.LDAP_FILTER = safe_str(ldap_filter)
 
     def _get_ldap_server(self):
         if hasattr(ldap, 'OPT_X_TLS_CACERTDIR'):
             ldap.set_option(ldap.OPT_X_TLS_CACERTDIR,
                             '/etc/openldap/cacerts')
         ldap.set_option(ldap.OPT_REFERRALS, ldap.OPT_OFF)
         ldap.set_option(ldap.OPT_RESTART, ldap.OPT_ON)
         ldap.set_option(ldap.OPT_TIMEOUT, 20)
         ldap.set_option(ldap.OPT_NETWORK_TIMEOUT, 10)
         ldap.set_option(ldap.OPT_TIMELIMIT, 15)
         if self.TLS_KIND != 'PLAIN':
             ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, self.TLS_REQCERT)
         server = ldap.initialize(self.LDAP_SERVER)
         if self.ldap_version == 2:
             server.protocol = ldap.VERSION2
         else:
             server.protocol = ldap.VERSION3
 
         if self.TLS_KIND == 'START_TLS':
             server.start_tls_s()
 
         if self.LDAP_BIND_DN and self.LDAP_BIND_PASS:
             log.debug('Trying simple_bind with password and given DN: %s',
                       self.LDAP_BIND_DN)
             server.simple_bind_s(self.LDAP_BIND_DN, self.LDAP_BIND_PASS)
 
         return server
 
     def get_uid(self, username):
         from rhodecode.lib.helpers import chop_at
         uid = username
         for server_addr in self.SERVER_ADDRESSES:
             uid = chop_at(username, "@%s" % server_addr)
         return uid
 
     def fetch_attrs_from_simple_bind(self, server, dn, username, password):
         try:
             log.debug('Trying simple bind with %s', dn)
             server.simple_bind_s(dn, safe_str(password))
             user = server.search_ext_s(
                 dn, ldap.SCOPE_BASE, '(objectClass=*)', )[0]
             _, attrs = user
             return attrs
 
         except ldap.INVALID_CREDENTIALS:
             log.debug(
                 "LDAP rejected password for user '%s': %s, org_exc:",
                 username, dn, exc_info=True)
 
     def authenticate_ldap(self, username, password):
         """
         Authenticate a user via LDAP and return his/her LDAP properties.
 
         Raises AuthenticationError if the credentials are rejected, or
         EnvironmentError if the LDAP server can't be reached.
 
         :param username: username
         :param password: password
         """
 
         uid = self.get_uid(username)
 
         if not password:
             msg = "Authenticating user %s with blank password not allowed"
             log.warning(msg, username)
             raise LdapPasswordError(msg)
         if "," in username:
             raise LdapUsernameError("invalid character in username: ,")
         try:
             server = self._get_ldap_server()
             filter_ = '(&%s(%s=%s))' % (
                 self.LDAP_FILTER, self.attr_login, username)
             log.debug("Authenticating %r filter %s at %s", self.BASE_DN,
                       filter_, self.LDAP_SERVER)
             lobjects = server.search_ext_s(
                 self.BASE_DN, self.SEARCH_SCOPE, filter_)
 
             if not lobjects:
                 raise ldap.NO_SUCH_OBJECT()
 
             for (dn, _attrs) in lobjects:
                 if dn is None:
                     continue
 
                 user_attrs = self.fetch_attrs_from_simple_bind(
                     server, dn, username, password)
                 if user_attrs:
                     break
 
             else:
                 log.debug("No matching LDAP objects for authentication "
                           "of '%s' (%s)", uid, username)
                 raise LdapPasswordError('Failed to authenticate user '
                                         'with given password')
 
         except ldap.NO_SUCH_OBJECT:
             log.debug("LDAP says no such user '%s' (%s), org_exc:",
                       uid, username, exc_info=True)
             raise LdapUsernameError()
         except ldap.SERVER_DOWN:
             org_exc = traceback.format_exc()
             raise LdapConnectionError(
                 "LDAP can't access authentication "
                 "server, org_exc:%s" % org_exc)
 
         return dn, user_attrs
 
 
 class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
     # used to define dynamic binding in the
     DYNAMIC_BIND_VAR = '$login'
 
     def includeme(self, config):
         config.add_authn_plugin(self)
         config.add_authn_resource(self.get_id(), LdapAuthnResource(self))
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_get',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
             request_method='GET',
             route_name='auth_home',
             context=LdapAuthnResource)
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_post',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
             request_method='POST',
             route_name='auth_home',
             context=LdapAuthnResource)
 
     def get_settings_schema(self):
         return LdapSettingsSchema()
 
     def get_display_name(self):
         return _('LDAP')
 
     @hybrid_property
     def name(self):
         return "ldap"
 
     def use_fake_password(self):
         return True
 
     def user_activation_state(self):
         def_user_perms = User.get_default_user().AuthUser.permissions['global']
         return 'hg.extern_activate.auto' in def_user_perms
 
     def try_dynamic_binding(self, username, password, current_args):
         """
         Detects marker inside our original bind, and uses dynamic auth if
         present
         """
 
         org_bind = current_args['bind_dn']
         passwd = current_args['bind_pass']
 
         def has_bind_marker(username):
             if self.DYNAMIC_BIND_VAR in username:
                 return True
 
         # we only passed in user with "special" variable
         if org_bind and has_bind_marker(org_bind) and not passwd:
             log.debug('Using dynamic user/password binding for ldap '
                       'authentication. Replacing `%s` with username',
                       self.DYNAMIC_BIND_VAR)
             current_args['bind_dn'] = org_bind.replace(
                 self.DYNAMIC_BIND_VAR, username)
             current_args['bind_pass'] = password
 
         return current_args
 
     def auth(self, userobj, username, password, settings, **kwargs):
         """
         Given a user object (which may be null), username, a plaintext password,
         and a settings object (containing all the keys needed as listed in
         settings()), authenticate this user's login attempt.
 
         Return None on failure. On success, return a dictionary of the form:
 
             see: RhodeCodeAuthPluginBase.auth_func_attrs
         This is later validated for correctness
         """
 
         if not username or not password:
             log.debug('Empty username or password skipping...')
             return None
 
         ldap_args = {
             'server': settings.get('host', ''),
             'base_dn': settings.get('base_dn', ''),
             'port': settings.get('port'),
             'bind_dn': settings.get('dn_user'),
             'bind_pass': settings.get('dn_pass'),
             'tls_kind': settings.get('tls_kind'),
             'tls_reqcert': settings.get('tls_reqcert'),
             'search_scope': settings.get('search_scope'),
             'attr_login': settings.get('attr_login'),
             'ldap_version': 3,
             'ldap_filter': settings.get('filter'),
         }
 
         ldap_attrs = self.try_dynamic_binding(username, password, ldap_args)
 
         log.debug('Checking for ldap authentication.')
 
         try:
             aldap = AuthLdap(**ldap_args)
             (user_dn, ldap_attrs) = aldap.authenticate_ldap(username, password)
             log.debug('Got ldap DN response %s', user_dn)
 
             def get_ldap_attr(k):
                 return ldap_attrs.get(settings.get(k), [''])[0]
 
             # old attrs fetched from RhodeCode database
             admin = getattr(userobj, 'admin', False)
             active = getattr(userobj, 'active', True)
             email = getattr(userobj, 'email', '')
             username = getattr(userobj, 'username', username)
             firstname = getattr(userobj, 'firstname', '')
             lastname = getattr(userobj, 'lastname', '')
             extern_type = getattr(userobj, 'extern_type', '')
 
             groups = []
             user_attrs = {
                 'username': username,
                 'firstname': safe_unicode(
                     get_ldap_attr('attr_firstname') or firstname),
                 'lastname': safe_unicode(
                     get_ldap_attr('attr_lastname') or lastname),
                 'groups': groups,
                 'email': get_ldap_attr('attr_email' or email),
                 'admin': admin,
                 'active': active,
                 "active_from_extern": None,
                 'extern_name': user_dn,
                 'extern_type': extern_type,
             }
             log.debug('ldap user: %s', user_attrs)
             log.info('user %s authenticated correctly', user_attrs['username'])
 
             return user_attrs
 
         except (LdapUsernameError, LdapPasswordError, LdapImportError):
             log.exception("LDAP related exception")
             return None
         except (Exception,):
             log.exception("Other exception")
             return None

rhodecode/authentication/plugins/auth_pam.py

@@ -1,155 +1,160 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 """
 RhodeCode authentication library for PAM
 """
 
 import colander
 import grp
 import logging
 import pam
 import pwd
 import re
 import socket
 
 from pylons.i18n.translation import lazy_ugettext as _
 from sqlalchemy.ext.hybrid import hybrid_property
 
 from rhodecode.authentication.base import RhodeCodeExternalAuthPlugin
 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
 from rhodecode.authentication.routes import AuthnPluginResourceBase
+from rhodecode.lib.colander_utils import strip_whitespace
 
 log = logging.getLogger(__name__)
 
 
 def plugin_factory(plugin_id, *args, **kwds):
     """
     Factory function that is called during plugin discovery.
     It returns the plugin instance.
     """
     plugin = RhodeCodeAuthPlugin(plugin_id)
     return plugin
 
 
 class PamAuthnResource(AuthnPluginResourceBase):
     pass
 
 
 class PamSettingsSchema(AuthnPluginSettingsSchemaBase):
     service = colander.SchemaNode(
         colander.String(),
         default='login',
         description=_('PAM service name to use for authentication.'),
+        preparer=strip_whitespace,
         title=_('PAM service name'),
         widget='string')
     gecos = colander.SchemaNode(
         colander.String(),
         default='(?P<last_name>.+),\s*(?P<first_name>\w+)',
         description=_('Regular expression for extracting user name/email etc. '
                       'from Unix userinfo.'),
+        preparer=strip_whitespace,
         title=_('Gecos Regex'),
         widget='string')
 
 
 class RhodeCodeAuthPlugin(RhodeCodeExternalAuthPlugin):
     # PAM authentication can be slow. Repository operations involve a lot of
     # auth calls. Little caching helps speedup push/pull operations significantly
     AUTH_CACHE_TTL = 4
 
     def includeme(self, config):
         config.add_authn_plugin(self)
         config.add_authn_resource(self.get_id(), PamAuthnResource(self))
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_get',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
             request_method='GET',
             route_name='auth_home',
             context=PamAuthnResource)
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_post',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
             request_method='POST',
             route_name='auth_home',
             context=PamAuthnResource)
 
     def get_display_name(self):
         return _('PAM')
 
     @hybrid_property
     def name(self):
         return "pam"
 
     def get_settings_schema(self):
         return PamSettingsSchema()
 
     def use_fake_password(self):
         return True
 
     def auth(self, userobj, username, password, settings, **kwargs):
         if not username or not password:
             log.debug('Empty username or password skipping...')
             return None
 
         auth_result = pam.authenticate(username, password, settings["service"])
 
         if not auth_result:
             log.error("PAM was unable to authenticate user: %s" % (username, ))
             return None
 
         log.debug('Got PAM response %s' % (auth_result, ))
 
         # old attrs fetched from RhodeCode database
         default_email = "%s@%s" % (username, socket.gethostname())
         admin = getattr(userobj, 'admin', False)
         active = getattr(userobj, 'active', True)
         email = getattr(userobj, 'email', '') or default_email
         username = getattr(userobj, 'username', username)
         firstname = getattr(userobj, 'firstname', '')
         lastname = getattr(userobj, 'lastname', '')
         extern_type = getattr(userobj, 'extern_type', '')
 
         user_attrs = {
             'username': username,
             'firstname': firstname,
             'lastname': lastname,
             'groups': [g.gr_name for g in grp.getgrall()
                        if username in g.gr_mem],
             'email': email,
             'admin': admin,
             'active': active,
             'active_from_extern': None,
             'extern_name': username,
             'extern_type': extern_type,
         }
 
         try:
             user_data = pwd.getpwnam(username)
             regex = settings["gecos"]
             match = re.search(regex, user_data.pw_gecos)
             if match:
                 user_attrs["firstname"] = match.group('first_name')
                 user_attrs["lastname"] = match.group('last_name')
         except Exception:
             log.warning("Cannot extract additional info for PAM user")
             pass
 
         log.debug("pamuser: %s", user_attrs)
         log.info('user %s authenticated correctly' % user_attrs['username'])
         return user_attrs

rhodecode/authentication/plugins/auth_rhodecode.py

@@ -1,139 +1,141 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode authentication plugin for built in internal auth
 """
 
 import logging
 
 from pylons.i18n.translation import lazy_ugettext as _
 from sqlalchemy.ext.hybrid import hybrid_property
 
 from rhodecode.authentication.base import RhodeCodeAuthPluginBase
 from rhodecode.authentication.routes import AuthnPluginResourceBase
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.model.db import User
 
 log = logging.getLogger(__name__)
 
 
 def plugin_factory(plugin_id, *args, **kwds):
     plugin = RhodeCodeAuthPlugin(plugin_id)
     return plugin
 
 
 class RhodecodeAuthnResource(AuthnPluginResourceBase):
     pass
 
 
 class RhodeCodeAuthPlugin(RhodeCodeAuthPluginBase):
 
     def includeme(self, config):
         config.add_authn_plugin(self)
         config.add_authn_resource(self.get_id(), RhodecodeAuthnResource(self))
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_get',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
             request_method='GET',
             route_name='auth_home',
             context=RhodecodeAuthnResource)
         config.add_view(
             'rhodecode.authentication.views.AuthnPluginViewBase',
             attr='settings_post',
+            renderer='rhodecode:templates/admin/auth/plugin_settings.html',
             request_method='POST',
             route_name='auth_home',
             context=RhodecodeAuthnResource)
 
     def get_display_name(self):
         return _('Rhodecode')
 
     @hybrid_property
     def name(self):
         return "rhodecode"
 
     def user_activation_state(self):
         def_user_perms = User.get_default_user().AuthUser.permissions['global']
         return 'hg.register.auto_activate' in def_user_perms
 
     def allows_authentication_from(
             self, user, allows_non_existing_user=True,
             allowed_auth_plugins=None, allowed_auth_sources=None):
         """
         Custom method for this auth that doesn't accept non existing users.
         We know that user exists in our database.
         """
         allows_non_existing_user = False
         return super(RhodeCodeAuthPlugin, self).allows_authentication_from(
             user, allows_non_existing_user=allows_non_existing_user)
 
     def auth(self, userobj, username, password, settings, **kwargs):
         if not userobj:
             log.debug('userobj was:%s skipping' % (userobj, ))
             return None
         if userobj.extern_type != self.name:
             log.warning(
                 "userobj:%s extern_type mismatch got:`%s` expected:`%s`" %
                 (userobj, userobj.extern_type, self.name))
             return None
 
         user_attrs = {
             "username": userobj.username,
             "firstname": userobj.firstname,
             "lastname": userobj.lastname,
             "groups": [],
             "email": userobj.email,
             "admin": userobj.admin,
             "active": userobj.active,
             "active_from_extern": userobj.active,
             "extern_name": userobj.user_id,
             "extern_type": userobj.extern_type,
         }
 
         log.debug("User attributes:%s" % (user_attrs, ))
         if userobj.active:
             from rhodecode.lib import auth
             crypto_backend = auth.crypto_backend()
             password_encoded = safe_str(password)
             password_match, new_hash = crypto_backend.hash_check_with_upgrade(
                 password_encoded, userobj.password)
 
             if password_match and new_hash:
                 log.debug('user %s properly authenticated, but '
                           'requires hash change to bcrypt', userobj)
                 # if password match, and we use OLD deprecated hash,
                 # we should migrate this user hash password to the new hash
                 # we store the new returned by hash_check_with_upgrade function
                 user_attrs['_hash_migrate'] = new_hash
 
             if userobj.username == User.DEFAULT_USER and userobj.active:
                 log.info(
                     'user %s authenticated correctly as anonymous user', userobj)
                 return user_attrs
 
             elif userobj.username == username and password_match:
                 log.info('user %s authenticated correctly', userobj)
                 return user_attrs
             log.info("user %s had a bad password when "
                      "authenticating on this plugin", userobj)
             return None
         else:
             log.warning('user %s tried auth but is disabled', userobj)
             return None

rhodecode/authentication/registry.py

@@ -1,53 +1,87 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 from pyramid.exceptions import ConfigurationError
 from zope.interface import implementer
 
 from rhodecode.authentication.interface import IAuthnPluginRegistry
 from rhodecode.lib.utils2 import safe_str
+from rhodecode.model.settings import SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 @implementer(IAuthnPluginRegistry)
 class AuthenticationPluginRegistry(object):
-    def __init__(self):
+
+    # INI settings key to set a fallback authentication plugin.
+    fallback_plugin_key = 'rhodecode.auth_plugin_fallback'
+
+    def __init__(self, settings):
         self._plugins = {}
+        self._fallback_plugin = settings.get(self.fallback_plugin_key, None)
 
     def add_authn_plugin(self, config, plugin):
         plugin_id = plugin.get_id()
         if plugin_id in self._plugins.keys():
             raise ConfigurationError(
                 'Cannot register authentication plugin twice: "%s"', plugin_id)
         else:
             log.debug('Register authentication plugin: "%s"', plugin_id)
             self._plugins[plugin_id] = plugin
 
     def get_plugins(self):
         def sort_key(plugin):
             return str.lower(safe_str(plugin.get_display_name()))
 
         return sorted(self._plugins.values(), key=sort_key)
 
     def get_plugin(self, plugin_id):
         return self._plugins.get(plugin_id, None)
+
+    def get_plugins_for_authentication(self):
+        """
+        Returns a list of plugins which should be consulted when authenticating
+        a user. It only returns plugins which are enabled and active.
+        Additionally it includes the fallback plugin from the INI file, if
+        `rhodecode.auth_plugin_fallback` is set to a plugin ID.
+        """
+        plugins = []
+
+        # Add all enabled and active plugins to the list. We iterate over the
+        # auth_plugins setting from DB beacuse it also represents the ordering.
+        enabled_plugins = SettingsModel().get_auth_plugins()
+        for plugin_id in enabled_plugins:
+            plugin = self.get_plugin(plugin_id)
+            if plugin is not None and plugin.is_active():
+                plugins.append(plugin)
+
+        # Add the fallback plugin from ini file.
+        if self._fallback_plugin:
+            log.warn(
+                'Using fallback authentication plugin from INI file: "%s"',
+                self._fallback_plugin)
+            plugin = self.get_plugin(self._fallback_plugin)
+            if plugin is not None and plugin not in plugins:
+                plugins.append(plugin)
+
+        return plugins

rhodecode/authentication/routes.py

@@ -1,151 +1,150 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 from pyramid.exceptions import ConfigurationError
-from pyramid.i18n import TranslationStringFactory
 
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.model.settings import SettingsModel
+from rhodecode.translation import _
 
-_ = TranslationStringFactory('rhodecode-enterprise')
 
 log = logging.getLogger(__name__)
 
 
 class AuthnResourceBase(object):
     __name__ = None
     __parent__ = None
 
     def get_root(self):
         current = self
         while current.__parent__ is not None:
             current = current.__parent__
         return current
 
 
 class AuthnPluginResourceBase(AuthnResourceBase):
 
     def __init__(self, plugin):
         self.plugin = plugin
         self.__name__ = plugin.name
         self.display_name = plugin.get_display_name()
 
 
 class AuthnRootResource(AuthnResourceBase):
     """
     This is the root traversal resource object for the authentication settings.
     """
 
     def __init__(self):
         self._store = {}
         self._resource_name_map = {}
         self.display_name = _('Global')
 
     def __getitem__(self, key):
         """
         Customized get item function to return only items (plugins) that are
         activated.
         """
         if self._is_item_active(key):
             return self._store[key]
         else:
             raise KeyError('Authentication plugin "{}" is not active.'.format(
                 key))
 
     def __iter__(self):
         for key in self._store.keys():
             if self._is_item_active(key):
                 yield self._store[key]
 
     def _is_item_active(self, key):
         activated_plugins = SettingsModel().get_auth_plugins()
         plugin_id = self.get_plugin_id(key)
         return plugin_id in activated_plugins
 
     def get_plugin_id(self, resource_name):
         """
         Return the plugin id for the given traversal resource name.
         """
         # TODO: Store this info in the resource element.
         return self._resource_name_map[resource_name]
 
     def get_sorted_list(self):
         """
         Returns a sorted list of sub resources for displaying purposes.
         """
         def sort_key(resource):
             return str.lower(safe_str(resource.display_name))
 
         active = [item for item in self]
         return sorted(active, key=sort_key)
 
     def get_nav_list(self):
         """
         Returns a sorted list of resources for displaying the navigation.
         """
         list = self.get_sorted_list()
         list.insert(0, self)
         return list
 
     def add_authn_resource(self, config, plugin_id, resource):
         """
         Register a traversal resource as a sub element to the authentication
         settings. This method is registered as a directive on the pyramid
         configurator object and called by plugins.
         """
 
         def _ensure_unique_name(name, limit=100):
             counter = 1
             current = name
             while current in self._store.keys():
                 current = '{}{}'.format(name, counter)
                 counter += 1
                 if counter > limit:
                     raise ConfigurationError(
                         'Cannot build unique name for traversal resource "%s" '
                         'registered by plugin "%s"', name, plugin_id)
             return current
 
         # Allow plugin resources with identical names by rename duplicates.
         unique_name = _ensure_unique_name(resource.__name__)
         if unique_name != resource.__name__:
-            log.warn('Name collision for traversal resource "%s" registered',
+            log.warn('Name collision for traversal resource "%s" registered '
                      'by authentication plugin "%s"', resource.__name__,
                      plugin_id)
             resource.__name__ = unique_name
 
         log.debug('Register traversal resource "%s" for plugin "%s"',
                   unique_name, plugin_id)
         self._resource_name_map[unique_name] = plugin_id
         resource.__parent__ = self
         self._store[unique_name] = resource
 
 
 root = AuthnRootResource()
 
 
 def root_factory(request=None):
     """
     Returns the root traversal resource instance used for the authentication
     settings route.
     """
     return root

rhodecode/authentication/schema.py

@@ -1,53 +1,51 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import colander
 
-from pyramid.i18n import TranslationStringFactory
-
-_ = TranslationStringFactory('rhodecode-enterprise')
+from rhodecode.translation import _
 
 
 class AuthnPluginSettingsSchemaBase(colander.MappingSchema):
     """
     This base schema is intended for use in authentication plugins.
     It adds a few default settings (e.g., "enabled"), so that plugin
     authors don't have to maintain a bunch of boilerplate.
     """
     enabled = colander.SchemaNode(
         colander.Bool(),
         default=False,
         description=_('Enable or disable this authentication plugin.'),
         missing=False,
         title=_('Enabled'),
         widget='bool',
     )
     cache_ttl = colander.SchemaNode(
         colander.Int(),
         default=0,
         description=_('Amount of seconds to cache the authentication '
                       'call for this plugin. Useful for long calls like '
                       'LDAP to improve the responsiveness of the '
                       'authentication system (0 means disabled).'),
         missing=0,
         title=_('Auth Cache TTL'),
         validator=colander.Range(min=0, max=None),
         widget='int',
     )

rhodecode/authentication/views.py

@@ -1,220 +1,182 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import colander
 import formencode.htmlfill
 import logging
 
 from pyramid.httpexceptions import HTTPFound
-from pyramid.i18n import TranslationStringFactory
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.authentication.base import get_auth_cache_manager
 from rhodecode.authentication.interface import IAuthnPluginRegistry
 from rhodecode.lib import auth
 from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
 from rhodecode.model.forms import AuthSettingsForm
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import SettingsModel
+from rhodecode.translation import _
 
 log = logging.getLogger(__name__)
 
-_ = TranslationStringFactory('rhodecode-enterprise')
-
 
 class AuthnPluginViewBase(object):
 
     def __init__(self, context, request):
         self.request = request
         self.context = context
         self.plugin = context.plugin
 
-    # TODO: Think about replacing the htmlfill stuff.
-    def _render_and_fill(self, template, template_context, request,
-                         form_defaults, validation_errors):
-        """
-        Helper to render a template and fill the HTML form fields with
-        defaults. Also displays the form errors.
-        """
-        # Render template to string.
-        html = render(template, template_context, request=request)
-
-        # Fill the HTML form fields with default values and add error messages.
-        html = formencode.htmlfill.render(
-            html,
-            defaults=form_defaults,
-            errors=validation_errors,
-            prefix_error=False,
-            encoding="UTF-8",
-            force_defaults=False)
-
-        return html
-
-    def settings_get(self):
+    def settings_get(self, defaults=None, errors=None):
         """
         View that displays the plugin settings as a form.
         """
-        form_defaults = {}
-        validation_errors = None
+        defaults = defaults or {}
+        errors = errors or {}
         schema = self.plugin.get_settings_schema()
 
         # Get default values for the form.
-        for node in schema.children:
-            value = self.plugin.get_setting_by_name(node.name) or node.default
-            form_defaults[node.name] = value
+        for node in schema:
+            db_value = self.plugin.get_setting_by_name(node.name)
+            defaults.setdefault(node.name, db_value)
 
         template_context = {
+            'defaults': defaults,
+            'errors': errors,
+            'plugin': self.context.plugin,
             'resource': self.context,
-            'plugin': self.context.plugin
         }
 
-        return Response(self._render_and_fill(
-            'rhodecode:templates/admin/auth/plugin_settings.html',
-            template_context,
-            self.request,
-            form_defaults,
-            validation_errors))
+        return template_context
 
     def settings_post(self):
         """
         View that validates and stores the plugin settings.
         """
         schema = self.plugin.get_settings_schema()
         try:
             valid_data = schema.deserialize(self.request.params)
         except colander.Invalid, e:
             # Display error message and display form again.
-            form_defaults = self.request.params
-            validation_errors = e.asdict()
             self.request.session.flash(
                 _('Errors exist when saving plugin settings. '
-                    'Please check the form inputs.'),
+                  'Please check the form inputs.'),
                 queue='error')
-
-            template_context = {
-                'resource': self.context,
-                'plugin': self.context.plugin
-            }
-
-            return Response(self._render_and_fill(
-                'rhodecode:templates/admin/auth/plugin_settings.html',
-                template_context,
-                self.request,
-                form_defaults,
-                validation_errors))
+            defaults = schema.flatten(self.request.params)
+            return self.settings_get(errors=e.asdict(), defaults=defaults)
 
         # Store validated data.
         for name, value in valid_data.items():
             self.plugin.create_or_update_setting(name, value)
         Session.commit()
 
         # Display success message and redirect.
         self.request.session.flash(
             _('Auth settings updated successfully.'),
             queue='success')
         redirect_to = self.request.resource_path(
             self.context, route_name='auth_home')
         return HTTPFound(redirect_to)
 
 
 # TODO: Ongoing migration in these views.
 # - Maybe we should also use a colander schema for these views.
 class AuthSettingsView(object):
     def __init__(self, context, request):
         self.context = context
         self.request = request
 
         # TODO: Move this into a utility function. It is needed in all view
         # classes during migration. Maybe a mixin?
 
         # Some of the decorators rely on this attribute to be present on the
         # class of the decorated method.
         self._rhodecode_user = request.user
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
-    def index(self, defaults={}, errors=None, prefix_error=False):
+    def index(self, defaults=None, errors=None, prefix_error=False):
+        defaults = defaults or {}
         authn_registry = self.request.registry.getUtility(IAuthnPluginRegistry)
-        default_plugins = ['egg:rhodecode-enterprise-ce#rhodecode']
-        enabled_plugins = SettingsModel().get_auth_plugins() or default_plugins
+        enabled_plugins = SettingsModel().get_auth_plugins()
 
         # Create template context and render it.
         template_context = {
             'resource': self.context,
             'available_plugins': authn_registry.get_plugins(),
             'enabled_plugins': enabled_plugins,
         }
         html = render('rhodecode:templates/admin/auth/auth_settings.html',
                       template_context,
                       request=self.request)
 
         # Create form default values and fill the form.
         form_defaults = {
             'auth_plugins': ','.join(enabled_plugins)
         }
         form_defaults.update(defaults)
         html = formencode.htmlfill.render(
             html,
             defaults=form_defaults,
             errors=errors,
             prefix_error=prefix_error,
             encoding="UTF-8",
             force_defaults=False)
 
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @auth.CSRFRequired()
     def auth_settings(self):
         try:
             form = AuthSettingsForm()()
             form_result = form.to_python(self.request.params)
             plugins = ','.join(form_result['auth_plugins'])
             setting = SettingsModel().create_or_update_setting(
                 'auth_plugins', plugins)
             Session().add(setting)
             Session().commit()
 
             cache_manager = get_auth_cache_manager()
             cache_manager.clear()
             self.request.session.flash(
                 _('Auth settings updated successfully.'),
                 queue='success')
         except formencode.Invalid as errors:
             e = errors.error_dict or {}
             self.request.session.flash(
                 _('Errors exist when saving plugin setting. '
                   'Please check the form inputs.'),
                 queue='error')
             return self.index(
                 defaults=errors.value,
                 errors=e,
                 prefix_error=False)
         except Exception:
             log.exception('Exception in auth_settings')
             self.request.session.flash(
                 _('Error occurred during update of auth settings.'),
                 queue='error')
 
         redirect_to = self.request.resource_path(
             self.context, route_name='auth_home')
         return HTTPFound(redirect_to)

rhodecode/config/environment.py

@@ -1,181 +1,192 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Pylons environment configuration
 """
 
 import os
 import logging
 import rhodecode
 import platform
 import re
+import io
 
 from mako.lookup import TemplateLookup
 from pylons.configuration import PylonsConfig
 from pylons.error import handle_mako_error
+from pyramid.settings import asbool
 
 # don't remove this import it does magic for celery
 from rhodecode.lib import celerypylons  # noqa
 
 import rhodecode.lib.app_globals as app_globals
 
 from rhodecode.config import utils
 from rhodecode.config.routing import make_map
+from rhodecode.config.jsroutes import generate_jsroutes_content
 
 from rhodecode.lib import helpers
 from rhodecode.lib.auth import set_available_permissions
 from rhodecode.lib.utils import (
     repo2db_mapper, make_db_config, set_rhodecode_config,
     load_rcextensions)
 from rhodecode.lib.utils2 import str2bool, aslist
 from rhodecode.lib.vcs import connect_vcs, start_vcs_server
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
-
 def load_environment(global_conf, app_conf, initial=False,
                      test_env=None, test_index=None):
     """
     Configure the Pylons environment via the ``pylons.config``
     object
     """
     config = PylonsConfig()
 
-    rhodecode.is_test = str2bool(app_conf.get('is_test', 'False'))
 
     # Pylons paths
     root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
     paths = {
         'root': root,
         'controllers': os.path.join(root, 'controllers'),
         'static_files': os.path.join(root, 'public'),
         'templates': [os.path.join(root, 'templates')],
     }
 
     # Initialize config with the basic options
     config.init_app(global_conf, app_conf, package='rhodecode', paths=paths)
 
     # store some globals into rhodecode
     rhodecode.CELERY_ENABLED = str2bool(config['app_conf'].get('use_celery'))
     rhodecode.CELERY_EAGER = str2bool(
         config['app_conf'].get('celery.always.eager'))
 
     config['routes.map'] = make_map(config)
+
+    if asbool(config['debug']):
+        jsroutes = config['routes.map'].jsroutes()
+        jsroutes_file_content = generate_jsroutes_content(jsroutes)
+        jsroutes_file_path = os.path.join(
+            paths['static_files'], 'js', 'rhodecode', 'routes.js')
+
+        with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
+            f.write(jsroutes_file_content)
+
     config['pylons.app_globals'] = app_globals.Globals(config)
     config['pylons.h'] = helpers
     rhodecode.CONFIG = config
 
     load_rcextensions(root_path=config['here'])
 
     # Setup cache object as early as possible
     import pylons
     pylons.cache._push_object(config['pylons.app_globals'].cache)
 
     # Create the Mako TemplateLookup, with the default auto-escaping
     config['pylons.app_globals'].mako_lookup = TemplateLookup(
         directories=paths['templates'],
         error_handler=handle_mako_error,
         module_directory=os.path.join(app_conf['cache_dir'], 'templates'),
         input_encoding='utf-8', default_filters=['escape'],
         imports=['from webhelpers.html import escape'])
 
     # sets the c attribute access when don't existing attribute are accessed
     config['pylons.strict_tmpl_context'] = True
-    config_file_name = os.path.split(config['__file__'])[-1]
-    test = re.match('^test[\w_]*\.ini$', config_file_name) is not None
-    if test:
-        if test_env is None:
-            test_env = not int(os.environ.get('RC_NO_TMP_PATH', 0))
-
-        from rhodecode.lib.utils import create_test_env, create_test_index
-        from rhodecode.tests import TESTS_TMP_PATH
-        # test repos
-        if test_env:
-            create_test_env(TESTS_TMP_PATH, config)
-            create_test_index(TESTS_TMP_PATH, config, True)
 
     # Limit backends to "vcs.backends" from configuration
     backends = config['vcs.backends'] = aslist(
         config.get('vcs.backends', 'hg,git'), sep=',')
     for alias in rhodecode.BACKENDS.keys():
         if alias not in backends:
             del rhodecode.BACKENDS[alias]
     log.info("Enabled backends: %s", backends)
 
     # initialize vcs client and optionally run the server if enabled
     vcs_server_uri = config.get('vcs.server', '')
     vcs_server_enabled = str2bool(config.get('vcs.server.enable', 'true'))
     start_server = (
         str2bool(config.get('vcs.start_server', 'false')) and
         not int(os.environ.get('RC_VCSSERVER_TEST_DISABLE', '0')))
     if vcs_server_enabled and start_server:
         log.info("Starting vcsserver")
         start_vcs_server(server_and_port=vcs_server_uri,
                          protocol=utils.get_vcs_server_protocol(config),
                          log_level=config['vcs.server.log_level'])
 
-    # MULTIPLE DB configs
-    # Setup the SQLAlchemy database engine
-    utils.initialize_database(config)
-
     set_available_permissions(config)
     db_cfg = make_db_config(clear_session=True)
 
     repos_path = list(db_cfg.items('paths'))[0][1]
     config['base_path'] = repos_path
 
     config['vcs.hooks.direct_calls'] = _use_direct_hook_calls(config)
     config['vcs.hooks.protocol'] = _get_vcs_hooks_protocol(config)
 
     # store db config also in main global CONFIG
     set_rhodecode_config(config)
 
     # configure instance id
     utils.set_instance_id(config)
 
     # CONFIGURATION OPTIONS HERE (note: all config options will override
     # any Pylons config options)
 
     # store config reference into our module to skip import magic of pylons
     rhodecode.CONFIG.update(config)
 
     utils.configure_pyro4(config)
     utils.configure_vcs(config)
     if vcs_server_enabled:
         connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(config))
 
     import_on_startup = str2bool(config.get('startup.import_repos', False))
     if vcs_server_enabled and import_on_startup:
         repo2db_mapper(ScmModel().repo_scan(repos_path), remove_obsolete=False)
     return config
 
 
 def _use_direct_hook_calls(config):
     default_direct_hook_calls = 'false'
     direct_hook_calls = str2bool(
         config.get('vcs.hooks.direct_calls', default_direct_hook_calls))
     return direct_hook_calls
 
 
 def _get_vcs_hooks_protocol(config):
     protocol = config.get('vcs.hooks.protocol', 'pyro4').lower()
     return protocol
+
+
+def load_pyramid_environment(global_config, settings):
+    # Some parts of the code expect a merge of global and app settings.
+    settings_merged = global_config.copy()
+    settings_merged.update(settings)
+
+    # If this is a test run we prepare the test environment like
+    # creating a test database, test search index and test repositories.
+    # This has to be done before the database connection is initialized.
+    if settings['is_test']:
+        rhodecode.is_test = True
+        utils.initialize_test_environment(settings_merged)
+
+    # Initialize the database connection.
+    utils.initialize_database(settings_merged)

rhodecode/config/middleware.py

@@ -1,309 +1,315 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Pylons middleware initialization
 """
 import logging
 
 from paste.registry import RegistryManager
 from paste.gzipper import make_gzip_middleware
 from pylons.middleware import ErrorHandler, StatusCodeRedirect
 from pylons.wsgiapp import PylonsApp
 from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid.config import Configurator
 from pyramid.static import static_view
 from pyramid.settings import asbool, aslist
 from pyramid.wsgi import wsgiapp
 from routes.middleware import RoutesMiddleware
 import routes.util
 
 import rhodecode
 from rhodecode.config import patches
-from rhodecode.config.environment import load_environment
+from rhodecode.config.environment import (
+    load_environment, load_pyramid_environment)
 from rhodecode.lib.middleware import csrf
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.disable_vcs import DisableVCSPagesWrapper
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.middleware.vcs import VCSMiddleware
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
 
 
 log = logging.getLogger(__name__)
 
 
 def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
     """Create a Pylons WSGI application and return it
 
     ``global_conf``
         The inherited configuration for this application. Normally from
         the [DEFAULT] section of the Paste ini file.
 
     ``full_stack``
         Whether or not this application provides a full WSGI stack (by
         default, meaning it handles its own exceptions and errors).
         Disable full_stack when this application is "managed" by
         another WSGI middleware.
 
     ``app_conf``
         The application's local configuration. Normally specified in
         the [app:<name>] section of the Paste ini file (where <name>
         defaults to main).
 
     """
     # Apply compatibility patches
     patches.kombu_1_5_1_python_2_7_11()
     patches.inspect_getargspec()
 
     # Configure the Pylons environment
     config = load_environment(global_conf, app_conf)
 
     # The Pylons WSGI app
     app = PylonsApp(config=config)
     if rhodecode.is_test:
         app = csrf.CSRFDetector(app)
 
     expected_origin = config.get('expected_origin')
     if expected_origin:
         # The API can be accessed from other Origins.
         app = csrf.OriginChecker(app, expected_origin,
                                  skip_urls=[routes.util.url_for('api')])
 
     # Add RoutesMiddleware. Currently we have two instances in the stack. This
     # is the lower one to make the StatusCodeRedirect middleware happy.
     # TODO: johbo: This is not optimal, search for a better solution.
     app = RoutesMiddleware(app, config['routes.map'])
 
     # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares)
     if asbool(config['pdebug']):
         from rhodecode.lib.profiler import ProfilingMiddleware
         app = ProfilingMiddleware(app)
 
     # Protect from VCS Server error related pages when server is not available
     vcs_server_enabled = asbool(config.get('vcs.server.enable', 'true'))
     if not vcs_server_enabled:
         app = DisableVCSPagesWrapper(app)
 
     if asbool(full_stack):
 
         # Appenlight monitoring and error handler
         app, appenlight_client = wrap_in_appenlight_if_enabled(app, config)
 
         # Handle Python exceptions
         app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
 
         # we want our low level middleware to get to the request ASAP. We don't
         # need any pylons stack middleware in them
         app = VCSMiddleware(app, config, appenlight_client)
         # Display error documents for 401, 403, 404 status codes (and
         # 500 when debug is disabled)
         if asbool(config['debug']):
             app = StatusCodeRedirect(app)
         else:
             app = StatusCodeRedirect(app, [400, 401, 403, 404, 500])
 
     # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     app = HttpsFixup(app, config)
 
     # Establish the Registry for this application
     app = RegistryManager(app)
 
     app.config = config
 
     return app
 
 
 def make_pyramid_app(global_config, **settings):
     """
     Constructs the WSGI application based on Pyramid and wraps the Pylons based
     application.
 
     Specials:
 
     * We migrate from Pylons to Pyramid. While doing this, we keep both
       frameworks functional. This involves moving some WSGI middlewares around
       and providing access to some data internals, so that the old code is
       still functional.
 
     * The application can also be integrated like a plugin via the call to
       `includeme`. This is accompanied with the other utility functions which
       are called. Changing this should be done with great care to not break
       cases when these fragments are assembled from another place.
 
     """
     # The edition string should be available in pylons too, so we add it here
     # before copying the settings.
     settings.setdefault('rhodecode.edition', 'Community Edition')
 
     # As long as our Pylons application does expect "unprepared" settings, make
     # sure that we keep an unmodified copy. This avoids unintentional change of
     # behavior in the old application.
     settings_pylons = settings.copy()
 
     sanitize_settings_and_apply_defaults(settings)
     config = Configurator(settings=settings)
     add_pylons_compat_data(config.registry, global_config, settings_pylons)
+
+    load_pyramid_environment(global_config, settings)
+
     includeme(config)
     includeme_last(config)
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     return pyramid_app
 
 
 def add_pylons_compat_data(registry, global_config, settings):
     """
     Attach data to the registry to support the Pylons integration.
     """
     registry._pylons_compat_global_config = global_config
     registry._pylons_compat_settings = settings
 
 
 def includeme(config):
     settings = config.registry.settings
 
     # Includes which are required. The application would fail without them.
     config.include('pyramid_mako')
     config.include('pyramid_beaker')
     config.include('rhodecode.authentication')
+    config.include('rhodecode.login')
     config.include('rhodecode.tweens')
     config.include('rhodecode.api')
 
     # Set the authorization policy.
     authz_policy = ACLAuthorizationPolicy()
     config.set_authorization_policy(authz_policy)
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')
 
     # plugin information
     config.registry.rhodecode_plugins = {}
 
     config.add_directive(
         'register_rhodecode_plugin', register_rhodecode_plugin)
     # include RhodeCode plugins
     includes = aslist(settings.get('rhodecode.includes', []))
     for inc in includes:
         config.include(inc)
 
     # This is the glue which allows us to migrate in chunks. By registering the
     # pylons based application as the "Not Found" view in Pyramid, we will
     # fallback to the old application each time the new one does not yet know
     # how to handle a request.
     pylons_app = make_app(
         config.registry._pylons_compat_global_config,
         **config.registry._pylons_compat_settings)
     config.registry._pylons_compat_config = pylons_app.config
     pylons_app_as_view = wsgiapp(pylons_app)
     config.add_notfound_view(pylons_app_as_view)
 
 
 def includeme_last(config):
     """
     The static file catchall needs to be last in the view configuration.
     """
     settings = config.registry.settings
 
     # Note: johbo: I would prefer to register a prefix for static files at some
     # point, e.g. move them under '_static/'. This would fully avoid that we
     # can have name clashes with a repository name. Imaging someone calling his
     # repo "css" ;-) Also having an external web server to serve out the static
     # files seems to be easier to set up if they have a common prefix.
     #
     # Example: config.add_static_view('_static', path='rhodecode:public')
     #
     # It might be an option to register both paths for a while and then migrate
     # over to the new location.
 
     # Serving static files with a catchall.
     if settings['static_files']:
         config.add_route('catchall_static', '/*subpath')
         config.add_view(
             static_view('rhodecode:public'), route_name='catchall_static')
 
 
 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
     """
     Apply outer WSGI middlewares around the application.
 
     Part of this has been moved up from the Pylons layer, so that the
     data is also available if old Pylons code is hit through an already ported
     view.
     """
     settings = config.registry.settings
 
     # Add RoutesMiddleware. Currently we have two instances in the stack. This
     # is the upper one to support the pylons compatibility tween during
     # migration to pyramid.
     pyramid_app = RoutesMiddleware(
         pyramid_app, config.registry._pylons_compat_config['routes.map'])
 
     # TODO: johbo: Don't really see why we enable the gzip middleware when
     # serving static files, might be something that should have its own setting
     # as well?
     if settings['static_files']:
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
     return pyramid_app
 
 
 def sanitize_settings_and_apply_defaults(settings):
     """
     Applies settings defaults and does all type conversion.
 
     We would move all settings parsing and preparation into this place, so that
     we have only one place left which deals with this part. The remaining parts
     of the application would start to rely fully on well prepared settings.
 
     This piece would later be split up per topic to avoid a big fat monster
     function.
     """
 
     # Pyramid's mako renderer has to search in the templates folder so that the
     # old templates still work. Ported and new templates are expected to use
     # real asset specifications for the includes.
     mako_directories = settings.setdefault('mako.directories', [
         # Base templates of the original Pylons application
         'rhodecode:templates',
     ])
     log.debug(
         "Using the following Mako template directories: %s",
         mako_directories)
 
     # Default includes, possible to change as a user
     pyramid_includes = settings.setdefault('pyramid.includes', [
         'rhodecode.lib.middleware.request_wrapper',
     ])
     log.debug(
         "Using the following pyramid.includes: %s",
         pyramid_includes)
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
     settings.setdefault('rhodecode.api.url', '/_admin/api')
 
     _bool_setting(settings, 'vcs.server.enable', 'true')
     _bool_setting(settings, 'static_files', 'true')
+    _bool_setting(settings, 'is_test', 'false')
 
     return settings
 
 
 def _bool_setting(settings, name, default):
     settings[name] = asbool(settings.get(name, default))

rhodecode/config/routing.py

@@ -1,1112 +1,1149 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Routes configuration
 
 The more specific and detailed routes should be defined first so they
 may take precedent over the more generic routes. For more information
 refer to the routes manual at http://routes.groovie.org/docs/
 
 IMPORTANT: if you change any routing here, make sure to take a look at lib/base.py
 and _route_name variable which uses some of stored naming here to do redirects.
 """
 import os
+import re
 from routes import Mapper
 
 from rhodecode.config import routing_links
 
 # prefix for non repository related links needs to be prefixed with `/`
 ADMIN_PREFIX = '/_admin'
 
 # Default requirements for URL parts
 URL_NAME_REQUIREMENTS = {
     # group name can have a slash in them, but they must not end with a slash
     'group_name': r'.*?[^/]',
     # repo names can have a slash in them, but they must not end with a slash
     'repo_name': r'.*?[^/]',
     # file path eats up everything at the end
     'f_path': r'.*',
     # reference types
     'source_ref_type': '(branch|book|tag|rev|\%\(source_ref_type\)s)',
     'target_ref_type': '(branch|book|tag|rev|\%\(target_ref_type\)s)',
 }
 
 
+class JSRoutesMapper(Mapper):
+    """
+    Wrapper for routes.Mapper to make pyroutes compatible url definitions
+    """
+    _named_route_regex = re.compile(r'^[a-z-_0-9A-Z]+$')
+    _argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')
+    def __init__(self, *args, **kw):
+        super(JSRoutesMapper, self).__init__(*args, **kw)
+        self._jsroutes = []
+
+    def connect(self, *args, **kw):
+        """
+        Wrapper for connect to take an extra argument jsroute=True
+
+        :param jsroute: boolean, if True will add the route to the pyroutes list
+        """
+        if kw.pop('jsroute', False):
+            if not self._named_route_regex.match(args[0]):
+                raise Exception('only named routes can be added to pyroutes')
+            self._jsroutes.append(args[0])
+
+        super(JSRoutesMapper, self).connect(*args, **kw)
+
+    def _extract_route_information(self, route):
+        """
+        Convert a route into tuple(name, path, args), eg:
+            ('user_profile', '/profile/%(username)s', ['username'])
+        """
+        routepath = route.routepath
+        def replace(matchobj):
+            if matchobj.group(1):
+                return "%%(%s)s" % matchobj.group(1).split(':')[0]
+            else:
+                return "%%(%s)s" % matchobj.group(2)
+
+        routepath = self._argument_prog.sub(replace, routepath)
+        return (
+            route.name,
+            routepath,
+            [(arg[0].split(':')[0] if arg[0] != '' else arg[1])
+              for arg in self._argument_prog.findall(route.routepath)]
+        )
+
+    def jsroutes(self):
+        """
+        Return a list of pyroutes.js compatible routes
+        """
+        for route_name in self._jsroutes:
+            yield self._extract_route_information(self._routenames[route_name])
+
+
 def make_map(config):
     """Create, configure and return the routes Mapper"""
-    rmap = Mapper(directory=config['pylons.paths']['controllers'],
+    rmap = JSRoutesMapper(directory=config['pylons.paths']['controllers'],
                   always_scan=config['debug'])
     rmap.minimization = False
     rmap.explicit = False
 
     from rhodecode.lib.utils2 import str2bool
     from rhodecode.model import repo, repo_group
 
     def check_repo(environ, match_dict):
         """
         check for valid repository for proper 404 handling
 
         :param environ:
         :param match_dict:
         """
         repo_name = match_dict.get('repo_name')
 
         if match_dict.get('f_path'):
             # fix for multiple initial slashes that causes errors
             match_dict['f_path'] = match_dict['f_path'].lstrip('/')
         repo_model = repo.RepoModel()
         by_name_match = repo_model.get_by_repo_name(repo_name)
         # if we match quickly from database, short circuit the operation,
         # and validate repo based on the type.
         if by_name_match:
             return True
 
         by_id_match = repo_model.get_repo_by_id(repo_name)
         if by_id_match:
             repo_name = by_id_match.repo_name
             match_dict['repo_name'] = repo_name
             return True
 
         return False
 
     def check_group(environ, match_dict):
         """
         check for valid repository group path for proper 404 handling
 
         :param environ:
         :param match_dict:
         """
         repo_group_name = match_dict.get('group_name')
         repo_group_model = repo_group.RepoGroupModel()
         by_name_match = repo_group_model.get_by_group_name(repo_group_name)
         if by_name_match:
             return True
 
         return False
 
     def check_user_group(environ, match_dict):
         """
         check for valid user group for proper 404 handling
 
         :param environ:
         :param match_dict:
         """
         return True
 
     def check_int(environ, match_dict):
         return match_dict.get('id').isdigit()
 
     # The ErrorController route (handles 404/500 error pages); it should
     # likely stay at the top, ensuring it can always be resolved
     rmap.connect('/error/{action}', controller='error')
     rmap.connect('/error/{action}/{id}', controller='error')
 
     #==========================================================================
     # CUSTOM ROUTES HERE
     #==========================================================================
 
     # MAIN PAGE
-    rmap.connect('home', '/', controller='home', action='index')
-    rmap.connect('repo_switcher_data', '/_repos_and_groups', controller='home',
-                 action='repo_switcher_data')
+    rmap.connect('home', '/', controller='home', action='index', jsroute=True)
+    rmap.connect('goto_switcher_data', '/_goto_data', controller='home',
+                 action='goto_switcher_data')
     rmap.connect('repo_list_data', '/_repos', controller='home',
                  action='repo_list_data')
 
     rmap.connect('user_autocomplete_data', '/_users', controller='home',
-                 action='user_autocomplete_data')
+                 action='user_autocomplete_data', jsroute=True)
     rmap.connect('user_group_autocomplete_data', '/_user_groups', controller='home',
                  action='user_group_autocomplete_data')
 
     rmap.connect(
         'user_profile', '/_profiles/{username}', controller='users',
         action='user_profile')
 
     # TODO: johbo: Static links, to be replaced by our redirection mechanism
     rmap.connect('rst_help',
                  'http://docutils.sourceforge.net/docs/user/rst/quickref.html',
                  _static=True)
     rmap.connect('markdown_help',
                  'http://daringfireball.net/projects/markdown/syntax',
                  _static=True)
     rmap.connect('rhodecode_official', 'https://rhodecode.com', _static=True)
     rmap.connect('rhodecode_support', 'https://rhodecode.com/help/', _static=True)
     rmap.connect('rhodecode_translations', 'https://rhodecode.com/translate/enterprise', _static=True)
     # TODO: anderson - making this a static link since redirect won't play
     # nice with POST requests
     rmap.connect('enterprise_license_convert_from_old',
                  'https://rhodecode.com/u/license-upgrade',
                  _static=True)
 
     routing_links.connect_redirection_links(rmap)
 
     rmap.connect('ping', '%s/ping' % (ADMIN_PREFIX,), controller='home', action='ping')
     rmap.connect('error_test', '%s/error_test' % (ADMIN_PREFIX,), controller='home', action='error_test')
 
     # ADMIN REPOSITORY ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/repos') as m:
         m.connect('repos', '/repos',
                   action='create', conditions={'method': ['POST']})
         m.connect('repos', '/repos',
                   action='index', conditions={'method': ['GET']})
-        m.connect('new_repo', '/create_repository',
+        m.connect('new_repo', '/create_repository', jsroute=True,
                   action='create_repository', conditions={'method': ['GET']})
         m.connect('/repos/{repo_name}',
                   action='update', conditions={'method': ['PUT'],
                                                'function': check_repo},
                   requirements=URL_NAME_REQUIREMENTS)
         m.connect('delete_repo', '/repos/{repo_name}',
                   action='delete', conditions={'method': ['DELETE']},
                   requirements=URL_NAME_REQUIREMENTS)
         m.connect('repo', '/repos/{repo_name}',
                   action='show', conditions={'method': ['GET'],
                                              'function': check_repo},
                   requirements=URL_NAME_REQUIREMENTS)
 
     # ADMIN REPOSITORY GROUPS ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/repo_groups') as m:
         m.connect('repo_groups', '/repo_groups',
                   action='create', conditions={'method': ['POST']})
         m.connect('repo_groups', '/repo_groups',
                   action='index', conditions={'method': ['GET']})
         m.connect('new_repo_group', '/repo_groups/new',
                   action='new', conditions={'method': ['GET']})
         m.connect('update_repo_group', '/repo_groups/{group_name}',
                   action='update', conditions={'method': ['PUT'],
                                                'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
 
         # EXTRAS REPO GROUP ROUTES
         m.connect('edit_repo_group', '/repo_groups/{group_name}/edit',
                   action='edit',
                   conditions={'method': ['GET'], 'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
         m.connect('edit_repo_group', '/repo_groups/{group_name}/edit',
                   action='edit',
                   conditions={'method': ['PUT'], 'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
 
         m.connect('edit_repo_group_advanced', '/repo_groups/{group_name}/edit/advanced',
                   action='edit_repo_group_advanced',
                   conditions={'method': ['GET'], 'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
         m.connect('edit_repo_group_advanced', '/repo_groups/{group_name}/edit/advanced',
                   action='edit_repo_group_advanced',
                   conditions={'method': ['PUT'], 'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
 
         m.connect('edit_repo_group_perms', '/repo_groups/{group_name}/edit/permissions',
                   action='edit_repo_group_perms',
                   conditions={'method': ['GET'], 'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
         m.connect('edit_repo_group_perms', '/repo_groups/{group_name}/edit/permissions',
                   action='update_perms',
                   conditions={'method': ['PUT'], 'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
 
         m.connect('delete_repo_group', '/repo_groups/{group_name}',
                   action='delete', conditions={'method': ['DELETE'],
                                                'function': check_group},
                   requirements=URL_NAME_REQUIREMENTS)
 
     # ADMIN USER ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/users') as m:
         m.connect('users', '/users',
                   action='create', conditions={'method': ['POST']})
         m.connect('users', '/users',
                   action='index', conditions={'method': ['GET']})
         m.connect('new_user', '/users/new',
                   action='new', conditions={'method': ['GET']})
         m.connect('update_user', '/users/{user_id}',
                   action='update', conditions={'method': ['PUT']})
         m.connect('delete_user', '/users/{user_id}',
                   action='delete', conditions={'method': ['DELETE']})
         m.connect('edit_user', '/users/{user_id}/edit',
                   action='edit', conditions={'method': ['GET']})
         m.connect('user', '/users/{user_id}',
                   action='show', conditions={'method': ['GET']})
         m.connect('force_password_reset_user', '/users/{user_id}/password_reset',
                   action='reset_password', conditions={'method': ['POST']})
         m.connect('create_personal_repo_group', '/users/{user_id}/create_repo_group',
                   action='create_personal_repo_group', conditions={'method': ['POST']})
 
         # EXTRAS USER ROUTES
         m.connect('edit_user_advanced', '/users/{user_id}/edit/advanced',
                   action='edit_advanced', conditions={'method': ['GET']})
         m.connect('edit_user_advanced', '/users/{user_id}/edit/advanced',
                   action='update_advanced', conditions={'method': ['PUT']})
 
         m.connect('edit_user_auth_tokens', '/users/{user_id}/edit/auth_tokens',
                   action='edit_auth_tokens', conditions={'method': ['GET']})
         m.connect('edit_user_auth_tokens', '/users/{user_id}/edit/auth_tokens',
                   action='add_auth_token', conditions={'method': ['PUT']})
         m.connect('edit_user_auth_tokens', '/users/{user_id}/edit/auth_tokens',
                   action='delete_auth_token', conditions={'method': ['DELETE']})
 
         m.connect('edit_user_global_perms', '/users/{user_id}/edit/global_permissions',
                   action='edit_global_perms', conditions={'method': ['GET']})
         m.connect('edit_user_global_perms', '/users/{user_id}/edit/global_permissions',
                   action='update_global_perms', conditions={'method': ['PUT']})
 
         m.connect('edit_user_perms_summary', '/users/{user_id}/edit/permissions_summary',
                   action='edit_perms_summary', conditions={'method': ['GET']})
 
         m.connect('edit_user_emails', '/users/{user_id}/edit/emails',
                   action='edit_emails', conditions={'method': ['GET']})
         m.connect('edit_user_emails', '/users/{user_id}/edit/emails',
                   action='add_email', conditions={'method': ['PUT']})
         m.connect('edit_user_emails', '/users/{user_id}/edit/emails',
                   action='delete_email', conditions={'method': ['DELETE']})
 
         m.connect('edit_user_ips', '/users/{user_id}/edit/ips',
                   action='edit_ips', conditions={'method': ['GET']})
         m.connect('edit_user_ips', '/users/{user_id}/edit/ips',
                   action='add_ip', conditions={'method': ['PUT']})
         m.connect('edit_user_ips', '/users/{user_id}/edit/ips',
                   action='delete_ip', conditions={'method': ['DELETE']})
 
     # ADMIN USER GROUPS REST ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/user_groups') as m:
         m.connect('users_groups', '/user_groups',
                   action='create', conditions={'method': ['POST']})
         m.connect('users_groups', '/user_groups',
                   action='index', conditions={'method': ['GET']})
         m.connect('new_users_group', '/user_groups/new',
                   action='new', conditions={'method': ['GET']})
         m.connect('update_users_group', '/user_groups/{user_group_id}',
                   action='update', conditions={'method': ['PUT']})
         m.connect('delete_users_group', '/user_groups/{user_group_id}',
                   action='delete', conditions={'method': ['DELETE']})
         m.connect('edit_users_group', '/user_groups/{user_group_id}/edit',
                   action='edit', conditions={'method': ['GET']},
                   function=check_user_group)
 
         # EXTRAS USER GROUP ROUTES
-        m.connect('edit_user_group_global_perms', '/user_groups/{user_group_id}/edit/global_permissions',
+        m.connect('edit_user_group_global_perms',
+                  '/user_groups/{user_group_id}/edit/global_permissions',
                   action='edit_global_perms', conditions={'method': ['GET']})
-        m.connect('edit_user_group_global_perms', '/user_groups/{user_group_id}/edit/global_permissions',
+        m.connect('edit_user_group_global_perms',
+                  '/user_groups/{user_group_id}/edit/global_permissions',
                   action='update_global_perms', conditions={'method': ['PUT']})
-        m.connect('edit_user_group_perms_summary', '/user_groups/{user_group_id}/edit/permissions_summary',
+        m.connect('edit_user_group_perms_summary',
+                  '/user_groups/{user_group_id}/edit/permissions_summary',
                   action='edit_perms_summary', conditions={'method': ['GET']})
 
-        m.connect('edit_user_group_perms', '/user_groups/{user_group_id}/edit/permissions',
+        m.connect('edit_user_group_perms',
+                  '/user_groups/{user_group_id}/edit/permissions',
                   action='edit_perms', conditions={'method': ['GET']})
-        m.connect('edit_user_group_perms', '/user_groups/{user_group_id}/edit/permissions',
+        m.connect('edit_user_group_perms',
+                  '/user_groups/{user_group_id}/edit/permissions',
                   action='update_perms', conditions={'method': ['PUT']})
 
-        m.connect('edit_user_group_advanced', '/user_groups/{user_group_id}/edit/advanced',
+        m.connect('edit_user_group_advanced',
+                  '/user_groups/{user_group_id}/edit/advanced',
                   action='edit_advanced', conditions={'method': ['GET']})
 
-        m.connect('edit_user_group_members', '/user_groups/{user_group_id}/edit/members',
+        m.connect('edit_user_group_members',
+                  '/user_groups/{user_group_id}/edit/members', jsroute=True,
                   action='edit_members', conditions={'method': ['GET']})
 
     # ADMIN PERMISSIONS ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/permissions') as m:
         m.connect('admin_permissions_application', '/permissions/application',
                   action='permission_application_update', conditions={'method': ['POST']})
         m.connect('admin_permissions_application', '/permissions/application',
                   action='permission_application', conditions={'method': ['GET']})
 
         m.connect('admin_permissions_global', '/permissions/global',
                   action='permission_global_update', conditions={'method': ['POST']})
         m.connect('admin_permissions_global', '/permissions/global',
                   action='permission_global', conditions={'method': ['GET']})
 
         m.connect('admin_permissions_object', '/permissions/object',
                   action='permission_objects_update', conditions={'method': ['POST']})
         m.connect('admin_permissions_object', '/permissions/object',
                   action='permission_objects', conditions={'method': ['GET']})
 
         m.connect('admin_permissions_ips', '/permissions/ips',
                   action='permission_ips', conditions={'method': ['POST']})
         m.connect('admin_permissions_ips', '/permissions/ips',
                   action='permission_ips', conditions={'method': ['GET']})
 
         m.connect('admin_permissions_overview', '/permissions/overview',
                   action='permission_perms', conditions={'method': ['GET']})
 
     # ADMIN DEFAULTS REST ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/defaults') as m:
         m.connect('admin_defaults_repositories', '/defaults/repositories',
                   action='update_repository_defaults', conditions={'method': ['POST']})
         m.connect('admin_defaults_repositories', '/defaults/repositories',
                   action='index', conditions={'method': ['GET']})
 
     # ADMIN DEBUG STYLE ROUTES
     if str2bool(config.get('debug_style')):
         with rmap.submapper(path_prefix=ADMIN_PREFIX + '/debug_style',
                             controller='debug_style') as m:
             m.connect('debug_style_home', '',
                       action='index', conditions={'method': ['GET']})
             m.connect('debug_style_template', '/t/{t_path}',
                       action='template', conditions={'method': ['GET']})
 
     # ADMIN SETTINGS ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/settings') as m:
 
         # default
         m.connect('admin_settings', '/settings',
                   action='settings_global_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings', '/settings',
                   action='settings_global', conditions={'method': ['GET']})
 
         m.connect('admin_settings_vcs', '/settings/vcs',
                   action='settings_vcs_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings_vcs', '/settings/vcs',
                   action='settings_vcs',
                   conditions={'method': ['GET']})
         m.connect('admin_settings_vcs', '/settings/vcs',
                   action='delete_svn_pattern',
                   conditions={'method': ['DELETE']})
 
         m.connect('admin_settings_mapping', '/settings/mapping',
                   action='settings_mapping_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings_mapping', '/settings/mapping',
                   action='settings_mapping', conditions={'method': ['GET']})
 
         m.connect('admin_settings_global', '/settings/global',
                   action='settings_global_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings_global', '/settings/global',
                   action='settings_global', conditions={'method': ['GET']})
 
         m.connect('admin_settings_visual', '/settings/visual',
                   action='settings_visual_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings_visual', '/settings/visual',
                   action='settings_visual', conditions={'method': ['GET']})
 
         m.connect('admin_settings_issuetracker',
                   '/settings/issue-tracker', action='settings_issuetracker',
                   conditions={'method': ['GET']})
         m.connect('admin_settings_issuetracker_save',
                   '/settings/issue-tracker/save',
                   action='settings_issuetracker_save',
                   conditions={'method': ['POST']})
         m.connect('admin_issuetracker_test', '/settings/issue-tracker/test',
                   action='settings_issuetracker_test',
                   conditions={'method': ['POST']})
         m.connect('admin_issuetracker_delete',
                   '/settings/issue-tracker/delete',
                   action='settings_issuetracker_delete',
                   conditions={'method': ['DELETE']})
 
         m.connect('admin_settings_email', '/settings/email',
                   action='settings_email_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings_email', '/settings/email',
                   action='settings_email', conditions={'method': ['GET']})
 
         m.connect('admin_settings_hooks', '/settings/hooks',
                   action='settings_hooks_update',
                   conditions={'method': ['POST', 'DELETE']})
         m.connect('admin_settings_hooks', '/settings/hooks',
                   action='settings_hooks', conditions={'method': ['GET']})
 
         m.connect('admin_settings_search', '/settings/search',
                   action='settings_search', conditions={'method': ['GET']})
 
         m.connect('admin_settings_system', '/settings/system',
                   action='settings_system', conditions={'method': ['GET']})
 
         m.connect('admin_settings_system_update', '/settings/system/updates',
                   action='settings_system_update', conditions={'method': ['GET']})
 
         m.connect('admin_settings_supervisor', '/settings/supervisor',
                   action='settings_supervisor', conditions={'method': ['GET']})
         m.connect('admin_settings_supervisor_log', '/settings/supervisor/{procid}/log',
                   action='settings_supervisor_log', conditions={'method': ['GET']})
 
         m.connect('admin_settings_labs', '/settings/labs',
                   action='settings_labs_update',
                   conditions={'method': ['POST']})
         m.connect('admin_settings_labs', '/settings/labs',
                   action='settings_labs', conditions={'method': ['GET']})
 
         m.connect('admin_settings_open_source', '/settings/open_source',
                   action='settings_open_source',
                   conditions={'method': ['GET']})
 
     # ADMIN MY ACCOUNT
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/my_account') as m:
 
         m.connect('my_account', '/my_account',
                   action='my_account', conditions={'method': ['GET']})
         m.connect('my_account_edit', '/my_account/edit',
                   action='my_account_edit', conditions={'method': ['GET']})
         m.connect('my_account', '/my_account',
                   action='my_account_update', conditions={'method': ['POST']})
 
         m.connect('my_account_password', '/my_account/password',
                   action='my_account_password', conditions={'method': ['GET']})
         m.connect('my_account_password', '/my_account/password',
                   action='my_account_password_update', conditions={'method': ['POST']})
 
         m.connect('my_account_repos', '/my_account/repos',
                   action='my_account_repos', conditions={'method': ['GET']})
 
         m.connect('my_account_watched', '/my_account/watched',
                   action='my_account_watched', conditions={'method': ['GET']})
 
         m.connect('my_account_pullrequests', '/my_account/pull_requests',
                   action='my_account_pullrequests', conditions={'method': ['GET']})
 
         m.connect('my_account_perms', '/my_account/perms',
                   action='my_account_perms', conditions={'method': ['GET']})
 
         m.connect('my_account_emails', '/my_account/emails',
                   action='my_account_emails', conditions={'method': ['GET']})
         m.connect('my_account_emails', '/my_account/emails',
                   action='my_account_emails_add', conditions={'method': ['POST']})
         m.connect('my_account_emails', '/my_account/emails',
                   action='my_account_emails_delete', conditions={'method': ['DELETE']})
 
         m.connect('my_account_auth_tokens', '/my_account/auth_tokens',
                   action='my_account_auth_tokens', conditions={'method': ['GET']})
         m.connect('my_account_auth_tokens', '/my_account/auth_tokens',
                   action='my_account_auth_tokens_add', conditions={'method': ['POST']})
         m.connect('my_account_auth_tokens', '/my_account/auth_tokens',
                   action='my_account_auth_tokens_delete', conditions={'method': ['DELETE']})
 
-        m.connect('my_account_oauth', '/my_account/oauth',
-                  action='my_account_oauth', conditions={'method': ['GET']})
-        m.connect('my_account_oauth', '/my_account/oauth',
-                  action='my_account_oauth_delete',
-                  conditions={'method': ['DELETE']})
-
     # NOTIFICATION REST ROUTES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/notifications') as m:
         m.connect('notifications', '/notifications',
                   action='index', conditions={'method': ['GET']})
         m.connect('notifications_mark_all_read', '/notifications/mark_all_read',
                   action='mark_all_read', conditions={'method': ['POST']})
 
         m.connect('/notifications/{notification_id}',
                   action='update', conditions={'method': ['PUT']})
         m.connect('/notifications/{notification_id}',
                   action='delete', conditions={'method': ['DELETE']})
         m.connect('notification', '/notifications/{notification_id}',
                   action='show', conditions={'method': ['GET']})
 
     # ADMIN GIST
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/gists') as m:
         m.connect('gists', '/gists',
                   action='create', conditions={'method': ['POST']})
-        m.connect('gists', '/gists',
+        m.connect('gists', '/gists', jsroute=True,
                   action='index', conditions={'method': ['GET']})
-        m.connect('new_gist', '/gists/new',
+        m.connect('new_gist', '/gists/new', jsroute=True,
                   action='new', conditions={'method': ['GET']})
 
         m.connect('/gists/{gist_id}',
                   action='delete', conditions={'method': ['DELETE']})
         m.connect('edit_gist', '/gists/{gist_id}/edit',
                   action='edit_form', conditions={'method': ['GET']})
         m.connect('edit_gist', '/gists/{gist_id}/edit',
                   action='edit', conditions={'method': ['POST']})
         m.connect(
             'edit_gist_check_revision', '/gists/{gist_id}/edit/check_revision',
             action='check_revision', conditions={'method': ['GET']})
 
         m.connect('gist', '/gists/{gist_id}',
                   action='show', conditions={'method': ['GET']})
         m.connect('gist_rev', '/gists/{gist_id}/{revision}',
                   revision='tip',
                   action='show', conditions={'method': ['GET']})
         m.connect('formatted_gist', '/gists/{gist_id}/{revision}/{format}',
                   revision='tip',
                   action='show', conditions={'method': ['GET']})
         m.connect('formatted_gist_file', '/gists/{gist_id}/{revision}/{format}/{f_path}',
                   revision='tip',
                   action='show', conditions={'method': ['GET']},
                   requirements=URL_NAME_REQUIREMENTS)
 
     # ADMIN MAIN PAGES
     with rmap.submapper(path_prefix=ADMIN_PREFIX,
                         controller='admin/admin') as m:
         m.connect('admin_home', '', action='index')
         m.connect('admin_add_repo', '/add_repo/{new_repo:[a-z0-9\. _-]*}',
                   action='add_repo')
         m.connect(
-            'pull_requests_global', '/pull_requests/{pull_request_id:[0-9]+}',
+            'pull_requests_global_0', '/pull_requests/{pull_request_id:[0-9]+}',
             action='pull_requests')
+        m.connect(
+            'pull_requests_global', '/pull-requests/{pull_request_id:[0-9]+}',
+            action='pull_requests')
+
 
     # USER JOURNAL
     rmap.connect('journal', '%s/journal' % (ADMIN_PREFIX,),
                  controller='journal', action='index')
     rmap.connect('journal_rss', '%s/journal/rss' % (ADMIN_PREFIX,),
                  controller='journal', action='journal_rss')
     rmap.connect('journal_atom', '%s/journal/atom' % (ADMIN_PREFIX,),
                  controller='journal', action='journal_atom')
 
     rmap.connect('public_journal', '%s/public_journal' % (ADMIN_PREFIX,),
                  controller='journal', action='public_journal')
 
     rmap.connect('public_journal_rss', '%s/public_journal/rss' % (ADMIN_PREFIX,),
                  controller='journal', action='public_journal_rss')
 
     rmap.connect('public_journal_rss_old', '%s/public_journal_rss' % (ADMIN_PREFIX,),
                  controller='journal', action='public_journal_rss')
 
     rmap.connect('public_journal_atom',
                  '%s/public_journal/atom' % (ADMIN_PREFIX,), controller='journal',
                  action='public_journal_atom')
 
     rmap.connect('public_journal_atom_old',
                  '%s/public_journal_atom' % (ADMIN_PREFIX,), controller='journal',
                  action='public_journal_atom')
 
     rmap.connect('toggle_following', '%s/toggle_following' % (ADMIN_PREFIX,),
-                 controller='journal', action='toggle_following',
+                 controller='journal', action='toggle_following', jsroute=True,
                  conditions={'method': ['POST']})
 
     # FULL TEXT SEARCH
     rmap.connect('search', '%s/search' % (ADMIN_PREFIX,),
                  controller='search')
     rmap.connect('search_repo_home', '/{repo_name}/search',
                  controller='search',
                  action='index',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
-    # LOGIN/LOGOUT/REGISTER/SIGN IN
-    rmap.connect('login_home', '%s/login' % (ADMIN_PREFIX,), controller='login',
-                 action='index')
-
-    rmap.connect('logout_home', '%s/logout' % (ADMIN_PREFIX,), controller='login',
-                 action='logout', conditions={'method': ['POST']})
-
-    rmap.connect('register', '%s/register' % (ADMIN_PREFIX,), controller='login',
-                 action='register')
-
-    rmap.connect('reset_password', '%s/password_reset' % (ADMIN_PREFIX,),
-                 controller='login', action='password_reset')
-
-    rmap.connect('reset_password_confirmation',
-                 '%s/password_reset_confirmation' % (ADMIN_PREFIX,),
-                 controller='login', action='password_reset_confirmation')
-
-    rmap.connect('social_auth',
-                 '%s/social_auth/{provider_name}' % (ADMIN_PREFIX,),
-                 controller='login', action='social_auth')
-
     # FEEDS
     rmap.connect('rss_feed_home', '/{repo_name}/feed/rss',
                  controller='feed', action='rss',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('atom_feed_home', '/{repo_name}/feed/atom',
                  controller='feed', action='atom',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     #==========================================================================
     # REPOSITORY ROUTES
     #==========================================================================
 
     rmap.connect('repo_creating_home', '/{repo_name}/repo_creating',
                  controller='admin/repos', action='repo_creating',
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_check_home', '/{repo_name}/crepo_check',
                  controller='admin/repos', action='repo_check',
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('repo_stats', '/{repo_name}/repo_stats/{commit_id}',
                  controller='summary', action='repo_stats',
                  conditions={'function': check_repo},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('repo_refs_data', '/{repo_name}/refs-data',
-                 controller='summary', action='repo_refs_data',
+                 controller='summary', action='repo_refs_data', jsroute=True,
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_refs_changelog_data', '/{repo_name}/refs-data-changelog',
                  controller='summary', action='repo_refs_changelog_data',
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('changeset_home', '/{repo_name}/changeset/{revision}',
-                 controller='changeset', revision='tip',
+                 controller='changeset', revision='tip', jsroute=True,
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('changeset_children', '/{repo_name}/changeset_children/{revision}',
                  controller='changeset', revision='tip', action='changeset_children',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('changeset_parents', '/{repo_name}/changeset_parents/{revision}',
                  controller='changeset', revision='tip', action='changeset_parents',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     # repo edit options
-    rmap.connect('edit_repo', '/{repo_name}/settings',
+    rmap.connect('edit_repo', '/{repo_name}/settings', jsroute=True,
                  controller='admin/repos', action='edit',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_perms', '/{repo_name}/settings/permissions',
+                 jsroute=True,
                  controller='admin/repos', action='edit_permissions',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('edit_repo_perms_update', '/{repo_name}/settings/permissions',
                  controller='admin/repos', action='edit_permissions_update',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_fields', '/{repo_name}/settings/fields',
                  controller='admin/repos', action='edit_fields',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('create_repo_fields', '/{repo_name}/settings/fields/new',
                  controller='admin/repos', action='create_repo_field',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('delete_repo_fields', '/{repo_name}/settings/fields/{field_id}',
                  controller='admin/repos', action='delete_repo_field',
                  conditions={'method': ['DELETE'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_advanced', '/{repo_name}/settings/advanced',
                  controller='admin/repos', action='edit_advanced',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_advanced_locking', '/{repo_name}/settings/advanced/locking',
                  controller='admin/repos', action='edit_advanced_locking',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('toggle_locking', '/{repo_name}/settings/advanced/locking_toggle',
                  controller='admin/repos', action='toggle_locking',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_advanced_journal', '/{repo_name}/settings/advanced/journal',
                  controller='admin/repos', action='edit_advanced_journal',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_advanced_fork', '/{repo_name}/settings/advanced/fork',
                  controller='admin/repos', action='edit_advanced_fork',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_caches', '/{repo_name}/settings/caches',
                  controller='admin/repos', action='edit_caches_form',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('edit_repo_caches', '/{repo_name}/settings/caches',
                  controller='admin/repos', action='edit_caches',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_remote', '/{repo_name}/settings/remote',
                  controller='admin/repos', action='edit_remote_form',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('edit_repo_remote', '/{repo_name}/settings/remote',
                  controller='admin/repos', action='edit_remote',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('edit_repo_statistics', '/{repo_name}/settings/statistics',
                  controller='admin/repos', action='edit_statistics_form',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('edit_repo_statistics', '/{repo_name}/settings/statistics',
                  controller='admin/repos', action='edit_statistics',
                  conditions={'method': ['PUT'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_settings_issuetracker',
                  '/{repo_name}/settings/issue-tracker',
                  controller='admin/repos', action='repo_issuetracker',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_issuetracker_test',
                  '/{repo_name}/settings/issue-tracker/test',
                  controller='admin/repos', action='repo_issuetracker_test',
                  conditions={'method': ['POST'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_issuetracker_delete',
                  '/{repo_name}/settings/issue-tracker/delete',
                  controller='admin/repos', action='repo_issuetracker_delete',
                  conditions={'method': ['DELETE'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_issuetracker_save',
                  '/{repo_name}/settings/issue-tracker/save',
                  controller='admin/repos', action='repo_issuetracker_save',
                  conditions={'method': ['POST'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_vcs_settings', '/{repo_name}/settings/vcs',
                  controller='admin/repos', action='repo_settings_vcs_update',
                  conditions={'method': ['POST'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_vcs_settings', '/{repo_name}/settings/vcs',
                  controller='admin/repos', action='repo_settings_vcs',
                  conditions={'method': ['GET'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
     rmap.connect('repo_vcs_settings', '/{repo_name}/settings/vcs',
                  controller='admin/repos', action='repo_delete_svn_pattern',
                  conditions={'method': ['DELETE'], 'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     # still working url for backward compat.
     rmap.connect('raw_changeset_home_depraced',
                  '/{repo_name}/raw-changeset/{revision}',
                  controller='changeset', action='changeset_raw',
                  revision='tip', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     # new URLs
     rmap.connect('changeset_raw_home',
                  '/{repo_name}/changeset-diff/{revision}',
                  controller='changeset', action='changeset_raw',
                  revision='tip', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changeset_patch_home',
                  '/{repo_name}/changeset-patch/{revision}',
                  controller='changeset', action='changeset_patch',
                  revision='tip', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changeset_download_home',
                  '/{repo_name}/changeset-download/{revision}',
                  controller='changeset', action='changeset_download',
                  revision='tip', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changeset_comment',
-                 '/{repo_name}/changeset/{revision}/comment',
+                 '/{repo_name}/changeset/{revision}/comment', jsroute=True,
                  controller='changeset', revision='tip', action='comment',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changeset_comment_preview',
-                 '/{repo_name}/changeset/comment/preview',
+                 '/{repo_name}/changeset/comment/preview', jsroute=True,
                  controller='changeset', action='preview_comment',
                  conditions={'function': check_repo, 'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changeset_comment_delete',
                  '/{repo_name}/changeset/comment/{comment_id}/delete',
                  controller='changeset', action='delete_comment',
                  conditions={'function': check_repo, 'method': ['DELETE']},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('changeset_info', '/changeset_info/{repo_name}/{revision}',
                  controller='changeset', action='changeset_info',
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('compare_home',
                  '/{repo_name}/compare',
                  controller='compare', action='index',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('compare_url',
                  '/{repo_name}/compare/{source_ref_type}@{source_ref:.*?}...{target_ref_type}@{target_ref:.*?}',
                  controller='compare', action='compare',
                  conditions={'function': check_repo},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_home',
                  '/{repo_name}/pull-request/new', controller='pullrequests',
                  action='index', conditions={'function': check_repo,
                                              'method': ['GET']},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest',
                  '/{repo_name}/pull-request/new', controller='pullrequests',
                  action='create', conditions={'function': check_repo,
                                               'method': ['POST']},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_repo_refs',
                  '/{repo_name}/pull-request/refs/{target_repo_name:.*?[^/]}',
                  controller='pullrequests',
                  action='get_repo_refs',
                  conditions={'function': check_repo, 'method': ['GET']},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_repo_destinations',
                  '/{repo_name}/pull-request/repo-destinations',
                  controller='pullrequests',
                  action='get_repo_destinations',
                  conditions={'function': check_repo, 'method': ['GET']},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_show',
                  '/{repo_name}/pull-request/{pull_request_id}',
                  controller='pullrequests',
                  action='show', conditions={'function': check_repo,
                                             'method': ['GET']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('pullrequest_update',
                  '/{repo_name}/pull-request/{pull_request_id}',
                  controller='pullrequests',
                  action='update', conditions={'function': check_repo,
                                               'method': ['PUT']},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_merge',
                  '/{repo_name}/pull-request/{pull_request_id}',
                  controller='pullrequests',
                  action='merge', conditions={'function': check_repo,
                                              'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('pullrequest_delete',
                  '/{repo_name}/pull-request/{pull_request_id}',
                  controller='pullrequests',
                  action='delete', conditions={'function': check_repo,
                                               'method': ['DELETE']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('pullrequest_show_all',
                  '/{repo_name}/pull-request',
                  controller='pullrequests',
                  action='show_all', conditions={'function': check_repo,
                                                 'method': ['GET']},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_comment',
                  '/{repo_name}/pull-request-comment/{pull_request_id}',
                  controller='pullrequests',
                  action='comment', conditions={'function': check_repo,
                                                'method': ['POST']},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('pullrequest_comment_delete',
                  '/{repo_name}/pull-request-comment/{comment_id}/delete',
                  controller='pullrequests', action='delete_comment',
                  conditions={'function': check_repo, 'method': ['DELETE']},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('summary_home_explicit', '/{repo_name}/summary',
                  controller='summary', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('branches_home', '/{repo_name}/branches',
                  controller='branches', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('tags_home', '/{repo_name}/tags',
                  controller='tags', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('bookmarks_home', '/{repo_name}/bookmarks',
                  controller='bookmarks', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
-    rmap.connect('changelog_home', '/{repo_name}/changelog',
+    rmap.connect('changelog_home', '/{repo_name}/changelog', jsroute=True,
                  controller='changelog', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('changelog_summary_home', '/{repo_name}/changelog_summary',
                  controller='changelog', action='changelog_summary',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
-    rmap.connect('changelog_file_home', '/{repo_name}/changelog/{revision}/{f_path}',
+    rmap.connect('changelog_file_home',
+                 '/{repo_name}/changelog/{revision}/{f_path}',
                  controller='changelog', f_path=None,
                  conditions={'function': check_repo},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('changelog_details', '/{repo_name}/changelog_details/{cs}',
                  controller='changelog', action='changelog_details',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
-    rmap.connect('files_home',
-                 '/{repo_name}/files/{revision}/{f_path}',
+    rmap.connect('files_home',  '/{repo_name}/files/{revision}/{f_path}',
                  controller='files', revision='tip', f_path='',
                  conditions={'function': check_repo},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('files_home_simple_catchrev',
                  '/{repo_name}/files/{revision}',
                  controller='files', revision='tip', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_home_simple_catchall',
                  '/{repo_name}/files',
                  controller='files', revision='tip', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_history_home',
                  '/{repo_name}/history/{revision}/{f_path}',
                  controller='files', action='history', revision='tip', f_path='',
                  conditions={'function': check_repo},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('files_authors_home',
                  '/{repo_name}/authors/{revision}/{f_path}',
                  controller='files', action='authors', revision='tip', f_path='',
                  conditions={'function': check_repo},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('files_diff_home', '/{repo_name}/diff/{f_path}',
                  controller='files', action='diff', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_diff_2way_home',
                  '/{repo_name}/diff-2way/{f_path}',
                  controller='files', action='diff_2way', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_rawfile_home',
                  '/{repo_name}/rawfile/{revision}/{f_path}',
                  controller='files', action='rawfile', revision='tip',
                  f_path='', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_raw_home',
                  '/{repo_name}/raw/{revision}/{f_path}',
                  controller='files', action='raw', revision='tip', f_path='',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_render_home',
                  '/{repo_name}/render/{revision}/{f_path}',
                  controller='files', action='index', revision='tip', f_path='',
                  rendered=True, conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_annotate_home',
                  '/{repo_name}/annotate/{revision}/{f_path}',
                  controller='files', action='index', revision='tip',
                  f_path='', annotate=True, conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_edit',
                  '/{repo_name}/edit/{revision}/{f_path}',
                  controller='files', action='edit', revision='tip',
                  f_path='',
                  conditions={'function': check_repo, 'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_edit_home',
                  '/{repo_name}/edit/{revision}/{f_path}',
                  controller='files', action='edit_home', revision='tip',
                  f_path='', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_add',
                  '/{repo_name}/add/{revision}/{f_path}',
                  controller='files', action='add', revision='tip',
                  f_path='',
                  conditions={'function': check_repo, 'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_add_home',
                  '/{repo_name}/add/{revision}/{f_path}',
                  controller='files', action='add_home', revision='tip',
                  f_path='', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_delete',
                  '/{repo_name}/delete/{revision}/{f_path}',
                  controller='files', action='delete', revision='tip',
                  f_path='',
                  conditions={'function': check_repo, 'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_delete_home',
                  '/{repo_name}/delete/{revision}/{f_path}',
                  controller='files', action='delete_home', revision='tip',
                  f_path='', conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('files_archive_home', '/{repo_name}/archive/{fname}',
                  controller='files', action='archivefile',
                  conditions={'function': check_repo},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('files_nodelist_home',
                  '/{repo_name}/nodelist/{revision}/{f_path}',
                  controller='files', action='nodelist',
                  conditions={'function': check_repo},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('files_metadata_list_home',
                  '/{repo_name}/metadata_list/{revision}/{f_path}',
                  controller='files', action='metadata_list',
                  conditions={'function': check_repo},
-                 requirements=URL_NAME_REQUIREMENTS)
+                 requirements=URL_NAME_REQUIREMENTS, jsroute=True)
 
     rmap.connect('repo_fork_create_home', '/{repo_name}/fork',
                  controller='forks', action='fork_create',
                  conditions={'function': check_repo, 'method': ['POST']},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('repo_fork_home', '/{repo_name}/fork',
                  controller='forks', action='fork',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('repo_forks_home', '/{repo_name}/forks',
                  controller='forks', action='forks',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     rmap.connect('repo_followers_home', '/{repo_name}/followers',
                  controller='followers', action='followers',
                  conditions={'function': check_repo},
                  requirements=URL_NAME_REQUIREMENTS)
 
     # must be here for proper group/repo catching pattern
     _connect_with_slash(
         rmap, 'repo_group_home', '/{group_name}',
         controller='home', action='index_repo_group',
         conditions={'function': check_group},
         requirements=URL_NAME_REQUIREMENTS)
 
     # catch all, at the end
     _connect_with_slash(
-        rmap, 'summary_home', '/{repo_name}',
+        rmap, 'summary_home', '/{repo_name}', jsroute=True,
         controller='summary', action='index',
         conditions={'function': check_repo},
         requirements=URL_NAME_REQUIREMENTS)
 
     return rmap
 
 
 def _connect_with_slash(mapper, name, path, *args, **kwargs):
     """
     Connect a route with an optional trailing slash in `path`.
     """
     mapper.connect(name + '_slash', path + '/', *args, **kwargs)
     mapper.connect(name, path, *args, **kwargs)

rhodecode/config/utils.py

@@ -1,87 +1,99 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import shlex
 import Pyro4
 import platform
 
 from rhodecode.model import init_model
 
 
 def configure_pyro4(config):
     """
     Configure Pyro4 based on `config`.
 
     This will mainly set the different configuration parameters of the Pyro4
     library based on the settings in our INI files. The Pyro4 documentation
     lists more details about the specific settings and their meaning.
     """
     Pyro4.config.COMMTIMEOUT = float(config['vcs.connection_timeout'])
     Pyro4.config.SERIALIZER = 'pickle'
     Pyro4.config.SERIALIZERS_ACCEPTED.add('pickle')
 
     # Note: We need server configuration in the WSGI processes
     # because we provide a callback server in certain vcs operations.
     Pyro4.config.SERVERTYPE = "multiplex"
     Pyro4.config.POLLTIMEOUT = 0.01
 
 
 def configure_vcs(config):
     """
     Patch VCS config with some RhodeCode specific stuff
     """
     from rhodecode.lib.vcs import conf
     from rhodecode.lib.utils2 import aslist
     conf.settings.BACKENDS = {
         'hg': 'rhodecode.lib.vcs.backends.hg.MercurialRepository',
         'git': 'rhodecode.lib.vcs.backends.git.GitRepository',
         'svn': 'rhodecode.lib.vcs.backends.svn.SubversionRepository',
     }
 
     conf.settings.HG_USE_REBASE_FOR_MERGING = config.get(
         'rhodecode_hg_use_rebase_for_merging', False)
     conf.settings.GIT_REV_FILTER = shlex.split(
         config.get('git_rev_filter', '--all').strip())
     conf.settings.DEFAULT_ENCODINGS = aslist(config.get('default_encoding',
                                                         'UTF-8'), sep=',')
     conf.settings.ALIASES[:] = config.get('vcs.backends')
     conf.settings.SVN_COMPATIBLE_VERSION = config.get(
         'vcs.svn.compatible_version')
 
 
 def initialize_database(config):
     from rhodecode.lib.utils2 import engine_from_config
     engine = engine_from_config(config, 'sqlalchemy.db1.')
     init_model(engine, encryption_key=config['beaker.session.secret'])
 
 
+def initialize_test_environment(settings, test_env=None):
+    if test_env is None:
+        test_env = not int(os.environ.get('RC_NO_TMP_PATH', 0))
+
+    from rhodecode.lib.utils import create_test_env, create_test_index
+    from rhodecode.tests import TESTS_TMP_PATH
+    # test repos
+    if test_env:
+        create_test_env(TESTS_TMP_PATH, settings)
+        create_test_index(TESTS_TMP_PATH, settings, True)
+
+
 def get_vcs_server_protocol(config):
     protocol = config.get('vcs.server.protocol', 'pyro4')
     return protocol
 
 
 def set_instance_id(config):
     """ Sets a dynamic generated config['instance_id'] if missing or '*' """
 
     config['instance_id'] = config.get('instance_id') or ''
     if config['instance_id'] == '*' or not config['instance_id']:
         _platform_id = platform.uname()[1] or 'instance'
         config['instance_id'] = '%s-%s' % (_platform_id, os.getpid())

rhodecode/controllers/admin/my_account.py

@@ -1,373 +1,348 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2013-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 my account controller for RhodeCode admin
 """
 
 import logging
 
 import formencode
 from formencode import htmlfill
 from pylons import request, tmpl_context as c, url, session
 from pylons.controllers.util import redirect
 from pylons.i18n.translation import _
 from sqlalchemy.orm import joinedload
 
 from rhodecode.lib import helpers as h
 from rhodecode.lib import auth
 from rhodecode.lib.auth import (
     LoginRequired, NotAnonymous, AuthUser, generate_auth_token)
 from rhodecode.lib.base import BaseController, render
 from rhodecode.lib.utils2 import safe_int, md5
 from rhodecode.lib.ext_json import json
-from rhodecode.model.db import (Repository, PullRequest, PullRequestReviewers,
-                                UserEmailMap, User, UserFollowing,
-                                ExternalIdentity)
+from rhodecode.model.db import (
+    Repository, PullRequest, PullRequestReviewers, UserEmailMap, User,
+    UserFollowing)
 from rhodecode.model.forms import UserForm, PasswordChangeForm
 from rhodecode.model.scm import RepoList
 from rhodecode.model.user import UserModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.meta import Session
-from rhodecode.model.settings import SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 class MyAccountController(BaseController):
     """REST Controller styled on the Atom Publishing Protocol"""
     # To properly map this controller, ensure your config/routing.py
     # file has a resource setup:
     #     map.resource('setting', 'settings', controller='admin/settings',
     #         path_prefix='/admin', name_prefix='admin_')
 
     @LoginRequired()
     @NotAnonymous()
     def __before__(self):
         super(MyAccountController, self).__before__()
 
     def __load_data(self):
         c.user = User.get(c.rhodecode_user.user_id)
         if c.user.username == User.DEFAULT_USER:
             h.flash(_("You can't edit this user since it's"
                       " crucial for entire application"), category='warning')
             return redirect(url('users'))
 
     def _load_my_repos_data(self, watched=False):
         if watched:
             admin = False
             follows_repos = Session().query(UserFollowing)\
                 .filter(UserFollowing.user_id == c.rhodecode_user.user_id)\
                 .options(joinedload(UserFollowing.follows_repository))\
                 .all()
             repo_list = [x.follows_repository for x in follows_repos]
         else:
             admin = True
             repo_list = Repository.get_all_repos(
                 user_id=c.rhodecode_user.user_id)
             repo_list = RepoList(repo_list, perm_set=[
                 'repository.read', 'repository.write', 'repository.admin'])
 
         repos_data = RepoModel().get_repos_as_dict(
             repo_list=repo_list, admin=admin)
         # json used to render the grid
         return json.dumps(repos_data)
 
     @auth.CSRFRequired()
     def my_account_update(self):
         """
         POST /_admin/my_account Updates info of my account
         """
         # url('my_account')
         c.active = 'profile_edit'
         self.__load_data()
         c.perm_user = AuthUser(user_id=c.rhodecode_user.user_id,
                                ip_addr=self.ip_addr)
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
 
         defaults = c.user.get_dict()
         update = False
         _form = UserForm(edit=True,
                          old_data={'user_id': c.rhodecode_user.user_id,
                                    'email': c.rhodecode_user.email})()
         form_result = {}
         try:
             post_data = dict(request.POST)
             post_data['new_password'] = ''
             post_data['password_confirmation'] = ''
             form_result = _form.to_python(post_data)
             # skip updating those attrs for my account
             skip_attrs = ['admin', 'active', 'extern_type', 'extern_name',
                           'new_password', 'password_confirmation']
             # TODO: plugin should define if username can be updated
             if c.extern_type != "rhodecode":
                 # forbid updating username for external accounts
                 skip_attrs.append('username')
 
             UserModel().update_user(
                 c.rhodecode_user.user_id, skip_attrs=skip_attrs, **form_result)
             h.flash(_('Your account was updated successfully'),
                     category='success')
             Session().commit()
             update = True
 
         except formencode.Invalid as errors:
             return htmlfill.render(
                 render('admin/my_account/my_account.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False)
         except Exception:
             log.exception("Exception updating user")
             h.flash(_('Error occurred during update of user %s')
                     % form_result.get('username'), category='error')
 
         if update:
             return redirect('my_account')
 
         return htmlfill.render(
             render('admin/my_account/my_account.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
 
     def my_account(self):
         """
         GET /_admin/my_account Displays info about my account
         """
         # url('my_account')
         c.active = 'profile'
         self.__load_data()
 
         defaults = c.user.get_dict()
         return htmlfill.render(
             render('admin/my_account/my_account.html'),
             defaults=defaults, encoding="UTF-8", force_defaults=False)
 
     def my_account_edit(self):
         """
         GET /_admin/my_account/edit Displays edit form of my account
         """
         c.active = 'profile_edit'
         self.__load_data()
         c.perm_user = AuthUser(user_id=c.rhodecode_user.user_id,
                                ip_addr=self.ip_addr)
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
 
         defaults = c.user.get_dict()
         return htmlfill.render(
             render('admin/my_account/my_account.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
 
     @auth.CSRFRequired()
     def my_account_password_update(self):
         c.active = 'password'
         self.__load_data()
         _form = PasswordChangeForm(c.rhodecode_user.username)()
         try:
             form_result = _form.to_python(request.POST)
             UserModel().update_user(c.rhodecode_user.user_id, **form_result)
             instance = c.rhodecode_user.get_instance()
             instance.update_userdata(force_password_change=False)
             Session().commit()
             session.setdefault('rhodecode_user', {}).update(
                 {'password': md5(instance.password)})
             session.save()
             h.flash(_("Successfully updated password"), category='success')
         except formencode.Invalid as errors:
             return htmlfill.render(
                 render('admin/my_account/my_account.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False)
         except Exception:
             log.exception("Exception updating password")
             h.flash(_('Error occurred during update of user password'),
                     category='error')
         return render('admin/my_account/my_account.html')
 
     def my_account_password(self):
         c.active = 'password'
         self.__load_data()
         return render('admin/my_account/my_account.html')
 
     def my_account_repos(self):
         c.active = 'repos'
         self.__load_data()
 
         # json used to render the grid
         c.data = self._load_my_repos_data()
         return render('admin/my_account/my_account.html')
 
     def my_account_watched(self):
         c.active = 'watched'
         self.__load_data()
 
         # json used to render the grid
         c.data = self._load_my_repos_data(watched=True)
         return render('admin/my_account/my_account.html')
 
     def my_account_perms(self):
         c.active = 'perms'
         self.__load_data()
         c.perm_user = AuthUser(user_id=c.rhodecode_user.user_id,
                                ip_addr=self.ip_addr)
 
         return render('admin/my_account/my_account.html')
 
     def my_account_emails(self):
         c.active = 'emails'
         self.__load_data()
 
         c.user_email_map = UserEmailMap.query()\
             .filter(UserEmailMap.user == c.user).all()
         return render('admin/my_account/my_account.html')
 
     @auth.CSRFRequired()
     def my_account_emails_add(self):
         email = request.POST.get('new_email')
 
         try:
             UserModel().add_extra_email(c.rhodecode_user.user_id, email)
             Session().commit()
             h.flash(_("Added new email address `%s` for user account") % email,
                     category='success')
         except formencode.Invalid as error:
             msg = error.error_dict['email']
             h.flash(msg, category='error')
         except Exception:
             log.exception("Exception in my_account_emails")
             h.flash(_('An error occurred during email saving'),
                     category='error')
         return redirect(url('my_account_emails'))
 
     @auth.CSRFRequired()
     def my_account_emails_delete(self):
         email_id = request.POST.get('del_email_id')
         user_model = UserModel()
         user_model.delete_extra_email(c.rhodecode_user.user_id, email_id)
         Session().commit()
         h.flash(_("Removed email address from user account"),
                 category='success')
         return redirect(url('my_account_emails'))
 
     def my_account_pullrequests(self):
         c.active = 'pullrequests'
         self.__load_data()
         c.show_closed = request.GET.get('pr_show_closed')
 
         def _filter(pr):
             s = sorted(pr, key=lambda o: o.created_on, reverse=True)
             if not c.show_closed:
                 s = filter(lambda p: p.status != PullRequest.STATUS_CLOSED, s)
             return s
 
         c.my_pull_requests = _filter(
             PullRequest.query().filter(
                 PullRequest.user_id == c.rhodecode_user.user_id).all())
         my_prs = [
             x.pull_request for x in PullRequestReviewers.query().filter(
                 PullRequestReviewers.user_id == c.rhodecode_user.user_id).all()]
         c.participate_in_pull_requests = _filter(my_prs)
         return render('admin/my_account/my_account.html')
 
     def my_account_auth_tokens(self):
         c.active = 'auth_tokens'
         self.__load_data()
         show_expired = True
         c.lifetime_values = [
             (str(-1), _('forever')),
             (str(5), _('5 minutes')),
             (str(60), _('1 hour')),
             (str(60 * 24), _('1 day')),
             (str(60 * 24 * 30), _('1 month')),
         ]
         c.lifetime_options = [(c.lifetime_values, _("Lifetime"))]
         c.role_values = [(x, AuthTokenModel.cls._get_role_name(x))
                          for x in AuthTokenModel.cls.ROLES]
         c.role_options = [(c.role_values, _("Role"))]
         c.user_auth_tokens = AuthTokenModel().get_auth_tokens(
             c.rhodecode_user.user_id, show_expired=show_expired)
         return render('admin/my_account/my_account.html')
 
     @auth.CSRFRequired()
     def my_account_auth_tokens_add(self):
         lifetime = safe_int(request.POST.get('lifetime'), -1)
         description = request.POST.get('description')
         role = request.POST.get('role')
         AuthTokenModel().create(c.rhodecode_user.user_id, description, lifetime,
                                 role)
         Session().commit()
         h.flash(_("Auth token successfully created"), category='success')
         return redirect(url('my_account_auth_tokens'))
 
     @auth.CSRFRequired()
     def my_account_auth_tokens_delete(self):
         auth_token = request.POST.get('del_auth_token')
         user_id = c.rhodecode_user.user_id
         if request.POST.get('del_auth_token_builtin'):
             user = User.get(user_id)
             if user:
                 user.api_key = generate_auth_token(user.username)
                 Session().add(user)
                 Session().commit()
                 h.flash(_("Auth token successfully reset"), category='success')
         elif auth_token:
             AuthTokenModel().delete(auth_token, c.rhodecode_user.user_id)
             Session().commit()
             h.flash(_("Auth token successfully deleted"), category='success')
 
         return redirect(url('my_account_auth_tokens'))
-
-    def my_account_oauth(self):
-        c.active = 'oauth'
-        self.__load_data()
-        c.user_oauth_tokens = ExternalIdentity().by_local_user_id(
-            c.rhodecode_user.user_id).all()
-        settings = SettingsModel().get_all_settings()
-        c.social_plugins = SettingsModel().list_enabled_social_plugins(
-            settings)
-        return render('admin/my_account/my_account.html')
-
-    @auth.CSRFRequired()
-    def my_account_oauth_delete(self):
-        token = ExternalIdentity.by_external_id_and_provider(
-            request.params.get('external_id'),
-            request.params.get('provider_name'),
-            local_user_id=c.rhodecode_user.user_id
-        )
-        if token:
-            Session().delete(token)
-            Session().commit()
-            h.flash(_("OAuth token successfully deleted"), category='success')
-
-        return redirect(url('my_account_oauth'))

rhodecode/controllers/admin/user_groups.py

@@ -1,471 +1,480 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 User Groups crud controller for pylons
 """
 
 import logging
 import formencode
 
 from formencode import htmlfill
 from pylons import request, tmpl_context as c, url, config
 from pylons.controllers.util import redirect
 from pylons.i18n.translation import _
 
 from sqlalchemy.orm import joinedload
 
 from rhodecode.lib import auth
 from rhodecode.lib import helpers as h
 from rhodecode.lib.exceptions import UserGroupAssignedException,\
     RepoGroupAssignmentError
+from rhodecode.lib.utils import jsonify, action_logger
 from rhodecode.lib.utils2 import safe_unicode, str2bool, safe_int
 from rhodecode.lib.auth import (
     LoginRequired, NotAnonymous, HasUserGroupPermissionAnyDecorator,
     HasPermissionAnyDecorator)
 from rhodecode.lib.base import BaseController, render
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.scm import UserGroupList
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.db import (
     User, UserGroup, UserGroupRepoToPerm, UserGroupRepoGroupToPerm)
 from rhodecode.model.forms import (
     UserGroupForm, UserGroupPermsForm, UserIndividualPermissionsForm,
     UserPermissionsForm)
 from rhodecode.model.meta import Session
 from rhodecode.lib.utils import action_logger
 from rhodecode.lib.ext_json import json
 
 log = logging.getLogger(__name__)
 
 
 class UserGroupsController(BaseController):
     """REST Controller styled on the Atom Publishing Protocol"""
 
     @LoginRequired()
     def __before__(self):
         super(UserGroupsController, self).__before__()
         c.available_permissions = config['available_permissions']
         PermissionModel().set_global_permission_choices(c, translator=_)
 
     def __load_data(self, user_group_id):
         c.group_members_obj = [x.user for x in c.user_group.members]
         c.group_members_obj.sort(key=lambda u: u.username.lower())
 
         c.group_members = [(x.user_id, x.username) for x in c.group_members_obj]
 
         c.available_members = [(x.user_id, x.username)
                                for x in User.query().all()]
         c.available_members.sort(key=lambda u: u[1].lower())
 
     def __load_defaults(self, user_group_id):
         """
         Load defaults settings for edit, and update
 
         :param user_group_id:
         """
         user_group = UserGroup.get_or_404(user_group_id)
         data = user_group.get_dict()
         # fill owner
         if user_group.user:
             data.update({'user': user_group.user.username})
         else:
             replacement_user = User.get_first_admin().username
             data.update({'user': replacement_user})
         return data
 
     def _revoke_perms_on_yourself(self, form_result):
         _updates = filter(lambda u: c.rhodecode_user.user_id == int(u[0]),
                           form_result['perm_updates'])
         _additions = filter(lambda u: c.rhodecode_user.user_id == int(u[0]),
                             form_result['perm_additions'])
         _deletions = filter(lambda u: c.rhodecode_user.user_id == int(u[0]),
                             form_result['perm_deletions'])
         admin_perm = 'usergroup.admin'
         if _updates   and _updates[0][1]   != admin_perm or \
            _additions and _additions[0][1] != admin_perm or \
            _deletions and _deletions[0][1] != admin_perm:
             return True
         return False
 
     # permission check inside
     @NotAnonymous()
     def index(self):
         """GET /users_groups: All items in the collection"""
         # url('users_groups')
 
         from rhodecode.lib.utils import PartialRenderer
         _render = PartialRenderer('data_table/_dt_elements.html')
 
         def user_group_name(user_group_id, user_group_name):
             return _render("user_group_name", user_group_id, user_group_name)
 
         def user_group_actions(user_group_id, user_group_name):
             return _render("user_group_actions", user_group_id, user_group_name)
 
         ## json generate
         group_iter = UserGroupList(UserGroup.query().all(),
                                    perm_set=['usergroup.admin'])
 
         user_groups_data = []
         for user_gr in group_iter:
             user_groups_data.append({
                 "group_name": user_group_name(
                     user_gr.users_group_id, h.escape(user_gr.users_group_name)),
                 "group_name_raw": user_gr.users_group_name,
                 "desc": h.escape(user_gr.user_group_description),
                 "members": len(user_gr.members),
                 "active": h.bool2icon(user_gr.users_group_active),
                 "owner": h.escape(h.link_to_user(user_gr.user.username)),
                 "action": user_group_actions(
                     user_gr.users_group_id, user_gr.users_group_name)
             })
 
         c.data = json.dumps(user_groups_data)
         return render('admin/user_groups/user_groups.html')
 
     @HasPermissionAnyDecorator('hg.admin', 'hg.usergroup.create.true')
     @auth.CSRFRequired()
     def create(self):
         """POST /users_groups: Create a new item"""
         # url('users_groups')
 
         users_group_form = UserGroupForm()()
         try:
             form_result = users_group_form.to_python(dict(request.POST))
             user_group = UserGroupModel().create(
                 name=form_result['users_group_name'],
                 description=form_result['user_group_description'],
                 owner=c.rhodecode_user.user_id,
                 active=form_result['users_group_active'])
             Session().flush()
 
             user_group_name = form_result['users_group_name']
             action_logger(c.rhodecode_user,
                           'admin_created_users_group:%s' % user_group_name,
                           None, self.ip_addr, self.sa)
             user_group_link = h.link_to(h.escape(user_group_name),
                                         url('edit_users_group',
                                             user_group_id=user_group.users_group_id))
             h.flash(h.literal(_('Created user group %(user_group_link)s')
                               % {'user_group_link': user_group_link}),
                     category='success')
             Session().commit()
         except formencode.Invalid as errors:
             return htmlfill.render(
                 render('admin/user_groups/user_group_add.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False)
         except Exception:
             log.exception("Exception creating user group")
             h.flash(_('Error occurred during creation of user group %s') \
                     % request.POST.get('users_group_name'), category='error')
 
-        return redirect(url('users_groups'))
+        return redirect(
+            url('edit_users_group', user_group_id=user_group.users_group_id))
 
     @HasPermissionAnyDecorator('hg.admin', 'hg.usergroup.create.true')
     def new(self):
         """GET /user_groups/new: Form to create a new item"""
         # url('new_users_group')
         return render('admin/user_groups/user_group_add.html')
 
     @HasUserGroupPermissionAnyDecorator('usergroup.admin')
     @auth.CSRFRequired()
     def update(self, user_group_id):
         """PUT /user_groups/user_group_id: Update an existing item"""
         # Forms posted to this method should contain a hidden field:
         #    <input type="hidden" name="_method" value="PUT" />
         # Or using helpers:
         #    h.form(url('users_group', user_group_id=ID),
         #           method='put')
         # url('users_group', user_group_id=ID)
 
         user_group_id = safe_int(user_group_id)
         c.user_group = UserGroup.get_or_404(user_group_id)
         c.active = 'settings'
         self.__load_data(user_group_id)
 
         available_members = [safe_unicode(x[0]) for x in c.available_members]
 
         users_group_form = UserGroupForm(edit=True,
                                          old_data=c.user_group.get_dict(),
                                          available_members=available_members)()
 
         try:
             form_result = users_group_form.to_python(request.POST)
             UserGroupModel().update(c.user_group, form_result)
             gr = form_result['users_group_name']
             action_logger(c.rhodecode_user,
                           'admin_updated_users_group:%s' % gr,
                           None, self.ip_addr, self.sa)
             h.flash(_('Updated user group %s') % gr, category='success')
             Session().commit()
         except formencode.Invalid as errors:
             defaults = errors.value
             e = errors.error_dict or {}
 
             return htmlfill.render(
                 render('admin/user_groups/user_group_edit.html'),
                 defaults=defaults,
                 errors=e,
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False)
         except Exception:
             log.exception("Exception during update of user group")
             h.flash(_('Error occurred during update of user group %s')
                     % request.POST.get('users_group_name'), category='error')
 
         return redirect(url('edit_users_group', user_group_id=user_group_id))
 
     @HasUserGroupPermissionAnyDecorator('usergroup.admin')
     @auth.CSRFRequired()
     def delete(self, user_group_id):
         """DELETE /user_groups/user_group_id: Delete an existing item"""
         # Forms posted to this method should contain a hidden field:
         #    <input type="hidden" name="_method" value="DELETE" />
         # Or using helpers:
         #    h.form(url('users_group', user_group_id=ID),
         #           method='delete')
         # url('users_group', user_group_id=ID)
         user_group_id = safe_int(user_group_id)
         c.user_group = UserGroup.get_or_404(user_group_id)
         force = str2bool(request.POST.get('force'))
 
         try:
             UserGroupModel().delete(c.user_group, force=force)
             Session().commit()
             h.flash(_('Successfully deleted user group'), category='success')
         except UserGroupAssignedException as e:
             h.flash(str(e), category='error')
         except Exception:
             log.exception("Exception during deletion of user group")
             h.flash(_('An error occurred during deletion of user group'),
                     category='error')
         return redirect(url('users_groups'))
 
     @HasUserGroupPermissionAnyDecorator('usergroup.admin')
     def edit(self, user_group_id):
         """GET /user_groups/user_group_id/edit: Form to edit an existing item"""
         # url('edit_users_group', user_group_id=ID)
 
         user_group_id = safe_int(user_group_id)
         c.user_group = UserGroup.get_or_404(user_group_id)
         c.active = 'settings'
         self.__load_data(user_group_id)
 
         defaults = self.__load_defaults(user_group_id)
 
         return htmlfill.render(
             render('admin/user_groups/user_group_edit.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
 
     @HasUserGroupPermissionAnyDecorator('usergroup.admin')
     def edit_perms(self, user_group_id):
         user_group_id = safe_int(user_group_id)
         c.user_group = UserGroup.get_or_404(user_group_id)
         c.active = 'perms'
 
         defaults = {}
         # fill user group users
         for p in c.user_group.user_user_group_to_perm:
             defaults.update({'u_perm_%s' % p.user.user_id:
                              p.permission.permission_name})
 
         for p in c.user_group.user_group_user_group_to_perm:
             defaults.update({'g_perm_%s' % p.user_group.users_group_id:
                              p.permission.permission_name})
 
         return htmlfill.render(
             render('admin/user_groups/user_group_edit.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
 
     @HasUserGroupPermissionAnyDecorator('usergroup.admin')
     @auth.CSRFRequired()
     def update_perms(self, user_group_id):
         """
         grant permission for given usergroup
 
         :param user_group_id:
         """
         user_group_id = safe_int(user_group_id)
         c.user_group = UserGroup.get_or_404(user_group_id)
         form = UserGroupPermsForm()().to_python(request.POST)
 
         if not c.rhodecode_user.is_admin:
             if self._revoke_perms_on_yourself(form):
                 msg = _('Cannot change permission for yourself as admin')
                 h.flash(msg, category='warning')
                 return redirect(url('edit_user_group_perms', user_group_id=user_group_id))
 
         try:
             UserGroupModel().update_permissions(user_group_id,
                 form['perm_additions'], form['perm_updates'], form['perm_deletions'])
         except RepoGroupAssignmentError:
             h.flash(_('Target group cannot be the same'), category='error')
             return redirect(url('edit_user_group_perms', user_group_id=user_group_id))
         #TODO: implement this
         #action_logger(c.rhodecode_user, 'admin_changed_repo_permissions',
         #              repo_name, self.ip_addr, self.sa)
         Session().commit()
         h.flash(_('User Group permissions updated'), category='success')
         return redirect(url('edit_user_group_perms', user_group_id=user_group_id))
 
     @HasUserGroupPermissionAnyDecorator('usergroup.admin')
     def edit_perms_summary(self, user_group_id):
         user_group_id = safe_int(user_group_id)
         c.user_group = UserGroup.get_or_404(user_group_id)
         c.active = 'perms_summary'
         permissions = {
             'repositories': {},
             'repositories_groups': {},
         }
         ugroup_repo_perms = UserGroupRepoToPerm.query()\
             .options(joinedload(UserGroupRepoToPerm.permission))\
             .options(joinedload(UserGroupRepoToPerm.repository))\
             .filter(UserGroupRepoToPerm.users_group_id == user_group_id)\
             .all()
 
         for gr in ugroup_repo_perms:
             permissions['repositories'][gr.repository.repo_name]  \
                 = gr.permission.permission_name
 
         ugroup_group_perms = UserGroupRepoGroupToPerm.query()\
             .options(joinedload(UserGroupRepoGroupToPerm.permission))\
             .options(joinedload(UserGroupRepoGroupToPerm.group))\
             .filter(UserGroupRepoGroupToPerm.users_group_id == user_group_id)\
             .all()
 
         for gr in ugroup_group_perms:
             permissions['repositories_groups'][gr.group.group_name] \
                 = gr.permission.permission_name
         c.permissions = permissions
         return render('admin/user_groups/user_group_edit.html')
 
     @HasUserGroupPermissionAnyDecorator('usergroup.admin')
     def edit_global_perms(self, user_group_id):
         user_group_id = safe_int(user_group_id)
         c.user_group = UserGroup.get_or_404(user_group_id)
         c.active = 'global_perms'
 
         c.default_user = User.get_default_user()
         defaults = c.user_group.get_dict()
         defaults.update(c.default_user.get_default_perms(suffix='_inherited'))
         defaults.update(c.user_group.get_default_perms())
 
         return htmlfill.render(
             render('admin/user_groups/user_group_edit.html'),
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
 
     @HasUserGroupPermissionAnyDecorator('usergroup.admin')
     @auth.CSRFRequired()
     def update_global_perms(self, user_group_id):
         """PUT /users_perm/user_group_id: Update an existing item"""
         # url('users_group_perm', user_group_id=ID, method='put')
         user_group_id = safe_int(user_group_id)
         user_group = UserGroup.get_or_404(user_group_id)
         c.active = 'global_perms'
 
         try:
             # first stage that verifies the checkbox
             _form = UserIndividualPermissionsForm()
             form_result = _form.to_python(dict(request.POST))
             inherit_perms = form_result['inherit_default_permissions']
             user_group.inherit_default_permissions = inherit_perms
             Session().add(user_group)
 
             if not inherit_perms:
                 # only update the individual ones if we un check the flag
                 _form = UserPermissionsForm(
                     [x[0] for x in c.repo_create_choices],
                     [x[0] for x in c.repo_create_on_write_choices],
                     [x[0] for x in c.repo_group_create_choices],
                     [x[0] for x in c.user_group_create_choices],
                     [x[0] for x in c.fork_choices],
                     [x[0] for x in c.inherit_default_permission_choices])()
 
                 form_result = _form.to_python(dict(request.POST))
                 form_result.update({'perm_user_group_id': user_group.users_group_id})
 
                 PermissionModel().update_user_group_permissions(form_result)
 
             Session().commit()
             h.flash(_('User Group global permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             defaults = errors.value
             c.user_group = user_group
             return htmlfill.render(
                 render('admin/user_groups/user_group_edit.html'),
                 defaults=defaults,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False)
 
         except Exception:
             log.exception("Exception during permissions saving")
             h.flash(_('An error occurred during permissions saving'),
                     category='error')
 
         return redirect(url('edit_user_group_global_perms', user_group_id=user_group_id))
 
     @HasUserGroupPermissionAnyDecorator('usergroup.admin')
     def edit_advanced(self, user_group_id):
         user_group_id = safe_int(user_group_id)
         c.user_group = UserGroup.get_or_404(user_group_id)
         c.active = 'advanced'
         c.group_members_obj = sorted(
             (x.user for x in c.user_group.members),
             key=lambda u: u.username.lower())
 
         c.group_to_repos = sorted(
             (x.repository for x in c.user_group.users_group_repo_to_perm),
             key=lambda u: u.repo_name.lower())
 
         c.group_to_repo_groups = sorted(
             (x.group for x in c.user_group.users_group_repo_group_to_perm),
             key=lambda u: u.group_name.lower())
 
         return render('admin/user_groups/user_group_edit.html')
 
     @HasUserGroupPermissionAnyDecorator('usergroup.admin')
     def edit_members(self, user_group_id):
         user_group_id = safe_int(user_group_id)
         c.user_group = UserGroup.get_or_404(user_group_id)
         c.active = 'members'
         c.group_members_obj = sorted((x.user for x in c.user_group.members),
                                      key=lambda u: u.username.lower())
 
-        c.group_members = [(x.user_id, x.username) for x in c.group_members_obj]
+        group_members = [(x.user_id, x.username) for x in c.group_members_obj]
+
+        if request.is_xhr:
+            return jsonify(lambda *a, **k: {
+                'members': group_members
+            })
+
+        c.group_members = group_members
         return render('admin/user_groups/user_group_edit.html')

rhodecode/controllers/compare.py

@@ -1,265 +1,267 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Compare controller for showing differences between two commits/refs/tags etc.
 """
 
 import logging
 
 from webob.exc import HTTPBadRequest
 from pylons import request, tmpl_context as c, url
 from pylons.controllers.util import redirect
 from pylons.i18n.translation import _
 
 from rhodecode.controllers.utils import parse_path_ref, get_commit_from_ref_name
 from rhodecode.lib import helpers as h
 from rhodecode.lib import diffs
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
 from rhodecode.lib.utils import safe_str
 from rhodecode.lib.utils2 import safe_unicode, str2bool
 from rhodecode.lib.vcs.exceptions import (
     EmptyRepositoryError, RepositoryError, RepositoryRequirementError)
 from rhodecode.model.db import Repository, ChangesetStatus
 
 log = logging.getLogger(__name__)
 
 
 class CompareController(BaseRepoController):
 
     def __before__(self):
         super(CompareController, self).__before__()
 
     def _get_commit_or_redirect(
             self, ref, ref_type, repo, redirect_after=True, partial=False):
         """
         This is a safe way to get a commit. If an error occurs it
         redirects to a commit with a proper message. If partial is set
         then it does not do redirect raise and throws an exception instead.
         """
         try:
             return get_commit_from_ref_name(repo, safe_str(ref), ref_type)
         except EmptyRepositoryError:
             if not redirect_after:
                 return repo.scm_instance().EMPTY_COMMIT
             h.flash(h.literal(_('There are no commits yet')),
                     category='warning')
             redirect(url('summary_home', repo_name=repo.repo_name))
 
         except RepositoryError as e:
             msg = safe_str(e)
             log.exception(msg)
             h.flash(msg, category='warning')
             if not partial:
                 redirect(h.url('summary_home', repo_name=repo.repo_name))
             raise HTTPBadRequest()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def index(self, repo_name):
         c.compare_home = True
         c.commit_ranges = []
         c.files = []
         c.limited_diff = False
         source_repo = c.rhodecode_db_repo.repo_name
         target_repo = request.GET.get('target_repo', source_repo)
         c.source_repo = Repository.get_by_repo_name(source_repo)
         c.target_repo = Repository.get_by_repo_name(target_repo)
         c.source_ref = c.target_ref = _('Select commit')
         c.source_ref_type = ""
         c.target_ref_type = ""
         c.commit_statuses = ChangesetStatus.STATUSES
         c.preview_mode = False
         return render('compare/compare_diff.html')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def compare(self, repo_name, source_ref_type, source_ref,
                                  target_ref_type, target_ref):
         # source_ref will be evaluated in source_repo
         source_repo_name = c.rhodecode_db_repo.repo_name
         source_path, source_id = parse_path_ref(source_ref)
 
         # target_ref will be evaluated in target_repo
         target_repo_name = request.GET.get('target_repo', source_repo_name)
         target_path, target_id = parse_path_ref(target_ref)
 
         c.commit_statuses = ChangesetStatus.STATUSES
 
         # if merge is True
         #   Show what changes since the shared ancestor commit of target/source
         #   the source would get if it was merged with target. Only commits
         #   which are in target but not in source will be shown.
         merge = str2bool(request.GET.get('merge'))
         # if merge is False
         #   Show a raw diff of source/target refs even if no ancestor exists
 
 
         # c.fulldiff disables cut_off_limit
         c.fulldiff = str2bool(request.GET.get('fulldiff'))
 
         # if partial, returns just compare_commits.html (commits log)
         partial = request.is_xhr
 
         # swap url for compare_diff page
         c.swap_url = h.url(
             'compare_url',
             repo_name=target_repo_name,
             source_ref_type=target_ref_type,
             source_ref=target_ref,
             target_repo=source_repo_name,
             target_ref_type=source_ref_type,
             target_ref=source_ref,
             merge=merge and '1' or '')
 
         source_repo = Repository.get_by_repo_name(source_repo_name)
         target_repo = Repository.get_by_repo_name(target_repo_name)
 
         if source_repo is None:
             msg = _('Could not find the original repo: %(repo)s') % {
                 'repo': source_repo}
 
             log.error(msg)
             h.flash(msg, category='error')
             return redirect(url('compare_home', repo_name=c.repo_name))
 
         if target_repo is None:
             msg = _('Could not find the other repo: %(repo)s') % {
                 'repo': target_repo_name}
             log.error(msg)
             h.flash(msg, category='error')
             return redirect(url('compare_home', repo_name=c.repo_name))
 
         source_alias = source_repo.scm_instance().alias
         target_alias = target_repo.scm_instance().alias
         if source_alias != target_alias:
             msg = _('The comparison of two different kinds of remote repos '
                     'is not available')
             log.error(msg)
             h.flash(msg, category='error')
             return redirect(url('compare_home', repo_name=c.repo_name))
 
         source_commit = self._get_commit_or_redirect(
             ref=source_id, ref_type=source_ref_type, repo=source_repo,
             partial=partial)
         target_commit = self._get_commit_or_redirect(
             ref=target_id, ref_type=target_ref_type, repo=target_repo,
             partial=partial)
 
         c.compare_home = False
         c.source_repo = source_repo
         c.target_repo = target_repo
         c.source_ref = source_ref
         c.target_ref = target_ref
         c.source_ref_type = source_ref_type
         c.target_ref_type = target_ref_type
 
         source_scm = source_repo.scm_instance()
         target_scm = target_repo.scm_instance()
 
         pre_load = ["author", "branch", "date", "message"]
         c.ancestor = None
         try:
             c.commit_ranges = source_scm.compare(
                 source_commit.raw_id, target_commit.raw_id,
                 target_scm, merge, pre_load=pre_load)
             if merge:
                 c.ancestor = source_scm.get_common_ancestor(
                     source_commit.raw_id, target_commit.raw_id, target_scm)
         except RepositoryRequirementError:
             msg = _('Could not compare repos with different '
                     'large file settings')
             log.error(msg)
             if partial:
                 return msg
             h.flash(msg, category='error')
             return redirect(url('compare_home', repo_name=c.repo_name))
 
         c.statuses = c.rhodecode_db_repo.statuses(
             [x.raw_id for x in c.commit_ranges])
 
-        if partial:
+        if partial: # for PR ajax commits loader
+            if not c.ancestor:
+                return '' # cannot merge if there is no ancestor
             return render('compare/compare_commits.html')
 
         if c.ancestor:
             # case we want a simple diff without incoming commits,
             # previewing what will be merged.
             # Make the diff on target repo (which is known to have target_ref)
             log.debug('Using ancestor %s as source_ref instead of %s'
                       % (c.ancestor, source_ref))
             source_repo = target_repo
             source_commit = target_repo.get_commit(commit_id=c.ancestor)
 
         # diff_limit will cut off the whole diff if the limit is applied
         # otherwise it will just hide the big files from the front-end
         diff_limit = self.cut_off_limit_diff
         file_limit = self.cut_off_limit_file
 
         log.debug('calculating diff between '
                   'source_ref:%s and target_ref:%s for repo `%s`',
                   source_commit, target_commit,
                   safe_unicode(source_repo.scm_instance().path))
 
         if source_commit.repository != target_commit.repository:
             msg = _(
                 "Repositories unrelated. "
                 "Cannot compare commit %(commit1)s from repository %(repo1)s "
                 "with commit %(commit2)s from repository %(repo2)s.") % {
                     'commit1': h.show_id(source_commit),
                     'repo1': source_repo.repo_name,
                     'commit2': h.show_id(target_commit),
                     'repo2': target_repo.repo_name,
                 }
             h.flash(msg, category='error')
             raise HTTPBadRequest()
 
         txtdiff = source_repo.scm_instance().get_diff(
             commit1=source_commit, commit2=target_commit,
             path1=source_path, path=target_path)
         diff_processor = diffs.DiffProcessor(
             txtdiff, format='gitdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=c.fulldiff)
         _parsed = diff_processor.prepare()
 
         c.limited_diff = False
         if isinstance(_parsed, diffs.LimitedDiffContainer):
             c.limited_diff = True
 
         c.files = []
         c.changes = {}
         c.lines_added = 0
         c.lines_deleted = 0
         for f in _parsed:
             st = f['stats']
             if not st['binary']:
                 c.lines_added += st['added']
                 c.lines_deleted += st['deleted']
             fid = h.FID('', f['filename'])
             c.files.append([fid, f['operation'], f['filename'], f['stats'], f])
             htmldiff = diff_processor.as_html(
                 enable_comments=False, parsed_lines=[f])
             c.changes[fid] = [f['operation'], f['filename'], htmldiff, f]
 
         c.preview_mode = merge
 
         return render('compare/compare_diff.html')

rhodecode/controllers/home.py

@@ -1,232 +1,277 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Home controller for RhodeCode Enterprise
 """
 
 import logging
 import time
-
+import re
 
-from pylons import tmpl_context as c, request
+from pylons import tmpl_context as c, request, url, config
 from pylons.i18n.translation import _
 from sqlalchemy.sql import func
 
 from rhodecode.lib.auth import (
-    LoginRequired, HasPermissionAllDecorator,
+    LoginRequired, HasPermissionAllDecorator, AuthUser,
     HasRepoGroupPermissionAnyDecorator, XHRRequired)
 from rhodecode.lib.base import BaseController, render
+from rhodecode.lib.index import searcher_from_config
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils import jsonify
 from rhodecode.lib.utils2 import safe_unicode
 from rhodecode.model.db import Repository, RepoGroup
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.scm import RepoList, RepoGroupList
 
 
 log = logging.getLogger(__name__)
 
 
 class HomeController(BaseController):
     def __before__(self):
         super(HomeController, self).__before__()
 
     def ping(self):
         """
         Ping, doesn't require login, good for checking out the platform
         """
         instance_id = getattr(c, 'rhodecode_instanceid', '')
         return 'pong[%s] => %s' % (instance_id, self.ip_addr,)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def error_test(self):
         """
         Test exception handling and emails on errors
         """
         class TestException(Exception):
             pass
 
         msg = ('RhodeCode Enterprise %s test exception. Generation time: %s'
                % (c.rhodecode_name, time.time()))
         raise TestException(msg)
 
     def _get_groups_and_repos(self, repo_group_id=None):
         # repo groups groups
         repo_group_list = RepoGroup.get_all_repo_groups(group_id=repo_group_id)
         _perms = ['group.read', 'group.write', 'group.admin']
         repo_group_list_acl = RepoGroupList(repo_group_list, perm_set=_perms)
         repo_group_data = RepoGroupModel().get_repo_groups_as_dict(
             repo_group_list=repo_group_list_acl, admin=False)
 
         # repositories
         repo_list = Repository.get_all_repos(group_id=repo_group_id)
         _perms = ['repository.read', 'repository.write', 'repository.admin']
         repo_list_acl = RepoList(repo_list, perm_set=_perms)
         repo_data = RepoModel().get_repos_as_dict(
             repo_list=repo_list_acl, admin=False)
 
         return repo_data, repo_group_data
 
     @LoginRequired()
     def index(self):
         c.repo_group = None
 
         repo_data, repo_group_data = self._get_groups_and_repos()
         # json used to render the grids
         c.repos_data = json.dumps(repo_data)
         c.repo_groups_data = json.dumps(repo_group_data)
 
         return render('/index.html')
 
     @LoginRequired()
     @HasRepoGroupPermissionAnyDecorator('group.read', 'group.write',
                                         'group.admin')
     def index_repo_group(self, group_name):
         """GET /repo_group_name: Show a specific item"""
         c.repo_group = RepoGroupModel()._get_repo_group(group_name)
         repo_data, repo_group_data = self._get_groups_and_repos(
             c.repo_group.group_id)
 
         # json used to render the grids
         c.repos_data = json.dumps(repo_data)
         c.repo_groups_data = json.dumps(repo_group_data)
 
         return render('index_repo_group.html')
 
     def _get_repo_list(self, name_contains=None, repo_type=None, limit=20):
         query = Repository.query()\
             .order_by(func.length(Repository.repo_name))\
             .order_by(Repository.repo_name)
 
         if repo_type:
             query = query.filter(Repository.repo_type == repo_type)
 
         if name_contains:
             ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
             query = query.filter(
                 Repository.repo_name.ilike(ilike_expression))
             query = query.limit(limit)
 
         all_repos = query.all()
         repo_iter = self.scm_model.get_repos(all_repos)
         return [
             {
                 'id': obj['name'],
                 'text': obj['name'],
                 'type': 'repo',
-                'obj': obj['dbrepo']
+                'obj': obj['dbrepo'],
+                'url': url('summary_home', repo_name=obj['name'])
             }
             for obj in repo_iter]
 
     def _get_repo_group_list(self, name_contains=None, limit=20):
         query = RepoGroup.query()\
             .order_by(func.length(RepoGroup.group_name))\
             .order_by(RepoGroup.group_name)
 
         if name_contains:
             ilike_expression = u'%{}%'.format(safe_unicode(name_contains))
             query = query.filter(
                 RepoGroup.group_name.ilike(ilike_expression))
             query = query.limit(limit)
 
         all_groups = query.all()
         repo_groups_iter = self.scm_model.get_repo_groups(all_groups)
         return [
             {
                 'id': obj.group_name,
                 'text': obj.group_name,
                 'type': 'group',
-                'obj': {}
+                'obj': {},
+                'url': url('repo_group_home', group_name=obj.group_name)
             }
             for obj in repo_groups_iter]
 
+    def _get_hash_commit_list(self, hash_starts_with=None, limit=20):
+        if not hash_starts_with or len(hash_starts_with) < 3:
+            return []
+
+        commit_hashes = re.compile('([0-9a-f]{2,40})').findall(hash_starts_with)
+
+        if len(commit_hashes) != 1:
+            return []
+
+        commit_hash_prefix = commit_hashes[0]
+
+        auth_user = AuthUser(
+            user_id=c.rhodecode_user.user_id, ip_addr=self.ip_addr)
+        searcher = searcher_from_config(config)
+        result = searcher.search(
+            'commit_id:%s*' % commit_hash_prefix, 'commit', auth_user)
+
+        return [
+            {
+                'id': entry['commit_id'],
+                'text': entry['commit_id'],
+                'type': 'commit',
+                'obj': {'repo': entry['repository']},
+                'url': url('changeset_home',
+                    repo_name=entry['repository'], revision=entry['commit_id'])
+            }
+            for entry in result['results']]
+
     @LoginRequired()
     @XHRRequired()
     @jsonify
-    def repo_switcher_data(self):
+    def goto_switcher_data(self):
         query = request.GET.get('query')
-        log.debug('generating switcher repo/groups list, query %s', query)
+        log.debug('generating goto switcher list, query %s', query)
 
         res = []
         repo_groups = self._get_repo_group_list(query)
         if repo_groups:
             res.append({
                 'text': _('Groups'),
                 'children': repo_groups
             })
 
         repos = self._get_repo_list(query)
         if repos:
             res.append({
                 'text': _('Repositories'),
                 'children': repos
             })
 
+        commits = self._get_hash_commit_list(query)
+        if commits:
+            unique_repos = {}
+            for commit in commits:
+                unique_repos.setdefault(commit['obj']['repo'], []
+                    ).append(commit)
+
+            for repo in unique_repos:
+                res.append({
+                    'text': _('Commits in %(repo)s') % {'repo': repo},
+                    'children': unique_repos[repo]
+                })
+
         data = {
             'more': False,
             'results': res
         }
         return data
 
     @LoginRequired()
     @XHRRequired()
     @jsonify
     def repo_list_data(self):
         query = request.GET.get('query')
         repo_type = request.GET.get('repo_type')
         log.debug('generating repo list, query:%s', query)
 
         res = []
         repos = self._get_repo_list(query, repo_type=repo_type)
         if repos:
             res.append({
                 'text': _('Repositories'),
                 'children': repos
             })
+
         data = {
             'more': False,
             'results': res
         }
         return data
 
     @LoginRequired()
     @XHRRequired()
     @jsonify
     def user_autocomplete_data(self):
         query = request.GET.get('query')
 
         repo_model = RepoModel()
         _users = repo_model.get_users(name_contains=query)
 
         if request.GET.get('user_groups'):
             # extend with user groups
             _user_groups = repo_model.get_user_groups(name_contains=query)
             _users = _users + _user_groups
 
         return {'suggestions': _users}
 
     @LoginRequired()
     @XHRRequired()
     @jsonify
     def user_group_autocomplete_data(self):
         return {'suggestions': []}