Rewrite simehg for enabling cloning with raw url for anonymous access + some optimizations for making less queries when authenticating users.

added debug to test hg operations

rhodecode/controllers/summary.py

@@ -1,144 +1,147 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.controllers.summary
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Summary controller for Rhodecode
     
     :created_on: Apr 18, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <[e-mail]>    
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
 import calendar
 import logging
 from time import mktime
 from datetime import datetime, timedelta
 
 from vcs.exceptions import ChangesetError
 
 from pylons import tmpl_context as c, request, url
 from pylons.i18n.translation import _
 
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.db import Statistics
 
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseController, render
 from rhodecode.lib.utils import OrderedDict, EmptyChangeset
 
 from rhodecode.lib.celerylib import run_task
 from rhodecode.lib.celerylib.tasks import get_commits_stats
 
 from webhelpers.paginate import Page
 
 try:
     import json
 except ImportError:
     #python 2.5 compatibility
     import simplejson as json
 log = logging.getLogger(__name__)
 
 class SummaryController(BaseController):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
                                    'repository.admin')
     def __before__(self):
         super(SummaryController, self).__before__()
 
     def index(self):
         scm_model = ScmModel()
         c.repo_info = scm_model.get_repo(c.repo_name)
         c.following = scm_model.is_following_repo(c.repo_name,
                                              c.rhodecode_user.user_id)
         def url_generator(**kw):
             return url('shortlog_home', repo_name=c.repo_name, **kw)
 
         c.repo_changesets = Page(c.repo_info, page=1, items_per_page=10,
                                  url=url_generator)
 
         e = request.environ
 
         if self.rhodecode_user.username == 'default':
-            password = ':default'
+            #for default(anonymous) user we don't need to pass credentials
+            username = ''
+            password = ''
         else:
+            username = str(c.rhodecode_user.username)
             password = ''
 
         uri = u'%(protocol)s://%(user)s%(password)s@%(host)s%(prefix)s/%(repo_name)s' % {
                                         'protocol': e.get('wsgi.url_scheme'),
-                                        'user':str(c.rhodecode_user.username),
+                                        'user':username,
                                         'password':password,
                                         'host':e.get('HTTP_HOST'),
                                         'prefix':e.get('SCRIPT_NAME'),
                                         'repo_name':c.repo_name, }
         c.clone_repo_url = uri
         c.repo_tags = OrderedDict()
         for name, hash in c.repo_info.tags.items()[:10]:
             try:
                 c.repo_tags[name] = c.repo_info.get_changeset(hash)
             except ChangesetError:
                 c.repo_tags[name] = EmptyChangeset(hash)
 
         c.repo_branches = OrderedDict()
         for name, hash in c.repo_info.branches.items()[:10]:
             try:
                 c.repo_branches[name] = c.repo_info.get_changeset(hash)
             except ChangesetError:
                 c.repo_branches[name] = EmptyChangeset(hash)
 
         td = datetime.today() + timedelta(days=1)
         y, m, d = td.year, td.month, td.day
 
         ts_min_y = mktime((y - 1, (td - timedelta(days=calendar.mdays[m])).month,
                             d, 0, 0, 0, 0, 0, 0,))
         ts_min_m = mktime((y, (td - timedelta(days=calendar.mdays[m])).month,
                             d, 0, 0, 0, 0, 0, 0,))
 
         ts_max_y = mktime((y, m, d, 0, 0, 0, 0, 0, 0,))
         if c.repo_info.dbrepo.enable_statistics:
             c.no_data_msg = _('No data loaded yet')
             run_task(get_commits_stats, c.repo_info.name, ts_min_y, ts_max_y)
         else:
             c.no_data_msg = _('Statistics update are disabled for this repository')
         c.ts_min = ts_min_m
         c.ts_max = ts_max_y
 
         stats = self.sa.query(Statistics)\
             .filter(Statistics.repository == c.repo_info.dbrepo)\
             .scalar()
 
 
         if stats and stats.languages:
             c.no_data = False is c.repo_info.dbrepo.enable_statistics
             lang_stats = json.loads(stats.languages)
             c.commit_data = stats.commit_activity
             c.overview_data = stats.commit_activity_combined
             c.trending_languages = json.dumps(OrderedDict(
                                        sorted(lang_stats.items(), reverse=True,
                                             key=lambda k: k[1])[:10]
                                         )
                                     )
         else:
             c.commit_data = json.dumps({})
             c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 10] ])
             c.trending_languages = json.dumps({})
             c.no_data = True
 
         return render('summary/summary.html')
 

rhodecode/lib/middleware/simplehg.py

@@ -1,216 +1,275 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.lib.middleware.simplehg
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     SimpleHG middleware for handling mercurial protocol request 
     (push/clone etc.). It's implemented with basic auth function
     
     :created_on: Apr 28, 2010
     :author: marcink
     :copyright: (C) 2009-2010 Marcin Kuzminski <[e-mail]>    
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; version 2
 # of the License or (at your opinion) any later version of the license.
 # 
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301, USA.
 
+import os
+import logging
+import traceback
+
 from mercurial.error import RepoError
 from mercurial.hgweb import hgweb
 from mercurial.hgweb.request import wsgiapplication
+
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
+
 from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware
 from rhodecode.lib.utils import make_ui, invalidate_cache, \
     check_repo_fast, ui_sections
 from rhodecode.model.user import UserModel
+
 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError
-import logging
-import os
-import traceback
 
 log = logging.getLogger(__name__)
 
 def is_mercurial(environ):
     """Returns True if request's target is mercurial server - header
     ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
     """
     http_accept = environ.get('HTTP_ACCEPT')
     if http_accept and http_accept.startswith('application/mercurial'):
         return True
     return False
 
 class SimpleHg(object):
 
     def __init__(self, application, config):
         self.application = application
         self.config = config
         #authenticate this mercurial request using authfunc
         self.authenticate = AuthBasicAuthenticator('', authfunc)
         self.ipaddr = '0.0.0.0'
-        self.repository = None
+        self.repo_name = None
         self.username = None
         self.action = None
 
     def __call__(self, environ, start_response):
         if not is_mercurial(environ):
             return self.application(environ, start_response)
 
         proxy_key = 'HTTP_X_REAL_IP'
         def_key = 'REMOTE_ADDR'
         self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
-        #===================================================================
-        # AUTHENTICATE THIS MERCURIAL REQUEST
-        #===================================================================
-        username = REMOTE_USER(environ)
 
-        if not username:
-            self.authenticate.realm = self.config['rhodecode_realm']
-            result = self.authenticate(environ)
-            if isinstance(result, str):
-                AUTH_TYPE.update(environ, 'basic')
-                REMOTE_USER.update(environ, result)
-            else:
-                return result.wsgi_application(environ, start_response)
-
-        #=======================================================================
-        # GET REPOSITORY
-        #=======================================================================
+        #======================================================================
+        # GET ACTION PULL or PUSH
+        #======================================================================
+        self.action = self.__get_action(environ)
         try:
-            repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
-            if repo_name.endswith('/'):
-                repo_name = repo_name.rstrip('/')
-            self.repository = repo_name
+            #==================================================================
+            # GET REPOSITORY NAME
+            #==================================================================            
+            self.repo_name = self.__get_repository(environ)
         except:
-            log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
-        #===================================================================
-        # CHECK PERMISSIONS FOR THIS REQUEST
-        #===================================================================
-        self.action = self.__get_action(environ)
-        if self.action:
-            username = self.__get_environ_user(environ)
-            try:
-                user = self.__get_user(username)
-                self.username = user.username
-            except:
-                log.error(traceback.format_exc())
-                return HTTPInternalServerError()(environ, start_response)
+        #======================================================================
+        # CHECK ANONYMOUS PERMISSION
+        #======================================================================
+        if self.action in ['pull', 'push']:
+            anonymous_user = self.__get_user('default')
+            self.username = anonymous_user.username
+            anonymous_perm = self.__check_permission(self.action, anonymous_user ,
+                                           self.repo_name)
+
+            if anonymous_perm is not True or anonymous_user.active is False:
+                if anonymous_perm is not True:
+                    log.debug('Not enough credentials to access this repository'
+                              'as anonymous user')
+                if anonymous_user.active is False:
+                    log.debug('Anonymous access is disabled, running '
+                              'authentication')
+                #==================================================================
+                # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE NEED 
+                # TO AUTHENTICATE AND ASK FOR AUTHENTICATED USER PERMISSIONS
+                #==================================================================
 
-            #check permissions for this repository
-            if self.action == 'push':
-                if not HasPermissionAnyMiddleware('repository.write',
-                                                  'repository.admin')\
-                                                    (user, repo_name):
-                    return HTTPForbidden()(environ, start_response)
+                if not REMOTE_USER(environ):
+                    self.authenticate.realm = self.config['rhodecode_realm']
+                    result = self.authenticate(environ)
+                    if isinstance(result, str):
+                        AUTH_TYPE.update(environ, 'basic')
+                        REMOTE_USER.update(environ, result)
+                    else:
+                        return result.wsgi_application(environ, start_response)
+
 
-            else:
-                #any other action need at least read permission
-                if not HasPermissionAnyMiddleware('repository.read',
-                                                  'repository.write',
-                                                  'repository.admin')\
-                                                    (user, repo_name):
-                    return HTTPForbidden()(environ, start_response)
+                #==================================================================
+                # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME FROM
+                # BASIC AUTH
+                #==================================================================
+
+                if self.action in ['pull', 'push']:
+                    username = self.__get_environ_user(environ)
+                    try:
+                        user = self.__get_user(username)
+                        self.username = user.username
+                    except:
+                        log.error(traceback.format_exc())
+                        return HTTPInternalServerError()(environ, start_response)
+
+                    #check permissions for this repository
+                    perm = self.__check_permission(self.action, user, self.repo_name)
+                    if perm is not True:
+                        return HTTPForbidden()(environ, start_response)
 
         self.extras = {'ip':self.ipaddr,
                        'username':self.username,
                        'action':self.action,
-                       'repository':self.repository}
+                       'repository':self.repo_name}
 
         #===================================================================
         # MERCURIAL REQUEST HANDLING
         #===================================================================
         environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path
         self.baseui = make_ui('db')
         self.basepath = self.config['base_path']
-        self.repo_path = os.path.join(self.basepath, repo_name)
+        self.repo_path = os.path.join(self.basepath, self.repo_name)
 
         #quick check if that dir exists...
-        if check_repo_fast(repo_name, self.basepath):
+        if check_repo_fast(self.repo_name, self.basepath):
             return HTTPNotFound()(environ, start_response)
         try:
             app = wsgiapplication(self.__make_app)
         except RepoError, e:
             if str(e).find('not found') != -1:
                 return HTTPNotFound()(environ, start_response)
         except Exception:
             log.error(traceback.format_exc())
             return HTTPInternalServerError()(environ, start_response)
 
         #invalidate cache on push
         if self.action == 'push':
-            self.__invalidate_cache(repo_name)
+            self.__invalidate_cache(self.repo_name)
 
         return app(environ, start_response)
 
 
     def __make_app(self):
+        """Make an wsgi application using hgweb, and my generated baseui
+        instance
+        """
+
         hgserve = hgweb(str(self.repo_path), baseui=self.baseui)
         return  self.__load_web_settings(hgserve, self.extras)
 
+
+    def __check_permission(self, action, user, repo_name):
+        """Checks permissions using action (push/pull) user and repository
+        name
+        
+        :param action: push or pull action
+        :param user: user instance
+        :param repo_name: repository name
+        """
+        if action == 'push':
+            if not HasPermissionAnyMiddleware('repository.write',
+                                              'repository.admin')\
+                                                (user, repo_name):
+                return False
+
+        else:
+            #any other action need at least read permission
+            if not HasPermissionAnyMiddleware('repository.read',
+                                              'repository.write',
+                                              'repository.admin')\
+                                                (user, repo_name):
+                return False
+
+        return True
+
+
+    def __get_repository(self, environ):
+        """Get's repository name out of PATH_INFO header
+        
+        :param environ: environ where PATH_INFO is stored
+        """
+        try:
+            repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
+            if repo_name.endswith('/'):
+                repo_name = repo_name.rstrip('/')
+        except:
+            log.error(traceback.format_exc())
+            raise
+
+        return repo_name
+
     def __get_environ_user(self, environ):
         return environ.get('REMOTE_USER')
 
     def __get_user(self, username):
         return UserModel().get_by_username(username, cache=True)
 
     def __get_action(self, environ):
         """Maps mercurial request commands into a clone,pull or push command.
         This should always return a valid command string
+        
         :param environ:
         """
         mapping = {'changegroup': 'pull',
                    'changegroupsubset': 'pull',
                    'stream_out': 'pull',
                    'listkeys': 'pull',
                    'unbundle': 'push',
                    'pushkey': 'push', }
         for qry in environ['QUERY_STRING'].split('&'):
             if qry.startswith('cmd'):
                 cmd = qry.split('=')[-1]
                 if mapping.has_key(cmd):
                     return mapping[cmd]
                 else:
                     return cmd
 
     def __invalidate_cache(self, repo_name):
         """we know that some change was made to repositories and we should
         invalidate the cache to see the changes right away but only for
         push requests"""
         invalidate_cache('get_repo_cached_%s' % repo_name)
 
 
     def __load_web_settings(self, hgserve, extras={}):
         #set the global ui for hgserve instance passed
         hgserve.repo.ui = self.baseui
 
         hgrc = os.path.join(self.repo_path, '.hg', 'hgrc')
 
         #inject some additional parameters that will be available in ui
         #for hooks
         for k, v in extras.items():
             hgserve.repo.ui.setconfig('rhodecode_extras', k, v)
 
         repoui = make_ui('file', hgrc, False)
 
         if repoui:
             #overwrite our ui instance with the section from hgrc file
             for section in ui_sections:
                 for k, v in repoui.configitems(section):
                     hgserve.repo.ui.setconfig(section, k, v)
 
         return hgserve

rhodecode/tests/test_hg_operations.py

@@ -1,225 +1,228 b''
 # -*- coding: utf-8 -*-
 """
     rhodecode.tests.test_hg_operations
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Test suite for making push/pull operations
     
     :created_on: Dec 30, 2010
     :copyright: (c) 2010 by marcink.
     :license: LICENSE_NAME, see LICENSE_FILE for more details.
 """
 
 import os
 import shutil
 import logging
 
 from subprocess import Popen, PIPE
 
 from os.path import join as jn
 
 from rhodecode.tests import TESTS_TMP_PATH, NEW_HG_REPO, HG_REPO
 
 USER = 'test_admin'
 PASS = 'test12'
 HOST = '127.0.0.1:5000'
-
+DEBUG = True
 log = logging.getLogger(__name__)
 
 
 class Command(object):
 
     def __init__(self, cwd):
         self.cwd = cwd
 
     def execute(self, cmd, *args):
         """Runs command on the system with given ``args``.
         """
 
         command = cmd + ' ' + ' '.join(args)
         log.debug('Executing %s' % command)
-        print command
+        if DEBUG:
+            print command
         p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, cwd=self.cwd)
         stdout, stderr = p.communicate()
-        print stdout, stderr
+        if DEBUG:
+            print stdout, stderr
         return stdout, stderr
 
 
-#===============================================================================
+#==============================================================================
 # TESTS
-#===============================================================================
+#==============================================================================
 def test_clone():
     cwd = path = jn(TESTS_TMP_PATH, HG_REPO)
 
     try:
         shutil.rmtree(path, ignore_errors=True)
         os.makedirs(path)
         #print 'made dirs %s' % jn(path)
     except OSError:
         raise
 
 
     clone_url = 'http://%(user)s:%(pass)s@%(host)s/%(cloned_repo)s %(dest)s' % \
                   {'user':USER,
                    'pass':PASS,
                    'host':HOST,
                    'cloned_repo':HG_REPO,
                    'dest':path}
 
     stdout, stderr = Command(cwd).execute('hg clone', clone_url)
 
     assert """adding file changes""" in stdout, 'no messages about cloning'
     assert """abort""" not in stderr , 'got error from clone'
 
 
 
 def test_clone_anonymous_ok():
     cwd = path = jn(TESTS_TMP_PATH, HG_REPO)
 
     try:
         shutil.rmtree(path, ignore_errors=True)
         os.makedirs(path)
         #print 'made dirs %s' % jn(path)
     except OSError:
         raise
 
 
     clone_url = 'http://%(host)s/%(cloned_repo)s %(dest)s' % \
                   {'user':USER,
                    'pass':PASS,
                    'host':HOST,
                    'cloned_repo':HG_REPO,
                    'dest':path}
 
     stdout, stderr = Command(cwd).execute('hg clone', clone_url)
     print stdout, stderr
     assert """adding file changes""" in stdout, 'no messages about cloning'
     assert """abort""" not in stderr , 'got error from clone'
 
 def test_clone_wrong_credentials():
     cwd = path = jn(TESTS_TMP_PATH, HG_REPO)
 
     try:
         shutil.rmtree(path, ignore_errors=True)
         os.makedirs(path)
         #print 'made dirs %s' % jn(path)
     except OSError:
         raise
 
 
     clone_url = 'http://%(user)s:%(pass)s@%(host)s/%(cloned_repo)s %(dest)s' % \
                   {'user':USER + 'error',
                    'pass':PASS,
                    'host':HOST,
                    'cloned_repo':HG_REPO,
                    'dest':path}
 
     stdout, stderr = Command(cwd).execute('hg clone', clone_url)
 
     assert """abort: authorization failed""" in stderr , 'no error from wrong credentials'
 
 
 def test_pull():
     pass
 
 def test_push():
 
     modified_file = jn(TESTS_TMP_PATH, HG_REPO, 'setup.py')
     for i in xrange(5):
         cmd = """echo 'added_line%s' >> %s""" % (i, modified_file)
         Command(cwd).execute(cmd)
 
         cmd = """hg ci -m 'changed file %s' %s """ % (i, modified_file)
         Command(cwd).execute(cmd)
 
     Command(cwd).execute('hg push %s' % jn(TESTS_TMP_PATH, HG_REPO))
 
 def test_push_new_file():
 
     test_clone()
 
     cwd = path = jn(TESTS_TMP_PATH, HG_REPO)
     added_file = jn(path, 'setup.py')
 
     Command(cwd).execute('touch %s' % added_file)
 
     Command(cwd).execute('hg add %s' % added_file)
 
     for i in xrange(15):
         cmd = """echo 'added_line%s' >> %s""" % (i, added_file)
         Command(cwd).execute(cmd)
 
         cmd = """hg ci -m 'commited new %s' %s """ % (i, added_file)
         Command(cwd).execute(cmd)
 
     push_url = 'http://%(user)s:%(pass)s@%(host)s/%(cloned_repo)s' % \
                   {'user':USER,
                    'pass':PASS,
                    'host':HOST,
                    'cloned_repo':HG_REPO,
                    'dest':jn(TESTS_TMP_PATH, HG_REPO)}
 
     Command(cwd).execute('hg push %s' % push_url)
 
 def test_push_wrong_credentials():
 
     clone_url = 'http://%(user)s:%(pass)s@%(host)s/%(cloned_repo)s' % \
                   {'user':USER + 'xxx',
                    'pass':PASS,
                    'host':HOST,
                    'cloned_repo':HG_REPO,
                    'dest':jn(TESTS_TMP_PATH, HG_REPO)}
 
     modified_file = jn(TESTS_TMP_PATH, HG_REPO, 'setup.py')
     for i in xrange(5):
         cmd = """echo 'added_line%s' >> %s""" % (i, modified_file)
         Command(cwd).execute(cmd)
 
         cmd = """hg ci -m 'commited %s' %s """ % (i, modified_file)
         Command(cwd).execute(cmd)
 
     Command(cwd).execute('hg push %s' % clone_url)
 
 def test_push_wrong_path():
     cwd = path = jn(TESTS_TMP_PATH, HG_REPO)
     added_file = jn(path, 'somefile.py')
 
     try:
         shutil.rmtree(path, ignore_errors=True)
         os.makedirs(path)
         print 'made dirs %s' % jn(path)
     except OSError:
         raise
 
     Command(cwd).execute("""echo '' > %s""" % added_file)
     Command(cwd).execute("""hg init %s""" % path)
     Command(cwd).execute("""hg add %s""" % added_file)
 
     for i in xrange(2):
         cmd = """echo 'added_line%s' >> %s""" % (i, added_file)
         Command(cwd).execute(cmd)
 
         cmd = """hg ci -m 'commited new %s' %s """ % (i, added_file)
         Command(cwd).execute(cmd)
 
     clone_url = 'http://%(user)s:%(pass)s@%(host)s/%(cloned_repo)s' % \
                   {'user':USER,
                    'pass':PASS,
                    'host':HOST,
                    'cloned_repo':HG_REPO + '_error',
                    'dest':jn(TESTS_TMP_PATH, HG_REPO)}
 
     stdout, stderr = Command(cwd).execute('hg push %s' % clone_url)
     assert """abort: HTTP Error 403: Forbidden"""  in stderr
 
 
 if __name__ == '__main__':
     test_clone()
-    test_clone_wrong_credentials()
+
+    #test_clone_wrong_credentials()
     ##test_clone_anonymous_ok()
-
+    test_pull()
     #test_push_new_file()
     #test_push_wrong_path()
     #test_push_wrong_credentials()