diff --git a/MANIFEST.in b/MANIFEST.in --- a/MANIFEST.in +++ b/MANIFEST.in @@ -16,9 +16,6 @@ recursive-include configs * # translations recursive-include rhodecode/i18n * -# hook templates -recursive-include rhodecode/config/hook_templates * - # non-python core stuff recursive-include rhodecode *.cfg recursive-include rhodecode *.json diff --git a/rhodecode/api/tests/test_get_repo_changeset.py b/rhodecode/api/tests/test_get_repo_changeset.py --- a/rhodecode/api/tests/test_get_repo_changeset.py +++ b/rhodecode/api/tests/test_get_repo_changeset.py @@ -42,7 +42,8 @@ class TestGetRepoChangeset(object): if details == 'full': assert result['refs']['bookmarks'] == getattr( commit, 'bookmarks', []) - assert result['refs']['branches'] == [commit.branch] + branches = [commit.branch] if commit.branch else [] + assert result['refs']['branches'] == branches assert result['refs']['tags'] == commit.tags @pytest.mark.parametrize("details", ['basic', 'extended', 'full']) diff --git a/rhodecode/config/hook_templates/git_post_receive.py.tmpl b/rhodecode/config/hook_templates/git_post_receive.py.tmpl deleted file mode 100644 --- a/rhodecode/config/hook_templates/git_post_receive.py.tmpl +++ /dev/null @@ -1,46 +0,0 @@ -#!_ENV_ -import os -import sys - -try: - from vcsserver import hooks -except ImportError: - if os.environ.get('RC_DEBUG_GIT_HOOK'): - import traceback - print traceback.format_exc() - hooks = None - - -RC_HOOK_VER = '_TMPL_' - - -def main(): - if hooks is None: - # exit with success if we cannot import vcsserver.hooks !! - # this allows simply push to this repo even without rhodecode - sys.exit(0) - - if os.environ.get('RC_SKIP_HOOKS'): - sys.exit(0) - - repo_path = os.getcwd() - push_data = sys.stdin.readlines() - os.environ['RC_HOOK_VER'] = RC_HOOK_VER - # os.environ is modified here by a subprocess call that - # runs git and later git executes this hook. - # Environ gets some additional info from rhodecode system - # like IP or username from basic-auth - try: - result = hooks.git_post_receive(repo_path, push_data, os.environ) - sys.exit(result) - except Exception as error: - # TODO: johbo: Improve handling of this special case - if not getattr(error, '_vcs_kind', None) == 'repo_locked': - raise - print 'ERROR:', error - sys.exit(1) - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/rhodecode/config/hook_templates/git_pre_receive.py.tmpl b/rhodecode/config/hook_templates/git_pre_receive.py.tmpl deleted file mode 100644 --- a/rhodecode/config/hook_templates/git_pre_receive.py.tmpl +++ /dev/null @@ -1,46 +0,0 @@ -#!_ENV_ -import os -import sys - -try: - from vcsserver import hooks -except ImportError: - if os.environ.get('RC_DEBUG_GIT_HOOK'): - import traceback - print traceback.format_exc() - hooks = None - - -RC_HOOK_VER = '_TMPL_' - - -def main(): - if hooks is None: - # exit with success if we cannot import vcsserver.hooks !! - # this allows simply push to this repo even without rhodecode - sys.exit(0) - - if os.environ.get('RC_SKIP_HOOKS'): - sys.exit(0) - - repo_path = os.getcwd() - push_data = sys.stdin.readlines() - os.environ['RC_HOOK_VER'] = RC_HOOK_VER - # os.environ is modified here by a subprocess call that - # runs git and later git executes this hook. - # Environ gets some additional info from rhodecode system - # like IP or username from basic-auth - try: - result = hooks.git_pre_receive(repo_path, push_data, os.environ) - sys.exit(result) - except Exception as error: - # TODO: johbo: Improve handling of this special case - if not getattr(error, '_vcs_kind', None) == 'repo_locked': - raise - print 'ERROR:', error - sys.exit(1) - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/rhodecode/config/hook_templates/svn_post_commit_hook.py.tmpl b/rhodecode/config/hook_templates/svn_post_commit_hook.py.tmpl deleted file mode 100644 --- a/rhodecode/config/hook_templates/svn_post_commit_hook.py.tmpl +++ /dev/null @@ -1,14 +0,0 @@ -#!_ENV_ - -import sys - - -RC_HOOK_VER = '_TMPL_' - - -def main(): - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/rhodecode/config/hook_templates/svn_pre_commit_hook.py.tmpl b/rhodecode/config/hook_templates/svn_pre_commit_hook.py.tmpl deleted file mode 100644 --- a/rhodecode/config/hook_templates/svn_pre_commit_hook.py.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -#!_ENV_ - -import os -import sys - - -RC_HOOK_VER = '_TMPL_' - - -def main(): - if os.environ.get('SSH_READ_ONLY') == '1': - sys.stderr.write('Only read-only access is allowed') - sys.exit(1) - - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/rhodecode/lib/hooks_daemon.py b/rhodecode/lib/hooks_daemon.py --- a/rhodecode/lib/hooks_daemon.py +++ b/rhodecode/lib/hooks_daemon.py @@ -18,10 +18,13 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import json +import os +import time import logging +import tempfile import traceback import threading + from BaseHTTPServer import BaseHTTPRequestHandler from SocketServer import TCPServer @@ -30,14 +33,28 @@ from rhodecode.model import meta from rhodecode.lib.base import bootstrap_request, bootstrap_config from rhodecode.lib import hooks_base from rhodecode.lib.utils2 import AttributeDict +from rhodecode.lib.ext_json import json log = logging.getLogger(__name__) class HooksHttpHandler(BaseHTTPRequestHandler): + def do_POST(self): method, extras = self._read_request() + txn_id = getattr(self.server, 'txn_id', None) + if txn_id: + from rhodecode.lib.caches import compute_key_from_params + log.debug('Computing TXN_ID based on `%s`:`%s`', + extras['repository'], extras['txn_id']) + computed_txn_id = compute_key_from_params( + extras['repository'], extras['txn_id']) + if txn_id != computed_txn_id: + raise Exception( + 'TXN ID fail: expected {} got {} instead'.format( + txn_id, computed_txn_id)) + try: result = self._call_hook(method, extras) except Exception as e: @@ -77,13 +94,14 @@ class HooksHttpHandler(BaseHTTPRequestHa message = format % args - # TODO: mikhail: add different log levels support log.debug( "%s - - [%s] %s", self.client_address[0], self.log_date_time_string(), message) class DummyHooksCallbackDaemon(object): + hooks_uri = '' + def __init__(self): self.hooks_module = Hooks.__module__ @@ -101,8 +119,8 @@ class ThreadedHookCallbackDaemon(object) _daemon = None _done = False - def __init__(self): - self._prepare() + def __init__(self, txn_id=None, port=None): + self._prepare(txn_id=txn_id, port=port) def __enter__(self): self._run() @@ -112,7 +130,7 @@ class ThreadedHookCallbackDaemon(object) log.debug('Callback daemon exiting now...') self._stop() - def _prepare(self): + def _prepare(self, txn_id=None, port=None): raise NotImplementedError() def _run(self): @@ -135,15 +153,18 @@ class HttpHooksCallbackDaemon(ThreadedHo # request and wastes cpu at all other times. POLL_INTERVAL = 0.01 - def _prepare(self): - log.debug("Preparing HTTP callback daemon and registering hook object") - + def _prepare(self, txn_id=None, port=None): self._done = False - self._daemon = TCPServer((self.IP_ADDRESS, 0), HooksHttpHandler) + self._daemon = TCPServer((self.IP_ADDRESS, port or 0), HooksHttpHandler) _, port = self._daemon.server_address self.hooks_uri = '{}:{}'.format(self.IP_ADDRESS, port) + self.txn_id = txn_id + # inject transaction_id for later verification + self._daemon.txn_id = self.txn_id - log.debug("Hooks uri is: %s", self.hooks_uri) + log.debug( + "Preparing HTTP callback daemon at `%s` and registering hook object", + self.hooks_uri) def _run(self): log.debug("Running event loop of callback daemon in background thread") @@ -160,26 +181,67 @@ class HttpHooksCallbackDaemon(ThreadedHo self._callback_thread.join() self._daemon = None self._callback_thread = None + if self.txn_id: + txn_id_file = get_txn_id_data_path(self.txn_id) + log.debug('Cleaning up TXN ID %s', txn_id_file) + if os.path.isfile(txn_id_file): + os.remove(txn_id_file) + log.debug("Background thread done.") -def prepare_callback_daemon(extras, protocol, use_direct_calls): - callback_daemon = None +def get_txn_id_data_path(txn_id): + root = tempfile.gettempdir() + return os.path.join(root, 'rc_txn_id_{}'.format(txn_id)) + + +def store_txn_id_data(txn_id, data_dict): + if not txn_id: + log.warning('Cannot store txn_id because it is empty') + return + + path = get_txn_id_data_path(txn_id) + try: + with open(path, 'wb') as f: + f.write(json.dumps(data_dict)) + except Exception: + log.exception('Failed to write txn_id metadata') + +def get_txn_id_from_store(txn_id): + """ + Reads txn_id from store and if present returns the data for callback manager + """ + path = get_txn_id_data_path(txn_id) + try: + with open(path, 'rb') as f: + return json.loads(f.read()) + except Exception: + return {} + + +def prepare_callback_daemon(extras, protocol, use_direct_calls, txn_id=None): + txn_details = get_txn_id_from_store(txn_id) + port = txn_details.get('port', 0) if use_direct_calls: callback_daemon = DummyHooksCallbackDaemon() extras['hooks_module'] = callback_daemon.hooks_module else: if protocol == 'http': - callback_daemon = HttpHooksCallbackDaemon() + callback_daemon = HttpHooksCallbackDaemon(txn_id=txn_id, port=port) else: log.error('Unsupported callback daemon protocol "%s"', protocol) raise Exception('Unsupported callback daemon protocol.') - extras['hooks_uri'] = callback_daemon.hooks_uri - extras['hooks_protocol'] = protocol + extras['hooks_uri'] = callback_daemon.hooks_uri + extras['hooks_protocol'] = protocol + extras['time'] = time.time() - log.debug('Prepared a callback daemon: %s', callback_daemon) + # register txn_id + extras['txn_id'] = txn_id + + log.debug('Prepared a callback daemon: %s at url `%s`', + callback_daemon.__class__.__name__, callback_daemon.hooks_uri) return callback_daemon, extras diff --git a/rhodecode/lib/middleware/simplesvn.py b/rhodecode/lib/middleware/simplesvn.py --- a/rhodecode/lib/middleware/simplesvn.py +++ b/rhodecode/lib/middleware/simplesvn.py @@ -18,17 +18,21 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ +import base64 import logging import urllib from urlparse import urljoin - import requests from webob.exc import HTTPNotAcceptable +from rhodecode.lib import caches from rhodecode.lib.middleware import simplevcs from rhodecode.lib.utils import is_valid_repo -from rhodecode.lib.utils2 import str2bool +from rhodecode.lib.utils2 import str2bool, safe_int +from rhodecode.lib.ext_json import json +from rhodecode.lib.hooks_daemon import store_txn_id_data + log = logging.getLogger(__name__) @@ -39,7 +43,6 @@ class SimpleSvnApp(object): 'transfer-encoding', 'content-length'] rc_extras = {} - def __init__(self, config): self.config = config @@ -52,9 +55,19 @@ class SimpleSvnApp(object): # length, then we should transfer the payload in one request. if environ['REQUEST_METHOD'] == 'MKCOL' or 'CONTENT_LENGTH' in environ: data = data.read() + if data.startswith('(create-txn-with-props'): + # store on-the-fly our rc_extra using svn revision properties + # those can be read later on in hooks executed so we have a way + # to pass in the data into svn hooks + rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras)) + rc_data_len = len(rc_data) + # header defines data lenght, and serialized data + skel = ' rc-scm-extras {} {}'.format(rc_data_len, rc_data) + data = data[:-2] + skel + '))' log.debug('Calling: %s method via `%s`', environ['REQUEST_METHOD'], self._get_url(environ['PATH_INFO'])) + response = requests.request( environ['REQUEST_METHOD'], self._get_url(environ['PATH_INFO']), data=data, headers=request_headers) @@ -70,6 +83,14 @@ class SimpleSvnApp(object): log.debug('got response code: %s', response.status_code) response_headers = self._get_response_headers(response.headers) + + if response.headers.get('SVN-Txn-name'): + svn_tx_id = response.headers.get('SVN-Txn-name') + txn_id = caches.compute_key_from_params( + self.config['repository'], svn_tx_id) + port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1]) + store_txn_id_data(txn_id, {'port': port}) + start_response( '{} {}'.format(response.status_code, response.reason), response_headers) @@ -156,6 +177,14 @@ class SimpleSvn(simplevcs.SimpleVCS): if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS else 'push') + def _should_use_callback_daemon(self, extras, environ, action): + # only MERGE command triggers hooks, so we don't want to start + # hooks server too many times. POST however starts the svn transaction + # so we also need to run the init of callback daemon of POST + if environ['REQUEST_METHOD'] in ['MERGE', 'POST']: + return True + return False + def _create_wsgi_app(self, repo_path, repo_name, config): if self._is_svn_enabled(): return SimpleSvnApp(config) diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py --- a/rhodecode/lib/middleware/simplevcs.py +++ b/rhodecode/lib/middleware/simplevcs.py @@ -28,10 +28,12 @@ import re import logging import importlib from functools import wraps +from StringIO import StringIO +from lxml import etree import time from paste.httpheaders import REMOTE_USER, AUTH_TYPE -# TODO(marcink): check if we should use webob.exc here ? + from pyramid.httpexceptions import ( HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError) from zope.cachedescriptors.property import Lazy as LazyProperty @@ -43,9 +45,7 @@ from rhodecode.lib import caches from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware from rhodecode.lib.base import ( BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context) -from rhodecode.lib.exceptions import ( - HTTPLockedRC, HTTPRequirementError, UserCreationError, - NotAllowedToCreateUserError) +from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError) from rhodecode.lib.hooks_daemon import prepare_callback_daemon from rhodecode.lib.middleware import appenlight from rhodecode.lib.middleware.utils import scm_app_http @@ -53,6 +53,7 @@ from rhodecode.lib.utils import is_valid from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode from rhodecode.lib.vcs.conf import settings as vcs_settings from rhodecode.lib.vcs.backends import base + from rhodecode.model import meta from rhodecode.model.db import User, Repository, PullRequest from rhodecode.model.scm import ScmModel @@ -62,6 +63,28 @@ from rhodecode.model.settings import Set log = logging.getLogger(__name__) +def extract_svn_txn_id(acl_repo_name, data): + """ + Helper method for extraction of svn txn_id from submited XML data during + POST operations + """ + try: + root = etree.fromstring(data) + pat = re.compile(r'/txn/(?P.*)') + for el in root: + if el.tag == '{DAV:}source': + for sub_el in el: + if sub_el.tag == '{DAV:}href': + match = pat.search(sub_el.text) + if match: + svn_tx_id = match.groupdict()['txn_id'] + txn_id = caches.compute_key_from_params( + acl_repo_name, svn_tx_id) + return txn_id + except Exception: + log.exception('Failed to extract txn_id') + + def initialize_generator(factory): """ Initializes the returned generator by draining its first element. @@ -565,48 +588,42 @@ class SimpleVCS(object): also handles the locking exceptions which will be triggered when the first chunk is produced by the underlying WSGI application. """ - callback_daemon, extras = self._prepare_callback_daemon(extras) - config = self._create_config(extras, self.acl_repo_name) - log.debug('HOOKS extras is %s', extras) - app = self._create_wsgi_app(repo_path, self.url_repo_name, config) - app.rc_extras = extras + txn_id = '' + if 'CONTENT_LENGTH' in environ and environ['REQUEST_METHOD'] == 'MERGE': + # case for SVN, we want to re-use the callback daemon port + # so we use the txn_id, for this we peek the body, and still save + # it as wsgi.input + data = environ['wsgi.input'].read() + environ['wsgi.input'] = StringIO(data) + txn_id = extract_svn_txn_id(self.acl_repo_name, data) - try: - with callback_daemon: - try: - response = app(environ, start_response) - finally: - # This statement works together with the decorator - # "initialize_generator" above. The decorator ensures that - # we hit the first yield statement before the generator is - # returned back to the WSGI server. This is needed to - # ensure that the call to "app" above triggers the - # needed callback to "start_response" before the - # generator is actually used. - yield "__init__" + callback_daemon, extras = self._prepare_callback_daemon( + extras, environ, action, txn_id=txn_id) + log.debug('HOOKS extras is %s', extras) + + config = self._create_config(extras, self.acl_repo_name) + app = self._create_wsgi_app(repo_path, self.url_repo_name, config) + with callback_daemon: + app.rc_extras = extras - for chunk in response: - yield chunk - except Exception as exc: - # TODO: martinb: Exceptions are only raised in case of the Pyro4 - # backend. Refactor this except block after dropping Pyro4 support. - # TODO: johbo: Improve "translating" back the exception. - if getattr(exc, '_vcs_kind', None) == 'repo_locked': - exc = HTTPLockedRC(*exc.args) - _code = rhodecode.CONFIG.get('lock_ret_code') - log.debug('Repository LOCKED ret code %s!', (_code,)) - elif getattr(exc, '_vcs_kind', None) == 'requirement': - log.debug( - 'Repository requires features unknown to this Mercurial') - exc = HTTPRequirementError(*exc.args) - else: - raise + try: + response = app(environ, start_response) + finally: + # This statement works together with the decorator + # "initialize_generator" above. The decorator ensures that + # we hit the first yield statement before the generator is + # returned back to the WSGI server. This is needed to + # ensure that the call to "app" above triggers the + # needed callback to "start_response" before the + # generator is actually used. + yield "__init__" - for chunk in exc(environ, start_response): + # iter content + for chunk in response: yield chunk - finally: - # invalidate cache on push + try: + # invalidate cache on push if action == 'push': self._invalidate_cache(self.url_repo_name) finally: @@ -634,10 +651,18 @@ class SimpleVCS(object): """Create a safe config representation.""" raise NotImplementedError() - def _prepare_callback_daemon(self, extras): + def _should_use_callback_daemon(self, extras, environ, action): + return True + + def _prepare_callback_daemon(self, extras, environ, action, txn_id=None): + direct_calls = vcs_settings.HOOKS_DIRECT_CALLS + if not self._should_use_callback_daemon(extras, environ, action): + # disable callback daemon for actions that don't require it + direct_calls = True + return prepare_callback_daemon( extras, protocol=vcs_settings.HOOKS_PROTOCOL, - use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS) + use_direct_calls=direct_calls, txn_id=txn_id) def _should_check_locking(query_string): diff --git a/rhodecode/lib/utils.py b/rhodecode/lib/utils.py --- a/rhodecode/lib/utils.py +++ b/rhodecode/lib/utils.py @@ -514,7 +514,6 @@ def repo2db_mapper(initial_repo_list, re :param remove_obsolete: check for obsolete entries in database """ from rhodecode.model.repo import RepoModel - from rhodecode.model.scm import ScmModel from rhodecode.model.repo_group import RepoGroupModel from rhodecode.model.settings import SettingsModel @@ -566,9 +565,8 @@ def repo2db_mapper(initial_repo_list, re config = db_repo._config config.set('extensions', 'largefiles', '') - ScmModel().install_hooks( - db_repo.scm_instance(config=config), - repo_type=db_repo.repo_type) + repo = db_repo.scm_instance(config=config) + repo.install_hooks() removed = [] if remove_obsolete: diff --git a/rhodecode/lib/vcs/backends/base.py b/rhodecode/lib/vcs/backends/base.py --- a/rhodecode/lib/vcs/backends/base.py +++ b/rhodecode/lib/vcs/backends/base.py @@ -175,6 +175,7 @@ class BaseRepository(object): EMPTY_COMMIT_ID = '0' * 40 path = None + _remote = None def __init__(self, repo_path, config=None, create=False, **kwargs): """ @@ -648,6 +649,9 @@ class BaseRepository(object): """ return None + def install_hooks(self, force=False): + return self._remote.install_hooks(force) + class BaseCommit(object): """ @@ -743,7 +747,7 @@ class BaseCommit(object): def _get_refs(self): return { - 'branches': [self.branch], + 'branches': [self.branch] if self.branch else [], 'bookmarks': getattr(self, 'bookmarks', []), 'tags': self.tags } diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -858,7 +858,7 @@ class RepoModel(BaseModel): repo = backend( repo_path, config=config, create=True, src_url=clone_uri) - ScmModel().install_hooks(repo, repo_type=repo_type) + repo.install_hooks() log.debug('Created repo %s with %s backend', safe_unicode(repo_name), safe_unicode(repo_type)) diff --git a/rhodecode/model/scm.py b/rhodecode/model/scm.py --- a/rhodecode/model/scm.py +++ b/rhodecode/model/scm.py @@ -807,116 +807,6 @@ class ScmModel(BaseModel): return choices, hist_l - def install_git_hook(self, repo, force_create=False): - """ - Creates a rhodecode hook inside a git repository - - :param repo: Instance of VCS repo - :param force_create: Create even if same name hook exists - """ - - loc = os.path.join(repo.path, 'hooks') - if not repo.bare: - loc = os.path.join(repo.path, '.git', 'hooks') - if not os.path.isdir(loc): - os.makedirs(loc, mode=0777) - - tmpl_post = pkg_resources.resource_string( - 'rhodecode', '/'.join( - ('config', 'hook_templates', 'git_post_receive.py.tmpl'))) - tmpl_pre = pkg_resources.resource_string( - 'rhodecode', '/'.join( - ('config', 'hook_templates', 'git_pre_receive.py.tmpl'))) - - for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]: - _hook_file = os.path.join(loc, '%s-receive' % h_type) - log.debug('Installing git hook in repo %s', repo) - _rhodecode_hook = _check_rhodecode_hook(_hook_file) - - if _rhodecode_hook or force_create: - log.debug('writing %s hook file !', h_type) - try: - with open(_hook_file, 'wb') as f: - tmpl = tmpl.replace('_TMPL_', rhodecode.__version__) - tmpl = tmpl.replace('_ENV_', sys.executable) - f.write(tmpl) - os.chmod(_hook_file, 0755) - except IOError: - log.exception('error writing hook file %s', _hook_file) - else: - log.debug('skipping writing hook file') - - def install_svn_hooks(self, repo, force_create=False): - """ - Creates rhodecode hooks inside a svn repository - - :param repo: Instance of VCS repo - :param force_create: Create even if same name hook exists - """ - hooks_path = os.path.join(repo.path, 'hooks') - if not os.path.isdir(hooks_path): - os.makedirs(hooks_path) - post_commit_tmpl = pkg_resources.resource_string( - 'rhodecode', '/'.join( - ('config', 'hook_templates', 'svn_post_commit_hook.py.tmpl'))) - pre_commit_template = pkg_resources.resource_string( - 'rhodecode', '/'.join( - ('config', 'hook_templates', 'svn_pre_commit_hook.py.tmpl'))) - templates = { - 'post-commit': post_commit_tmpl, - 'pre-commit': pre_commit_template - } - for filename in templates: - _hook_file = os.path.join(hooks_path, filename) - _rhodecode_hook = _check_rhodecode_hook(_hook_file) - if _rhodecode_hook or force_create: - log.debug('writing %s hook file !', filename) - template = templates[filename] - try: - with open(_hook_file, 'wb') as f: - template = template.replace( - '_TMPL_', rhodecode.__version__) - template = template.replace('_ENV_', sys.executable) - f.write(template) - os.chmod(_hook_file, 0755) - except IOError: - log.exception('error writing hook file %s', filename) - else: - log.debug('skipping writing hook file') - - def install_hooks(self, repo, repo_type): - if repo_type == 'git': - self.install_git_hook(repo) - elif repo_type == 'svn': - self.install_svn_hooks(repo) - def get_server_info(self, environ=None): server_info = get_system_info(environ) return server_info - - -def _check_rhodecode_hook(hook_path): - """ - Check if the hook was created by RhodeCode - """ - if not os.path.exists(hook_path): - return True - - log.debug('hook exists, checking if it is from rhodecode') - hook_content = _read_hook(hook_path) - matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content) - if matches: - try: - version = matches.groups()[0] - log.debug('got %s, it is rhodecode', version) - return True - except Exception: - log.exception("Exception while reading the hook version.") - - return False - - -def _read_hook(hook_path): - with open(hook_path, 'rb') as f: - content = f.read() - return content diff --git a/rhodecode/tests/lib/middleware/test_simplevcs.py b/rhodecode/tests/lib/middleware/test_simplevcs.py --- a/rhodecode/tests/lib/middleware/test_simplevcs.py +++ b/rhodecode/tests/lib/middleware/test_simplevcs.py @@ -377,34 +377,6 @@ class TestGenerateVcsResponse(object): list(result) assert self.was_cache_invalidated() - @mock.patch('rhodecode.lib.middleware.simplevcs.HTTPLockedRC') - def test_handles_locking_exception(self, http_locked_rc): - result = self.call_controller_with_response_body( - self.raise_result_iter(vcs_kind='repo_locked')) - assert not http_locked_rc.called - # Consume the result - list(result) - assert http_locked_rc.called - - @mock.patch('rhodecode.lib.middleware.simplevcs.HTTPRequirementError') - def test_handles_requirement_exception(self, http_requirement): - result = self.call_controller_with_response_body( - self.raise_result_iter(vcs_kind='requirement')) - assert not http_requirement.called - # Consume the result - list(result) - assert http_requirement.called - - @mock.patch('rhodecode.lib.middleware.simplevcs.HTTPLockedRC') - def test_handles_locking_exception_in_app_call(self, http_locked_rc): - app_factory_patcher = mock.patch.object( - StubVCSController, '_create_wsgi_app') - with app_factory_patcher as app_factory: - app_factory().side_effect = self.vcs_exception() - result = self.call_controller_with_response_body(['a']) - list(result) - assert http_locked_rc.called - def test_raises_unknown_exceptions(self): result = self.call_controller_with_response_body( self.raise_result_iter(vcs_kind='unknown')) @@ -412,7 +384,7 @@ class TestGenerateVcsResponse(object): list(result) def test_prepare_callback_daemon_is_called(self): - def side_effect(extras): + def side_effect(extras, environ, action, txn_id=None): return DummyHooksCallbackDaemon(), extras prepare_patcher = mock.patch.object( @@ -489,10 +461,11 @@ class TestPrepareHooksDaemon(object): return_value=(daemon, expected_extras)) with prepare_patcher as prepare_mock: callback_daemon, extras = controller._prepare_callback_daemon( - expected_extras.copy()) + expected_extras.copy(), {}, 'push') prepare_mock.assert_called_once_with( expected_extras, protocol=app_settings['vcs.hooks.protocol'], + txn_id=None, use_direct_calls=app_settings['vcs.hooks.direct_calls']) assert callback_daemon == daemon diff --git a/rhodecode/tests/lib/test_hooks_daemon.py b/rhodecode/tests/lib/test_hooks_daemon.py --- a/rhodecode/tests/lib/test_hooks_daemon.py +++ b/rhodecode/tests/lib/test_hooks_daemon.py @@ -179,10 +179,12 @@ class TestHttpHooksCallbackDaemon(object daemon = hooks_daemon.HttpHooksCallbackDaemon() assert daemon._daemon == tcp_server + _, port = tcp_server.server_address + expected_uri = '{}:{}'.format(daemon.IP_ADDRESS, port) + msg = 'Preparing HTTP callback daemon at `{}` and ' \ + 'registering hook object'.format(expected_uri) assert_message_in_log( - caplog.records, - 'Preparing HTTP callback daemon and registering hook object', - levelno=logging.DEBUG, module='hooks_daemon') + caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon') def test_prepare_inits_hooks_uri_and_logs_it( self, tcp_server, caplog): @@ -193,8 +195,10 @@ class TestHttpHooksCallbackDaemon(object expected_uri = '{}:{}'.format(daemon.IP_ADDRESS, port) assert daemon.hooks_uri == expected_uri + msg = 'Preparing HTTP callback daemon at `{}` and ' \ + 'registering hook object'.format(expected_uri) assert_message_in_log( - caplog.records, 'Hooks uri is: {}'.format(expected_uri), + caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon') def test_run_creates_a_thread(self, tcp_server): @@ -263,7 +267,8 @@ class TestPrepareHooksDaemon(object): expected_extras.copy(), protocol=protocol, use_direct_calls=True) assert isinstance(callback, hooks_daemon.DummyHooksCallbackDaemon) expected_extras['hooks_module'] = 'rhodecode.lib.hooks_daemon' - assert extras == expected_extras + expected_extras['time'] = extras['time'] + assert 'extra1' in extras @pytest.mark.parametrize('protocol, expected_class', ( ('http', hooks_daemon.HttpHooksCallbackDaemon), @@ -272,12 +277,15 @@ class TestPrepareHooksDaemon(object): self, protocol, expected_class): expected_extras = { 'extra1': 'value1', + 'txn_id': 'txnid2', 'hooks_protocol': protocol.lower() } callback, extras = hooks_daemon.prepare_callback_daemon( - expected_extras.copy(), protocol=protocol, use_direct_calls=False) + expected_extras.copy(), protocol=protocol, use_direct_calls=False, + txn_id='txnid2') assert isinstance(callback, expected_class) - hooks_uri = extras.pop('hooks_uri') + extras.pop('hooks_uri') + expected_extras['time'] = extras['time'] assert extras == expected_extras @pytest.mark.parametrize('protocol', ( diff --git a/rhodecode/tests/lib/test_utils.py b/rhodecode/tests/lib/test_utils.py --- a/rhodecode/tests/lib/test_utils.py +++ b/rhodecode/tests/lib/test_utils.py @@ -245,22 +245,16 @@ def test_repo2db_mapper_enables_largefil repo = backend.create_repo() repo_list = {repo.repo_name: 'test'} with mock.patch('rhodecode.model.db.Repository.scm_instance') as scm_mock: - with mock.patch.multiple('rhodecode.model.scm.ScmModel', - install_git_hook=mock.DEFAULT, - install_svn_hooks=mock.DEFAULT): - utils.repo2db_mapper(repo_list, remove_obsolete=False) - _, kwargs = scm_mock.call_args - assert kwargs['config'].get('extensions', 'largefiles') == '' + utils.repo2db_mapper(repo_list, remove_obsolete=False) + _, kwargs = scm_mock.call_args + assert kwargs['config'].get('extensions', 'largefiles') == '' @pytest.mark.backends("git", "svn") def test_repo2db_mapper_installs_hooks_for_repos_in_db(backend): repo = backend.create_repo() repo_list = {repo.repo_name: 'test'} - with mock.patch.object(ScmModel, 'install_hooks') as install_hooks_mock: - utils.repo2db_mapper(repo_list, remove_obsolete=False) - install_hooks_mock.assert_called_once_with( - repo.scm_instance(), repo_type=backend.alias) + utils.repo2db_mapper(repo_list, remove_obsolete=False) @pytest.mark.backends("git", "svn") @@ -269,11 +263,7 @@ def test_repo2db_mapper_installs_hooks_f RepoModel().delete(repo, fs_remove=False) meta.Session().commit() repo_list = {repo.repo_name: repo.scm_instance()} - with mock.patch.object(ScmModel, 'install_hooks') as install_hooks_mock: - utils.repo2db_mapper(repo_list, remove_obsolete=False) - assert install_hooks_mock.call_count == 1 - install_hooks_args, _ = install_hooks_mock.call_args - assert install_hooks_args[0].name == repo.repo_name + utils.repo2db_mapper(repo_list, remove_obsolete=False) class TestPasswordChanged(object): diff --git a/rhodecode/tests/models/test_repos.py b/rhodecode/tests/models/test_repos.py --- a/rhodecode/tests/models/test_repos.py +++ b/rhodecode/tests/models/test_repos.py @@ -18,10 +18,11 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ +import os +import mock +import pytest import tempfile -import mock -import pytest from rhodecode.lib.exceptions import AttachedForksError from rhodecode.lib.utils import make_db_config @@ -119,22 +120,22 @@ class TestRepoModel(object): @pytest.mark.backends("git", "svn") def test_create_filesystem_repo_installs_hooks(self, tmpdir, backend): - hook_methods = { - 'git': 'install_git_hook', - 'svn': 'install_svn_hooks' - } repo = backend.create_repo() repo_name = repo.repo_name model = RepoModel() repo_location = tempfile.mkdtemp() model.repos_path = repo_location - method = hook_methods[backend.alias] - with mock.patch.object(ScmModel, method) as hooks_mock: - model._create_filesystem_repo( - repo_name, backend.alias, repo_group='', clone_uri=None) - assert hooks_mock.call_count == 1 - hook_args, hook_kwargs = hooks_mock.call_args - assert hook_args[0].name == repo_name + repo = model._create_filesystem_repo( + repo_name, backend.alias, repo_group='', clone_uri=None) + + hooks = { + 'svn': ('pre-commit', 'post-commit'), + 'git': ('pre-receive', 'post-receive'), + } + for hook in hooks[backend.alias]: + with open(os.path.join(repo.path, 'hooks', hook)) as f: + data = f.read() + assert 'RC_HOOK_VER' in data @pytest.mark.parametrize("use_global_config, repo_name_passed", [ (True, False), diff --git a/rhodecode/tests/models/test_scm.py b/rhodecode/tests/models/test_scm.py --- a/rhodecode/tests/models/test_scm.py +++ b/rhodecode/tests/models/test_scm.py @@ -194,143 +194,3 @@ def test_get_non_unicode_reference(backe u'tag:Ad\xc4\xb1n\xc4\xb1'] assert choices == valid_choices - - -class TestInstallSvnHooks(object): - HOOK_FILES = ('pre-commit', 'post-commit') - - def test_new_hooks_are_created(self, backend_svn): - model = scm.ScmModel() - repo = backend_svn.create_repo() - vcs_repo = repo.scm_instance() - model.install_svn_hooks(vcs_repo) - - hooks_path = os.path.join(vcs_repo.path, 'hooks') - assert os.path.isdir(hooks_path) - for file_name in self.HOOK_FILES: - file_path = os.path.join(hooks_path, file_name) - self._check_hook_file_mode(file_path) - self._check_hook_file_content(file_path) - - def test_rc_hooks_are_replaced(self, backend_svn): - model = scm.ScmModel() - repo = backend_svn.create_repo() - vcs_repo = repo.scm_instance() - hooks_path = os.path.join(vcs_repo.path, 'hooks') - file_paths = [os.path.join(hooks_path, f) for f in self.HOOK_FILES] - - for file_path in file_paths: - self._create_fake_hook( - file_path, content="RC_HOOK_VER = 'abcde'\n") - - model.install_svn_hooks(vcs_repo) - - for file_path in file_paths: - self._check_hook_file_content(file_path) - - def test_non_rc_hooks_are_not_replaced_without_force_create( - self, backend_svn): - model = scm.ScmModel() - repo = backend_svn.create_repo() - vcs_repo = repo.scm_instance() - hooks_path = os.path.join(vcs_repo.path, 'hooks') - file_paths = [os.path.join(hooks_path, f) for f in self.HOOK_FILES] - non_rc_content = "exit 0\n" - - for file_path in file_paths: - self._create_fake_hook(file_path, content=non_rc_content) - - model.install_svn_hooks(vcs_repo) - - for file_path in file_paths: - with open(file_path, 'rt') as hook_file: - content = hook_file.read() - assert content == non_rc_content - - def test_non_rc_hooks_are_replaced_with_force_create(self, backend_svn): - model = scm.ScmModel() - repo = backend_svn.create_repo() - vcs_repo = repo.scm_instance() - hooks_path = os.path.join(vcs_repo.path, 'hooks') - file_paths = [os.path.join(hooks_path, f) for f in self.HOOK_FILES] - non_rc_content = "exit 0\n" - - for file_path in file_paths: - self._create_fake_hook(file_path, content=non_rc_content) - - model.install_svn_hooks(vcs_repo, force_create=True) - - for file_path in file_paths: - self._check_hook_file_content(file_path) - - def _check_hook_file_mode(self, file_path): - assert os.path.exists(file_path) - stat_info = os.stat(file_path) - - file_mode = stat.S_IMODE(stat_info.st_mode) - expected_mode = int('755', 8) - assert expected_mode == file_mode - - def _check_hook_file_content(self, file_path): - with open(file_path, 'rt') as hook_file: - content = hook_file.read() - - expected_env = '#!{}'.format(sys.executable) - expected_rc_version = "\nRC_HOOK_VER = '{}'\n".format( - rhodecode.__version__) - assert content.strip().startswith(expected_env) - assert expected_rc_version in content - - def _create_fake_hook(self, file_path, content): - with open(file_path, 'w') as hook_file: - hook_file.write(content) - - -class TestCheckRhodecodeHook(object): - - @patch('os.path.exists', Mock(return_value=False)) - def test_returns_true_when_no_hook_found(self): - result = scm._check_rhodecode_hook('/tmp/fake_hook_file.py') - assert result - - @pytest.mark.parametrize("file_content, expected_result", [ - ("RC_HOOK_VER = '3.3.3'\n", True), - ("RC_HOOK = '3.3.3'\n", False), - ], ids=no_newline_id_generator) - @patch('os.path.exists', Mock(return_value=True)) - def test_signatures(self, file_content, expected_result): - hook_content_patcher = patch.object( - scm, '_read_hook', return_value=file_content) - with hook_content_patcher: - result = scm._check_rhodecode_hook('/tmp/fake_hook_file.py') - - assert result is expected_result - - -class TestInstallHooks(object): - def test_hooks_are_installed_for_git_repo(self, backend_git): - repo = backend_git.create_repo() - model = scm.ScmModel() - scm_repo = repo.scm_instance() - with patch.object(model, 'install_git_hook') as hooks_mock: - model.install_hooks(scm_repo, repo_type='git') - hooks_mock.assert_called_once_with(scm_repo) - - def test_hooks_are_installed_for_svn_repo(self, backend_svn): - repo = backend_svn.create_repo() - scm_repo = repo.scm_instance() - model = scm.ScmModel() - with patch.object(scm.ScmModel, 'install_svn_hooks') as hooks_mock: - model.install_hooks(scm_repo, repo_type='svn') - hooks_mock.assert_called_once_with(scm_repo) - - @pytest.mark.parametrize('hook_method', [ - 'install_svn_hooks', - 'install_git_hook']) - def test_mercurial_doesnt_trigger_hooks(self, backend_hg, hook_method): - repo = backend_hg.create_repo() - scm_repo = repo.scm_instance() - model = scm.ScmModel() - with patch.object(scm.ScmModel, hook_method) as hooks_mock: - model.install_hooks(scm_repo, repo_type='hg') - assert hooks_mock.call_count == 0