##// END OF EJS Templates
svn: enable hooks and integration framework execution....
marcink -
r2677:25d65914 default
parent child
Show More
@@ -16,9 +16,6 recursive-include configs *
16 # translations
16 # translations
17 recursive-include rhodecode/i18n *
17 recursive-include rhodecode/i18n *
18
18
19 # hook templates
20 recursive-include rhodecode/config/hook_templates *
21
22 # non-python core stuff
19 # non-python core stuff
23 recursive-include rhodecode *.cfg
20 recursive-include rhodecode *.cfg
24 recursive-include rhodecode *.json
21 recursive-include rhodecode *.json
@@ -42,7 +42,8 class TestGetRepoChangeset(object):
42 if details == 'full':
42 if details == 'full':
43 assert result['refs']['bookmarks'] == getattr(
43 assert result['refs']['bookmarks'] == getattr(
44 commit, 'bookmarks', [])
44 commit, 'bookmarks', [])
45 assert result['refs']['branches'] == [commit.branch]
45 branches = [commit.branch] if commit.branch else []
46 assert result['refs']['branches'] == branches
46 assert result['refs']['tags'] == commit.tags
47 assert result['refs']['tags'] == commit.tags
47
48
48 @pytest.mark.parametrize("details", ['basic', 'extended', 'full'])
49 @pytest.mark.parametrize("details", ['basic', 'extended', 'full'])
@@ -18,10 +18,13
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import json
21 import os
22 import time
22 import logging
23 import logging
24 import tempfile
23 import traceback
25 import traceback
24 import threading
26 import threading
27
25 from BaseHTTPServer import BaseHTTPRequestHandler
28 from BaseHTTPServer import BaseHTTPRequestHandler
26 from SocketServer import TCPServer
29 from SocketServer import TCPServer
27
30
@@ -30,14 +33,28 from rhodecode.model import meta
30 from rhodecode.lib.base import bootstrap_request, bootstrap_config
33 from rhodecode.lib.base import bootstrap_request, bootstrap_config
31 from rhodecode.lib import hooks_base
34 from rhodecode.lib import hooks_base
32 from rhodecode.lib.utils2 import AttributeDict
35 from rhodecode.lib.utils2 import AttributeDict
36 from rhodecode.lib.ext_json import json
33
37
34
38
35 log = logging.getLogger(__name__)
39 log = logging.getLogger(__name__)
36
40
37
41
38 class HooksHttpHandler(BaseHTTPRequestHandler):
42 class HooksHttpHandler(BaseHTTPRequestHandler):
43
39 def do_POST(self):
44 def do_POST(self):
40 method, extras = self._read_request()
45 method, extras = self._read_request()
46 txn_id = getattr(self.server, 'txn_id', None)
47 if txn_id:
48 from rhodecode.lib.caches import compute_key_from_params
49 log.debug('Computing TXN_ID based on `%s`:`%s`',
50 extras['repository'], extras['txn_id'])
51 computed_txn_id = compute_key_from_params(
52 extras['repository'], extras['txn_id'])
53 if txn_id != computed_txn_id:
54 raise Exception(
55 'TXN ID fail: expected {} got {} instead'.format(
56 txn_id, computed_txn_id))
57
41 try:
58 try:
42 result = self._call_hook(method, extras)
59 result = self._call_hook(method, extras)
43 except Exception as e:
60 except Exception as e:
@@ -77,13 +94,14 class HooksHttpHandler(BaseHTTPRequestHa
77
94
78 message = format % args
95 message = format % args
79
96
80 # TODO: mikhail: add different log levels support
81 log.debug(
97 log.debug(
82 "%s - - [%s] %s", self.client_address[0],
98 "%s - - [%s] %s", self.client_address[0],
83 self.log_date_time_string(), message)
99 self.log_date_time_string(), message)
84
100
85
101
86 class DummyHooksCallbackDaemon(object):
102 class DummyHooksCallbackDaemon(object):
103 hooks_uri = ''
104
87 def __init__(self):
105 def __init__(self):
88 self.hooks_module = Hooks.__module__
106 self.hooks_module = Hooks.__module__
89
107
@@ -101,8 +119,8 class ThreadedHookCallbackDaemon(object)
101 _daemon = None
119 _daemon = None
102 _done = False
120 _done = False
103
121
104 def __init__(self):
122 def __init__(self, txn_id=None, port=None):
105 self._prepare()
123 self._prepare(txn_id=txn_id, port=port)
106
124
107 def __enter__(self):
125 def __enter__(self):
108 self._run()
126 self._run()
@@ -112,7 +130,7 class ThreadedHookCallbackDaemon(object)
112 log.debug('Callback daemon exiting now...')
130 log.debug('Callback daemon exiting now...')
113 self._stop()
131 self._stop()
114
132
115 def _prepare(self):
133 def _prepare(self, txn_id=None, port=None):
116 raise NotImplementedError()
134 raise NotImplementedError()
117
135
118 def _run(self):
136 def _run(self):
@@ -135,15 +153,18 class HttpHooksCallbackDaemon(ThreadedHo
135 # request and wastes cpu at all other times.
153 # request and wastes cpu at all other times.
136 POLL_INTERVAL = 0.01
154 POLL_INTERVAL = 0.01
137
155
138 def _prepare(self):
156 def _prepare(self, txn_id=None, port=None):
139 log.debug("Preparing HTTP callback daemon and registering hook object")
140
141 self._done = False
157 self._done = False
142 self._daemon = TCPServer((self.IP_ADDRESS, 0), HooksHttpHandler)
158 self._daemon = TCPServer((self.IP_ADDRESS, port or 0), HooksHttpHandler)
143 _, port = self._daemon.server_address
159 _, port = self._daemon.server_address
144 self.hooks_uri = '{}:{}'.format(self.IP_ADDRESS, port)
160 self.hooks_uri = '{}:{}'.format(self.IP_ADDRESS, port)
161 self.txn_id = txn_id
162 # inject transaction_id for later verification
163 self._daemon.txn_id = self.txn_id
145
164
146 log.debug("Hooks uri is: %s", self.hooks_uri)
165 log.debug(
166 "Preparing HTTP callback daemon at `%s` and registering hook object",
167 self.hooks_uri)
147
168
148 def _run(self):
169 def _run(self):
149 log.debug("Running event loop of callback daemon in background thread")
170 log.debug("Running event loop of callback daemon in background thread")
@@ -160,26 +181,67 class HttpHooksCallbackDaemon(ThreadedHo
160 self._callback_thread.join()
181 self._callback_thread.join()
161 self._daemon = None
182 self._daemon = None
162 self._callback_thread = None
183 self._callback_thread = None
184 if self.txn_id:
185 txn_id_file = get_txn_id_data_path(self.txn_id)
186 log.debug('Cleaning up TXN ID %s', txn_id_file)
187 if os.path.isfile(txn_id_file):
188 os.remove(txn_id_file)
189
163 log.debug("Background thread done.")
190 log.debug("Background thread done.")
164
191
165
192
166 def prepare_callback_daemon(extras, protocol, use_direct_calls):
193 def get_txn_id_data_path(txn_id):
167 callback_daemon = None
194 root = tempfile.gettempdir()
195 return os.path.join(root, 'rc_txn_id_{}'.format(txn_id))
196
197
198 def store_txn_id_data(txn_id, data_dict):
199 if not txn_id:
200 log.warning('Cannot store txn_id because it is empty')
201 return
202
203 path = get_txn_id_data_path(txn_id)
204 try:
205 with open(path, 'wb') as f:
206 f.write(json.dumps(data_dict))
207 except Exception:
208 log.exception('Failed to write txn_id metadata')
168
209
210
211 def get_txn_id_from_store(txn_id):
212 """
213 Reads txn_id from store and if present returns the data for callback manager
214 """
215 path = get_txn_id_data_path(txn_id)
216 try:
217 with open(path, 'rb') as f:
218 return json.loads(f.read())
219 except Exception:
220 return {}
221
222
223 def prepare_callback_daemon(extras, protocol, use_direct_calls, txn_id=None):
224 txn_details = get_txn_id_from_store(txn_id)
225 port = txn_details.get('port', 0)
169 if use_direct_calls:
226 if use_direct_calls:
170 callback_daemon = DummyHooksCallbackDaemon()
227 callback_daemon = DummyHooksCallbackDaemon()
171 extras['hooks_module'] = callback_daemon.hooks_module
228 extras['hooks_module'] = callback_daemon.hooks_module
172 else:
229 else:
173 if protocol == 'http':
230 if protocol == 'http':
174 callback_daemon = HttpHooksCallbackDaemon()
231 callback_daemon = HttpHooksCallbackDaemon(txn_id=txn_id, port=port)
175 else:
232 else:
176 log.error('Unsupported callback daemon protocol "%s"', protocol)
233 log.error('Unsupported callback daemon protocol "%s"', protocol)
177 raise Exception('Unsupported callback daemon protocol.')
234 raise Exception('Unsupported callback daemon protocol.')
178
235
179 extras['hooks_uri'] = callback_daemon.hooks_uri
236 extras['hooks_uri'] = callback_daemon.hooks_uri
180 extras['hooks_protocol'] = protocol
237 extras['hooks_protocol'] = protocol
238 extras['time'] = time.time()
181
239
182 log.debug('Prepared a callback daemon: %s', callback_daemon)
240 # register txn_id
241 extras['txn_id'] = txn_id
242
243 log.debug('Prepared a callback daemon: %s at url `%s`',
244 callback_daemon.__class__.__name__, callback_daemon.hooks_uri)
183 return callback_daemon, extras
245 return callback_daemon, extras
184
246
185
247
@@ -18,17 +18,21
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import base64
21 import logging
22 import logging
22 import urllib
23 import urllib
23 from urlparse import urljoin
24 from urlparse import urljoin
24
25
25
26 import requests
26 import requests
27 from webob.exc import HTTPNotAcceptable
27 from webob.exc import HTTPNotAcceptable
28
28
29 from rhodecode.lib import caches
29 from rhodecode.lib.middleware import simplevcs
30 from rhodecode.lib.middleware import simplevcs
30 from rhodecode.lib.utils import is_valid_repo
31 from rhodecode.lib.utils import is_valid_repo
31 from rhodecode.lib.utils2 import str2bool
32 from rhodecode.lib.utils2 import str2bool, safe_int
33 from rhodecode.lib.ext_json import json
34 from rhodecode.lib.hooks_daemon import store_txn_id_data
35
32
36
33 log = logging.getLogger(__name__)
37 log = logging.getLogger(__name__)
34
38
@@ -39,7 +43,6 class SimpleSvnApp(object):
39 'transfer-encoding', 'content-length']
43 'transfer-encoding', 'content-length']
40 rc_extras = {}
44 rc_extras = {}
41
45
42
43 def __init__(self, config):
46 def __init__(self, config):
44 self.config = config
47 self.config = config
45
48
@@ -52,9 +55,19 class SimpleSvnApp(object):
52 # length, then we should transfer the payload in one request.
55 # length, then we should transfer the payload in one request.
53 if environ['REQUEST_METHOD'] == 'MKCOL' or 'CONTENT_LENGTH' in environ:
56 if environ['REQUEST_METHOD'] == 'MKCOL' or 'CONTENT_LENGTH' in environ:
54 data = data.read()
57 data = data.read()
58 if data.startswith('(create-txn-with-props'):
59 # store on-the-fly our rc_extra using svn revision properties
60 # those can be read later on in hooks executed so we have a way
61 # to pass in the data into svn hooks
62 rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
63 rc_data_len = len(rc_data)
64 # header defines data lenght, and serialized data
65 skel = ' rc-scm-extras {} {}'.format(rc_data_len, rc_data)
66 data = data[:-2] + skel + '))'
55
67
56 log.debug('Calling: %s method via `%s`', environ['REQUEST_METHOD'],
68 log.debug('Calling: %s method via `%s`', environ['REQUEST_METHOD'],
57 self._get_url(environ['PATH_INFO']))
69 self._get_url(environ['PATH_INFO']))
70
58 response = requests.request(
71 response = requests.request(
59 environ['REQUEST_METHOD'], self._get_url(environ['PATH_INFO']),
72 environ['REQUEST_METHOD'], self._get_url(environ['PATH_INFO']),
60 data=data, headers=request_headers)
73 data=data, headers=request_headers)
@@ -70,6 +83,14 class SimpleSvnApp(object):
70 log.debug('got response code: %s', response.status_code)
83 log.debug('got response code: %s', response.status_code)
71
84
72 response_headers = self._get_response_headers(response.headers)
85 response_headers = self._get_response_headers(response.headers)
86
87 if response.headers.get('SVN-Txn-name'):
88 svn_tx_id = response.headers.get('SVN-Txn-name')
89 txn_id = caches.compute_key_from_params(
90 self.config['repository'], svn_tx_id)
91 port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1])
92 store_txn_id_data(txn_id, {'port': port})
93
73 start_response(
94 start_response(
74 '{} {}'.format(response.status_code, response.reason),
95 '{} {}'.format(response.status_code, response.reason),
75 response_headers)
96 response_headers)
@@ -156,6 +177,14 class SimpleSvn(simplevcs.SimpleVCS):
156 if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS
177 if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS
157 else 'push')
178 else 'push')
158
179
180 def _should_use_callback_daemon(self, extras, environ, action):
181 # only MERGE command triggers hooks, so we don't want to start
182 # hooks server too many times. POST however starts the svn transaction
183 # so we also need to run the init of callback daemon of POST
184 if environ['REQUEST_METHOD'] in ['MERGE', 'POST']:
185 return True
186 return False
187
159 def _create_wsgi_app(self, repo_path, repo_name, config):
188 def _create_wsgi_app(self, repo_path, repo_name, config):
160 if self._is_svn_enabled():
189 if self._is_svn_enabled():
161 return SimpleSvnApp(config)
190 return SimpleSvnApp(config)
@@ -28,10 +28,12 import re
28 import logging
28 import logging
29 import importlib
29 import importlib
30 from functools import wraps
30 from functools import wraps
31 from StringIO import StringIO
32 from lxml import etree
31
33
32 import time
34 import time
33 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
35 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
34 # TODO(marcink): check if we should use webob.exc here ?
36
35 from pyramid.httpexceptions import (
37 from pyramid.httpexceptions import (
36 HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
38 HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
37 from zope.cachedescriptors.property import Lazy as LazyProperty
39 from zope.cachedescriptors.property import Lazy as LazyProperty
@@ -43,9 +45,7 from rhodecode.lib import caches
43 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
45 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
44 from rhodecode.lib.base import (
46 from rhodecode.lib.base import (
45 BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
47 BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
46 from rhodecode.lib.exceptions import (
48 from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
47 HTTPLockedRC, HTTPRequirementError, UserCreationError,
48 NotAllowedToCreateUserError)
49 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
49 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
50 from rhodecode.lib.middleware import appenlight
50 from rhodecode.lib.middleware import appenlight
51 from rhodecode.lib.middleware.utils import scm_app_http
51 from rhodecode.lib.middleware.utils import scm_app_http
@@ -53,6 +53,7 from rhodecode.lib.utils import is_valid
53 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
53 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
54 from rhodecode.lib.vcs.conf import settings as vcs_settings
54 from rhodecode.lib.vcs.conf import settings as vcs_settings
55 from rhodecode.lib.vcs.backends import base
55 from rhodecode.lib.vcs.backends import base
56
56 from rhodecode.model import meta
57 from rhodecode.model import meta
57 from rhodecode.model.db import User, Repository, PullRequest
58 from rhodecode.model.db import User, Repository, PullRequest
58 from rhodecode.model.scm import ScmModel
59 from rhodecode.model.scm import ScmModel
@@ -62,6 +63,28 from rhodecode.model.settings import Set
62 log = logging.getLogger(__name__)
63 log = logging.getLogger(__name__)
63
64
64
65
66 def extract_svn_txn_id(acl_repo_name, data):
67 """
68 Helper method for extraction of svn txn_id from submited XML data during
69 POST operations
70 """
71 try:
72 root = etree.fromstring(data)
73 pat = re.compile(r'/txn/(?P<txn_id>.*)')
74 for el in root:
75 if el.tag == '{DAV:}source':
76 for sub_el in el:
77 if sub_el.tag == '{DAV:}href':
78 match = pat.search(sub_el.text)
79 if match:
80 svn_tx_id = match.groupdict()['txn_id']
81 txn_id = caches.compute_key_from_params(
82 acl_repo_name, svn_tx_id)
83 return txn_id
84 except Exception:
85 log.exception('Failed to extract txn_id')
86
87
65 def initialize_generator(factory):
88 def initialize_generator(factory):
66 """
89 """
67 Initializes the returned generator by draining its first element.
90 Initializes the returned generator by draining its first element.
@@ -565,48 +588,42 class SimpleVCS(object):
565 also handles the locking exceptions which will be triggered when
588 also handles the locking exceptions which will be triggered when
566 the first chunk is produced by the underlying WSGI application.
589 the first chunk is produced by the underlying WSGI application.
567 """
590 """
568 callback_daemon, extras = self._prepare_callback_daemon(extras)
591 txn_id = ''
569 config = self._create_config(extras, self.acl_repo_name)
592 if 'CONTENT_LENGTH' in environ and environ['REQUEST_METHOD'] == 'MERGE':
570 log.debug('HOOKS extras is %s', extras)
593 # case for SVN, we want to re-use the callback daemon port
571 app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
594 # so we use the txn_id, for this we peek the body, and still save
572 app.rc_extras = extras
595 # it as wsgi.input
596 data = environ['wsgi.input'].read()
597 environ['wsgi.input'] = StringIO(data)
598 txn_id = extract_svn_txn_id(self.acl_repo_name, data)
573
599
574 try:
600 callback_daemon, extras = self._prepare_callback_daemon(
575 with callback_daemon:
601 extras, environ, action, txn_id=txn_id)
576 try:
602 log.debug('HOOKS extras is %s', extras)
577 response = app(environ, start_response)
603
578 finally:
604 config = self._create_config(extras, self.acl_repo_name)
579 # This statement works together with the decorator
605 app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
580 # "initialize_generator" above. The decorator ensures that
606 with callback_daemon:
581 # we hit the first yield statement before the generator is
607 app.rc_extras = extras
582 # returned back to the WSGI server. This is needed to
583 # ensure that the call to "app" above triggers the
584 # needed callback to "start_response" before the
585 # generator is actually used.
586 yield "__init__"
587
608
588 for chunk in response:
609 try:
589 yield chunk
610 response = app(environ, start_response)
590 except Exception as exc:
611 finally:
591 # TODO: martinb: Exceptions are only raised in case of the Pyro4
612 # This statement works together with the decorator
592 # backend. Refactor this except block after dropping Pyro4 support.
613 # "initialize_generator" above. The decorator ensures that
593 # TODO: johbo: Improve "translating" back the exception.
614 # we hit the first yield statement before the generator is
594 if getattr(exc, '_vcs_kind', None) == 'repo_locked':
615 # returned back to the WSGI server. This is needed to
595 exc = HTTPLockedRC(*exc.args)
616 # ensure that the call to "app" above triggers the
596 _code = rhodecode.CONFIG.get('lock_ret_code')
617 # needed callback to "start_response" before the
597 log.debug('Repository LOCKED ret code %s!', (_code,))
618 # generator is actually used.
598 elif getattr(exc, '_vcs_kind', None) == 'requirement':
619 yield "__init__"
599 log.debug(
600 'Repository requires features unknown to this Mercurial')
601 exc = HTTPRequirementError(*exc.args)
602 else:
603 raise
604
620
605 for chunk in exc(environ, start_response):
621 # iter content
622 for chunk in response:
606 yield chunk
623 yield chunk
607 finally:
624
608 # invalidate cache on push
609 try:
625 try:
626 # invalidate cache on push
610 if action == 'push':
627 if action == 'push':
611 self._invalidate_cache(self.url_repo_name)
628 self._invalidate_cache(self.url_repo_name)
612 finally:
629 finally:
@@ -634,10 +651,18 class SimpleVCS(object):
634 """Create a safe config representation."""
651 """Create a safe config representation."""
635 raise NotImplementedError()
652 raise NotImplementedError()
636
653
637 def _prepare_callback_daemon(self, extras):
654 def _should_use_callback_daemon(self, extras, environ, action):
655 return True
656
657 def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
658 direct_calls = vcs_settings.HOOKS_DIRECT_CALLS
659 if not self._should_use_callback_daemon(extras, environ, action):
660 # disable callback daemon for actions that don't require it
661 direct_calls = True
662
638 return prepare_callback_daemon(
663 return prepare_callback_daemon(
639 extras, protocol=vcs_settings.HOOKS_PROTOCOL,
664 extras, protocol=vcs_settings.HOOKS_PROTOCOL,
640 use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
665 use_direct_calls=direct_calls, txn_id=txn_id)
641
666
642
667
643 def _should_check_locking(query_string):
668 def _should_check_locking(query_string):
@@ -514,7 +514,6 def repo2db_mapper(initial_repo_list, re
514 :param remove_obsolete: check for obsolete entries in database
514 :param remove_obsolete: check for obsolete entries in database
515 """
515 """
516 from rhodecode.model.repo import RepoModel
516 from rhodecode.model.repo import RepoModel
517 from rhodecode.model.scm import ScmModel
518 from rhodecode.model.repo_group import RepoGroupModel
517 from rhodecode.model.repo_group import RepoGroupModel
519 from rhodecode.model.settings import SettingsModel
518 from rhodecode.model.settings import SettingsModel
520
519
@@ -566,9 +565,8 def repo2db_mapper(initial_repo_list, re
566
565
567 config = db_repo._config
566 config = db_repo._config
568 config.set('extensions', 'largefiles', '')
567 config.set('extensions', 'largefiles', '')
569 ScmModel().install_hooks(
568 repo = db_repo.scm_instance(config=config)
570 db_repo.scm_instance(config=config),
569 repo.install_hooks()
571 repo_type=db_repo.repo_type)
572
570
573 removed = []
571 removed = []
574 if remove_obsolete:
572 if remove_obsolete:
@@ -175,6 +175,7 class BaseRepository(object):
175 EMPTY_COMMIT_ID = '0' * 40
175 EMPTY_COMMIT_ID = '0' * 40
176
176
177 path = None
177 path = None
178 _remote = None
178
179
179 def __init__(self, repo_path, config=None, create=False, **kwargs):
180 def __init__(self, repo_path, config=None, create=False, **kwargs):
180 """
181 """
@@ -648,6 +649,9 class BaseRepository(object):
648 """
649 """
649 return None
650 return None
650
651
652 def install_hooks(self, force=False):
653 return self._remote.install_hooks(force)
654
651
655
652 class BaseCommit(object):
656 class BaseCommit(object):
653 """
657 """
@@ -743,7 +747,7 class BaseCommit(object):
743
747
744 def _get_refs(self):
748 def _get_refs(self):
745 return {
749 return {
746 'branches': [self.branch],
750 'branches': [self.branch] if self.branch else [],
747 'bookmarks': getattr(self, 'bookmarks', []),
751 'bookmarks': getattr(self, 'bookmarks', []),
748 'tags': self.tags
752 'tags': self.tags
749 }
753 }
@@ -858,7 +858,7 class RepoModel(BaseModel):
858 repo = backend(
858 repo = backend(
859 repo_path, config=config, create=True, src_url=clone_uri)
859 repo_path, config=config, create=True, src_url=clone_uri)
860
860
861 ScmModel().install_hooks(repo, repo_type=repo_type)
861 repo.install_hooks()
862
862
863 log.debug('Created repo %s with %s backend',
863 log.debug('Created repo %s with %s backend',
864 safe_unicode(repo_name), safe_unicode(repo_type))
864 safe_unicode(repo_name), safe_unicode(repo_type))
@@ -807,116 +807,6 class ScmModel(BaseModel):
807
807
808 return choices, hist_l
808 return choices, hist_l
809
809
810 def install_git_hook(self, repo, force_create=False):
811 """
812 Creates a rhodecode hook inside a git repository
813
814 :param repo: Instance of VCS repo
815 :param force_create: Create even if same name hook exists
816 """
817
818 loc = os.path.join(repo.path, 'hooks')
819 if not repo.bare:
820 loc = os.path.join(repo.path, '.git', 'hooks')
821 if not os.path.isdir(loc):
822 os.makedirs(loc, mode=0777)
823
824 tmpl_post = pkg_resources.resource_string(
825 'rhodecode', '/'.join(
826 ('config', 'hook_templates', 'git_post_receive.py.tmpl')))
827 tmpl_pre = pkg_resources.resource_string(
828 'rhodecode', '/'.join(
829 ('config', 'hook_templates', 'git_pre_receive.py.tmpl')))
830
831 for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:
832 _hook_file = os.path.join(loc, '%s-receive' % h_type)
833 log.debug('Installing git hook in repo %s', repo)
834 _rhodecode_hook = _check_rhodecode_hook(_hook_file)
835
836 if _rhodecode_hook or force_create:
837 log.debug('writing %s hook file !', h_type)
838 try:
839 with open(_hook_file, 'wb') as f:
840 tmpl = tmpl.replace('_TMPL_', rhodecode.__version__)
841 tmpl = tmpl.replace('_ENV_', sys.executable)
842 f.write(tmpl)
843 os.chmod(_hook_file, 0755)
844 except IOError:
845 log.exception('error writing hook file %s', _hook_file)
846 else:
847 log.debug('skipping writing hook file')
848
849 def install_svn_hooks(self, repo, force_create=False):
850 """
851 Creates rhodecode hooks inside a svn repository
852
853 :param repo: Instance of VCS repo
854 :param force_create: Create even if same name hook exists
855 """
856 hooks_path = os.path.join(repo.path, 'hooks')
857 if not os.path.isdir(hooks_path):
858 os.makedirs(hooks_path)
859 post_commit_tmpl = pkg_resources.resource_string(
860 'rhodecode', '/'.join(
861 ('config', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
862 pre_commit_template = pkg_resources.resource_string(
863 'rhodecode', '/'.join(
864 ('config', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
865 templates = {
866 'post-commit': post_commit_tmpl,
867 'pre-commit': pre_commit_template
868 }
869 for filename in templates:
870 _hook_file = os.path.join(hooks_path, filename)
871 _rhodecode_hook = _check_rhodecode_hook(_hook_file)
872 if _rhodecode_hook or force_create:
873 log.debug('writing %s hook file !', filename)
874 template = templates[filename]
875 try:
876 with open(_hook_file, 'wb') as f:
877 template = template.replace(
878 '_TMPL_', rhodecode.__version__)
879 template = template.replace('_ENV_', sys.executable)
880 f.write(template)
881 os.chmod(_hook_file, 0755)
882 except IOError:
883 log.exception('error writing hook file %s', filename)
884 else:
885 log.debug('skipping writing hook file')
886
887 def install_hooks(self, repo, repo_type):
888 if repo_type == 'git':
889 self.install_git_hook(repo)
890 elif repo_type == 'svn':
891 self.install_svn_hooks(repo)
892
893 def get_server_info(self, environ=None):
810 def get_server_info(self, environ=None):
894 server_info = get_system_info(environ)
811 server_info = get_system_info(environ)
895 return server_info
812 return server_info
896
897
898 def _check_rhodecode_hook(hook_path):
899 """
900 Check if the hook was created by RhodeCode
901 """
902 if not os.path.exists(hook_path):
903 return True
904
905 log.debug('hook exists, checking if it is from rhodecode')
906 hook_content = _read_hook(hook_path)
907 matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content)
908 if matches:
909 try:
910 version = matches.groups()[0]
911 log.debug('got %s, it is rhodecode', version)
912 return True
913 except Exception:
914 log.exception("Exception while reading the hook version.")
915
916 return False
917
918
919 def _read_hook(hook_path):
920 with open(hook_path, 'rb') as f:
921 content = f.read()
922 return content
@@ -377,34 +377,6 class TestGenerateVcsResponse(object):
377 list(result)
377 list(result)
378 assert self.was_cache_invalidated()
378 assert self.was_cache_invalidated()
379
379
380 @mock.patch('rhodecode.lib.middleware.simplevcs.HTTPLockedRC')
381 def test_handles_locking_exception(self, http_locked_rc):
382 result = self.call_controller_with_response_body(
383 self.raise_result_iter(vcs_kind='repo_locked'))
384 assert not http_locked_rc.called
385 # Consume the result
386 list(result)
387 assert http_locked_rc.called
388
389 @mock.patch('rhodecode.lib.middleware.simplevcs.HTTPRequirementError')
390 def test_handles_requirement_exception(self, http_requirement):
391 result = self.call_controller_with_response_body(
392 self.raise_result_iter(vcs_kind='requirement'))
393 assert not http_requirement.called
394 # Consume the result
395 list(result)
396 assert http_requirement.called
397
398 @mock.patch('rhodecode.lib.middleware.simplevcs.HTTPLockedRC')
399 def test_handles_locking_exception_in_app_call(self, http_locked_rc):
400 app_factory_patcher = mock.patch.object(
401 StubVCSController, '_create_wsgi_app')
402 with app_factory_patcher as app_factory:
403 app_factory().side_effect = self.vcs_exception()
404 result = self.call_controller_with_response_body(['a'])
405 list(result)
406 assert http_locked_rc.called
407
408 def test_raises_unknown_exceptions(self):
380 def test_raises_unknown_exceptions(self):
409 result = self.call_controller_with_response_body(
381 result = self.call_controller_with_response_body(
410 self.raise_result_iter(vcs_kind='unknown'))
382 self.raise_result_iter(vcs_kind='unknown'))
@@ -412,7 +384,7 class TestGenerateVcsResponse(object):
412 list(result)
384 list(result)
413
385
414 def test_prepare_callback_daemon_is_called(self):
386 def test_prepare_callback_daemon_is_called(self):
415 def side_effect(extras):
387 def side_effect(extras, environ, action, txn_id=None):
416 return DummyHooksCallbackDaemon(), extras
388 return DummyHooksCallbackDaemon(), extras
417
389
418 prepare_patcher = mock.patch.object(
390 prepare_patcher = mock.patch.object(
@@ -489,10 +461,11 class TestPrepareHooksDaemon(object):
489 return_value=(daemon, expected_extras))
461 return_value=(daemon, expected_extras))
490 with prepare_patcher as prepare_mock:
462 with prepare_patcher as prepare_mock:
491 callback_daemon, extras = controller._prepare_callback_daemon(
463 callback_daemon, extras = controller._prepare_callback_daemon(
492 expected_extras.copy())
464 expected_extras.copy(), {}, 'push')
493 prepare_mock.assert_called_once_with(
465 prepare_mock.assert_called_once_with(
494 expected_extras,
466 expected_extras,
495 protocol=app_settings['vcs.hooks.protocol'],
467 protocol=app_settings['vcs.hooks.protocol'],
468 txn_id=None,
496 use_direct_calls=app_settings['vcs.hooks.direct_calls'])
469 use_direct_calls=app_settings['vcs.hooks.direct_calls'])
497
470
498 assert callback_daemon == daemon
471 assert callback_daemon == daemon
@@ -179,10 +179,12 class TestHttpHooksCallbackDaemon(object
179 daemon = hooks_daemon.HttpHooksCallbackDaemon()
179 daemon = hooks_daemon.HttpHooksCallbackDaemon()
180 assert daemon._daemon == tcp_server
180 assert daemon._daemon == tcp_server
181
181
182 _, port = tcp_server.server_address
183 expected_uri = '{}:{}'.format(daemon.IP_ADDRESS, port)
184 msg = 'Preparing HTTP callback daemon at `{}` and ' \
185 'registering hook object'.format(expected_uri)
182 assert_message_in_log(
186 assert_message_in_log(
183 caplog.records,
187 caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon')
184 'Preparing HTTP callback daemon and registering hook object',
185 levelno=logging.DEBUG, module='hooks_daemon')
186
188
187 def test_prepare_inits_hooks_uri_and_logs_it(
189 def test_prepare_inits_hooks_uri_and_logs_it(
188 self, tcp_server, caplog):
190 self, tcp_server, caplog):
@@ -193,8 +195,10 class TestHttpHooksCallbackDaemon(object
193 expected_uri = '{}:{}'.format(daemon.IP_ADDRESS, port)
195 expected_uri = '{}:{}'.format(daemon.IP_ADDRESS, port)
194 assert daemon.hooks_uri == expected_uri
196 assert daemon.hooks_uri == expected_uri
195
197
198 msg = 'Preparing HTTP callback daemon at `{}` and ' \
199 'registering hook object'.format(expected_uri)
196 assert_message_in_log(
200 assert_message_in_log(
197 caplog.records, 'Hooks uri is: {}'.format(expected_uri),
201 caplog.records, msg,
198 levelno=logging.DEBUG, module='hooks_daemon')
202 levelno=logging.DEBUG, module='hooks_daemon')
199
203
200 def test_run_creates_a_thread(self, tcp_server):
204 def test_run_creates_a_thread(self, tcp_server):
@@ -263,7 +267,8 class TestPrepareHooksDaemon(object):
263 expected_extras.copy(), protocol=protocol, use_direct_calls=True)
267 expected_extras.copy(), protocol=protocol, use_direct_calls=True)
264 assert isinstance(callback, hooks_daemon.DummyHooksCallbackDaemon)
268 assert isinstance(callback, hooks_daemon.DummyHooksCallbackDaemon)
265 expected_extras['hooks_module'] = 'rhodecode.lib.hooks_daemon'
269 expected_extras['hooks_module'] = 'rhodecode.lib.hooks_daemon'
266 assert extras == expected_extras
270 expected_extras['time'] = extras['time']
271 assert 'extra1' in extras
267
272
268 @pytest.mark.parametrize('protocol, expected_class', (
273 @pytest.mark.parametrize('protocol, expected_class', (
269 ('http', hooks_daemon.HttpHooksCallbackDaemon),
274 ('http', hooks_daemon.HttpHooksCallbackDaemon),
@@ -272,12 +277,15 class TestPrepareHooksDaemon(object):
272 self, protocol, expected_class):
277 self, protocol, expected_class):
273 expected_extras = {
278 expected_extras = {
274 'extra1': 'value1',
279 'extra1': 'value1',
280 'txn_id': 'txnid2',
275 'hooks_protocol': protocol.lower()