##// END OF EJS Templates
svn: enable hooks and integration framework execution....
marcink -
r2677:25d65914 default
parent child Browse files
Show More
@@ -16,9 +16,6 b' recursive-include configs *'
16 # translations
16 # translations
17 recursive-include rhodecode/i18n *
17 recursive-include rhodecode/i18n *
18
18
19 # hook templates
20 recursive-include rhodecode/config/hook_templates *
21
22 # non-python core stuff
19 # non-python core stuff
23 recursive-include rhodecode *.cfg
20 recursive-include rhodecode *.cfg
24 recursive-include rhodecode *.json
21 recursive-include rhodecode *.json
@@ -42,7 +42,8 b' class TestGetRepoChangeset(object):'
42 if details == 'full':
42 if details == 'full':
43 assert result['refs']['bookmarks'] == getattr(
43 assert result['refs']['bookmarks'] == getattr(
44 commit, 'bookmarks', [])
44 commit, 'bookmarks', [])
45 assert result['refs']['branches'] == [commit.branch]
45 branches = [commit.branch] if commit.branch else []
46 assert result['refs']['branches'] == branches
46 assert result['refs']['tags'] == commit.tags
47 assert result['refs']['tags'] == commit.tags
47
48
48 @pytest.mark.parametrize("details", ['basic', 'extended', 'full'])
49 @pytest.mark.parametrize("details", ['basic', 'extended', 'full'])
@@ -18,10 +18,13 b''
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import json
21 import os
22 import time
22 import logging
23 import logging
24 import tempfile
23 import traceback
25 import traceback
24 import threading
26 import threading
27
25 from BaseHTTPServer import BaseHTTPRequestHandler
28 from BaseHTTPServer import BaseHTTPRequestHandler
26 from SocketServer import TCPServer
29 from SocketServer import TCPServer
27
30
@@ -30,14 +33,28 b' from rhodecode.model import meta'
30 from rhodecode.lib.base import bootstrap_request, bootstrap_config
33 from rhodecode.lib.base import bootstrap_request, bootstrap_config
31 from rhodecode.lib import hooks_base
34 from rhodecode.lib import hooks_base
32 from rhodecode.lib.utils2 import AttributeDict
35 from rhodecode.lib.utils2 import AttributeDict
36 from rhodecode.lib.ext_json import json
33
37
34
38
35 log = logging.getLogger(__name__)
39 log = logging.getLogger(__name__)
36
40
37
41
38 class HooksHttpHandler(BaseHTTPRequestHandler):
42 class HooksHttpHandler(BaseHTTPRequestHandler):
43
39 def do_POST(self):
44 def do_POST(self):
40 method, extras = self._read_request()
45 method, extras = self._read_request()
46 txn_id = getattr(self.server, 'txn_id', None)
47 if txn_id:
48 from rhodecode.lib.caches import compute_key_from_params
49 log.debug('Computing TXN_ID based on `%s`:`%s`',
50 extras['repository'], extras['txn_id'])
51 computed_txn_id = compute_key_from_params(
52 extras['repository'], extras['txn_id'])
53 if txn_id != computed_txn_id:
54 raise Exception(
55 'TXN ID fail: expected {} got {} instead'.format(
56 txn_id, computed_txn_id))
57
41 try:
58 try:
42 result = self._call_hook(method, extras)
59 result = self._call_hook(method, extras)
43 except Exception as e:
60 except Exception as e:
@@ -77,13 +94,14 b' class HooksHttpHandler(BaseHTTPRequestHa'
77
94
78 message = format % args
95 message = format % args
79
96
80 # TODO: mikhail: add different log levels support
81 log.debug(
97 log.debug(
82 "%s - - [%s] %s", self.client_address[0],
98 "%s - - [%s] %s", self.client_address[0],
83 self.log_date_time_string(), message)
99 self.log_date_time_string(), message)
84
100
85
101
86 class DummyHooksCallbackDaemon(object):
102 class DummyHooksCallbackDaemon(object):
103 hooks_uri = ''
104
87 def __init__(self):
105 def __init__(self):
88 self.hooks_module = Hooks.__module__
106 self.hooks_module = Hooks.__module__
89
107
@@ -101,8 +119,8 b' class ThreadedHookCallbackDaemon(object)'
101 _daemon = None
119 _daemon = None
102 _done = False
120 _done = False
103
121
104 def __init__(self):
122 def __init__(self, txn_id=None, port=None):
105 self._prepare()
123 self._prepare(txn_id=txn_id, port=port)
106
124
107 def __enter__(self):
125 def __enter__(self):
108 self._run()
126 self._run()
@@ -112,7 +130,7 b' class ThreadedHookCallbackDaemon(object)'
112 log.debug('Callback daemon exiting now...')
130 log.debug('Callback daemon exiting now...')
113 self._stop()
131 self._stop()
114
132
115 def _prepare(self):
133 def _prepare(self, txn_id=None, port=None):
116 raise NotImplementedError()
134 raise NotImplementedError()
117
135
118 def _run(self):
136 def _run(self):
@@ -135,15 +153,18 b' class HttpHooksCallbackDaemon(ThreadedHo'
135 # request and wastes cpu at all other times.
153 # request and wastes cpu at all other times.
136 POLL_INTERVAL = 0.01
154 POLL_INTERVAL = 0.01
137
155
138 def _prepare(self):
156 def _prepare(self, txn_id=None, port=None):
139 log.debug("Preparing HTTP callback daemon and registering hook object")
140
141 self._done = False
157 self._done = False
142 self._daemon = TCPServer((self.IP_ADDRESS, 0), HooksHttpHandler)
158 self._daemon = TCPServer((self.IP_ADDRESS, port or 0), HooksHttpHandler)
143 _, port = self._daemon.server_address
159 _, port = self._daemon.server_address
144 self.hooks_uri = '{}:{}'.format(self.IP_ADDRESS, port)
160 self.hooks_uri = '{}:{}'.format(self.IP_ADDRESS, port)
161 self.txn_id = txn_id
162 # inject transaction_id for later verification
163 self._daemon.txn_id = self.txn_id
145
164
146 log.debug("Hooks uri is: %s", self.hooks_uri)
165 log.debug(
166 "Preparing HTTP callback daemon at `%s` and registering hook object",
167 self.hooks_uri)
147
168
148 def _run(self):
169 def _run(self):
149 log.debug("Running event loop of callback daemon in background thread")
170 log.debug("Running event loop of callback daemon in background thread")
@@ -160,26 +181,67 b' class HttpHooksCallbackDaemon(ThreadedHo'
160 self._callback_thread.join()
181 self._callback_thread.join()
161 self._daemon = None
182 self._daemon = None
162 self._callback_thread = None
183 self._callback_thread = None
184 if self.txn_id:
185 txn_id_file = get_txn_id_data_path(self.txn_id)
186 log.debug('Cleaning up TXN ID %s', txn_id_file)
187 if os.path.isfile(txn_id_file):
188 os.remove(txn_id_file)
189
163 log.debug("Background thread done.")
190 log.debug("Background thread done.")
164
191
165
192
166 def prepare_callback_daemon(extras, protocol, use_direct_calls):
193 def get_txn_id_data_path(txn_id):
167 callback_daemon = None
194 root = tempfile.gettempdir()
195 return os.path.join(root, 'rc_txn_id_{}'.format(txn_id))
196
197
198 def store_txn_id_data(txn_id, data_dict):
199 if not txn_id:
200 log.warning('Cannot store txn_id because it is empty')
201 return
202
203 path = get_txn_id_data_path(txn_id)
204 try:
205 with open(path, 'wb') as f:
206 f.write(json.dumps(data_dict))
207 except Exception:
208 log.exception('Failed to write txn_id metadata')
168
209
210
211 def get_txn_id_from_store(txn_id):
212 """
213 Reads txn_id from store and if present returns the data for callback manager
214 """
215 path = get_txn_id_data_path(txn_id)
216 try:
217 with open(path, 'rb') as f:
218 return json.loads(f.read())
219 except Exception:
220 return {}
221
222
223 def prepare_callback_daemon(extras, protocol, use_direct_calls, txn_id=None):
224 txn_details = get_txn_id_from_store(txn_id)
225 port = txn_details.get('port', 0)
169 if use_direct_calls:
226 if use_direct_calls:
170 callback_daemon = DummyHooksCallbackDaemon()
227 callback_daemon = DummyHooksCallbackDaemon()
171 extras['hooks_module'] = callback_daemon.hooks_module
228 extras['hooks_module'] = callback_daemon.hooks_module
172 else:
229 else:
173 if protocol == 'http':
230 if protocol == 'http':
174 callback_daemon = HttpHooksCallbackDaemon()
231 callback_daemon = HttpHooksCallbackDaemon(txn_id=txn_id, port=port)
175 else:
232 else:
176 log.error('Unsupported callback daemon protocol "%s"', protocol)
233 log.error('Unsupported callback daemon protocol "%s"', protocol)
177 raise Exception('Unsupported callback daemon protocol.')
234 raise Exception('Unsupported callback daemon protocol.')
178
235
179 extras['hooks_uri'] = callback_daemon.hooks_uri
236 extras['hooks_uri'] = callback_daemon.hooks_uri
180 extras['hooks_protocol'] = protocol
237 extras['hooks_protocol'] = protocol
238 extras['time'] = time.time()
181
239
182 log.debug('Prepared a callback daemon: %s', callback_daemon)
240 # register txn_id
241 extras['txn_id'] = txn_id
242
243 log.debug('Prepared a callback daemon: %s at url `%s`',
244 callback_daemon.__class__.__name__, callback_daemon.hooks_uri)
183 return callback_daemon, extras
245 return callback_daemon, extras
184
246
185
247
@@ -18,17 +18,21 b''
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import base64
21 import logging
22 import logging
22 import urllib
23 import urllib
23 from urlparse import urljoin
24 from urlparse import urljoin
24
25
25
26 import requests
26 import requests
27 from webob.exc import HTTPNotAcceptable
27 from webob.exc import HTTPNotAcceptable
28
28
29 from rhodecode.lib import caches
29 from rhodecode.lib.middleware import simplevcs
30 from rhodecode.lib.middleware import simplevcs
30 from rhodecode.lib.utils import is_valid_repo
31 from rhodecode.lib.utils import is_valid_repo
31 from rhodecode.lib.utils2 import str2bool
32 from rhodecode.lib.utils2 import str2bool, safe_int
33 from rhodecode.lib.ext_json import json
34 from rhodecode.lib.hooks_daemon import store_txn_id_data
35
32
36
33 log = logging.getLogger(__name__)
37 log = logging.getLogger(__name__)
34
38
@@ -39,7 +43,6 b' class SimpleSvnApp(object):'
39 'transfer-encoding', 'content-length']
43 'transfer-encoding', 'content-length']
40 rc_extras = {}
44 rc_extras = {}
41
45
42
43 def __init__(self, config):
46 def __init__(self, config):
44 self.config = config
47 self.config = config
45
48
@@ -52,9 +55,19 b' class SimpleSvnApp(object):'
52 # length, then we should transfer the payload in one request.
55 # length, then we should transfer the payload in one request.
53 if environ['REQUEST_METHOD'] == 'MKCOL' or 'CONTENT_LENGTH' in environ:
56 if environ['REQUEST_METHOD'] == 'MKCOL' or 'CONTENT_LENGTH' in environ:
54 data = data.read()
57 data = data.read()
58 if data.startswith('(create-txn-with-props'):
59 # store on-the-fly our rc_extra using svn revision properties
60 # those can be read later on in hooks executed so we have a way
61 # to pass in the data into svn hooks
62 rc_data = base64.urlsafe_b64encode(json.dumps(self.rc_extras))
63 rc_data_len = len(rc_data)
64 # header defines data lenght, and serialized data
65 skel = ' rc-scm-extras {} {}'.format(rc_data_len, rc_data)
66 data = data[:-2] + skel + '))'
55
67
56 log.debug('Calling: %s method via `%s`', environ['REQUEST_METHOD'],
68 log.debug('Calling: %s method via `%s`', environ['REQUEST_METHOD'],
57 self._get_url(environ['PATH_INFO']))
69 self._get_url(environ['PATH_INFO']))
70
58 response = requests.request(
71 response = requests.request(
59 environ['REQUEST_METHOD'], self._get_url(environ['PATH_INFO']),
72 environ['REQUEST_METHOD'], self._get_url(environ['PATH_INFO']),
60 data=data, headers=request_headers)
73 data=data, headers=request_headers)
@@ -70,6 +83,14 b' class SimpleSvnApp(object):'
70 log.debug('got response code: %s', response.status_code)
83 log.debug('got response code: %s', response.status_code)
71
84
72 response_headers = self._get_response_headers(response.headers)
85 response_headers = self._get_response_headers(response.headers)
86
87 if response.headers.get('SVN-Txn-name'):
88 svn_tx_id = response.headers.get('SVN-Txn-name')
89 txn_id = caches.compute_key_from_params(
90 self.config['repository'], svn_tx_id)
91 port = safe_int(self.rc_extras['hooks_uri'].split(':')[-1])
92 store_txn_id_data(txn_id, {'port': port})
93
73 start_response(
94 start_response(
74 '{} {}'.format(response.status_code, response.reason),
95 '{} {}'.format(response.status_code, response.reason),
75 response_headers)
96 response_headers)
@@ -156,6 +177,14 b' class SimpleSvn(simplevcs.SimpleVCS):'
156 if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS
177 if environ['REQUEST_METHOD'] in self.READ_ONLY_COMMANDS
157 else 'push')
178 else 'push')
158
179
180 def _should_use_callback_daemon(self, extras, environ, action):
181 # only MERGE command triggers hooks, so we don't want to start
182 # hooks server too many times. POST however starts the svn transaction
183 # so we also need to run the init of callback daemon of POST
184 if environ['REQUEST_METHOD'] in ['MERGE', 'POST']:
185 return True
186 return False
187
159 def _create_wsgi_app(self, repo_path, repo_name, config):
188 def _create_wsgi_app(self, repo_path, repo_name, config):
160 if self._is_svn_enabled():
189 if self._is_svn_enabled():
161 return SimpleSvnApp(config)
190 return SimpleSvnApp(config)
@@ -28,10 +28,12 b' import re'
28 import logging
28 import logging
29 import importlib
29 import importlib
30 from functools import wraps
30 from functools import wraps
31 from StringIO import StringIO
32 from lxml import etree
31
33
32 import time
34 import time
33 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
35 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
34 # TODO(marcink): check if we should use webob.exc here ?
36
35 from pyramid.httpexceptions import (
37 from pyramid.httpexceptions import (
36 HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
38 HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
37 from zope.cachedescriptors.property import Lazy as LazyProperty
39 from zope.cachedescriptors.property import Lazy as LazyProperty
@@ -43,9 +45,7 b' from rhodecode.lib import caches'
43 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
45 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
44 from rhodecode.lib.base import (
46 from rhodecode.lib.base import (
45 BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
47 BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
46 from rhodecode.lib.exceptions import (
48 from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
47 HTTPLockedRC, HTTPRequirementError, UserCreationError,
48 NotAllowedToCreateUserError)
49 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
49 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
50 from rhodecode.lib.middleware import appenlight
50 from rhodecode.lib.middleware import appenlight
51 from rhodecode.lib.middleware.utils import scm_app_http
51 from rhodecode.lib.middleware.utils import scm_app_http
@@ -53,6 +53,7 b' from rhodecode.lib.utils import is_valid'
53 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
53 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
54 from rhodecode.lib.vcs.conf import settings as vcs_settings
54 from rhodecode.lib.vcs.conf import settings as vcs_settings
55 from rhodecode.lib.vcs.backends import base
55 from rhodecode.lib.vcs.backends import base
56
56 from rhodecode.model import meta
57 from rhodecode.model import meta
57 from rhodecode.model.db import User, Repository, PullRequest
58 from rhodecode.model.db import User, Repository, PullRequest
58 from rhodecode.model.scm import ScmModel
59 from rhodecode.model.scm import ScmModel
@@ -62,6 +63,28 b' from rhodecode.model.settings import Set'
62 log = logging.getLogger(__name__)
63 log = logging.getLogger(__name__)
63
64
64
65
66 def extract_svn_txn_id(acl_repo_name, data):
67 """
68 Helper method for extraction of svn txn_id from submited XML data during
69 POST operations
70 """
71 try:
72 root = etree.fromstring(data)
73 pat = re.compile(r'/txn/(?P<txn_id>.*)')
74 for el in root:
75 if el.tag == '{DAV:}source':
76 for sub_el in el:
77 if sub_el.tag == '{DAV:}href':
78 match = pat.search(sub_el.text)
79 if match:
80 svn_tx_id = match.groupdict()['txn_id']
81 txn_id = caches.compute_key_from_params(
82 acl_repo_name, svn_tx_id)
83 return txn_id
84 except Exception:
85 log.exception('Failed to extract txn_id')
86
87
65 def initialize_generator(factory):
88 def initialize_generator(factory):
66 """
89 """
67 Initializes the returned generator by draining its first element.
90 Initializes the returned generator by draining its first element.
@@ -565,15 +588,25 b' class SimpleVCS(object):'
565 also handles the locking exceptions which will be triggered when
588 also handles the locking exceptions which will be triggered when
566 the first chunk is produced by the underlying WSGI application.
589 the first chunk is produced by the underlying WSGI application.
567 """
590 """
568 callback_daemon, extras = self._prepare_callback_daemon(extras)
591 txn_id = ''
592 if 'CONTENT_LENGTH' in environ and environ['REQUEST_METHOD'] == 'MERGE':
593 # case for SVN, we want to re-use the callback daemon port
594 # so we use the txn_id, for this we peek the body, and still save
595 # it as wsgi.input
596 data = environ['wsgi.input'].read()
597 environ['wsgi.input'] = StringIO(data)
598 txn_id = extract_svn_txn_id(self.acl_repo_name, data)
599
600 callback_daemon, extras = self._prepare_callback_daemon(
601 extras, environ, action, txn_id=txn_id)
602 log.debug('HOOKS extras is %s', extras)
603
569 config = self._create_config(extras, self.acl_repo_name)
604 config = self._create_config(extras, self.acl_repo_name)
570 log.debug('HOOKS extras is %s', extras)
571 app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
605 app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
606 with callback_daemon:
572 app.rc_extras = extras
607 app.rc_extras = extras
573
608
574 try:
609 try:
575 with callback_daemon:
576 try:
577 response = app(environ, start_response)
610 response = app(environ, start_response)
578 finally:
611 finally:
579 # This statement works together with the decorator
612 # This statement works together with the decorator
@@ -585,28 +618,12 b' class SimpleVCS(object):'
585 # generator is actually used.
618 # generator is actually used.
586 yield "__init__"
619 yield "__init__"
587
620
621 # iter content
588 for chunk in response:
622 for chunk in response:
589 yield chunk
623 yield chunk
590 except Exception as exc:
591 # TODO: martinb: Exceptions are only raised in case of the Pyro4
592 # backend. Refactor this except block after dropping Pyro4 support.
593 # TODO: johbo: Improve "translating" back the exception.
594 if getattr(exc, '_vcs_kind', None) == 'repo_locked':
595 exc = HTTPLockedRC(*exc.args)
596 _code = rhodecode.CONFIG.get('lock_ret_code')
597 log.debug('Repository LOCKED ret code %s!', (_code,))
598 elif getattr(exc, '_vcs_kind', None) == 'requirement':
599 log.debug(
600 'Repository requires features unknown to this Mercurial')
601 exc = HTTPRequirementError(*exc.args)
602 else:
603 raise
604
624
605 for chunk in exc(environ, start_response):
625 try:
606 yield chunk
607 finally:
608 # invalidate cache on push
626 # invalidate cache on push
609 try:
610 if action == 'push':
627 if action == 'push':
611 self._invalidate_cache(self.url_repo_name)
628 self._invalidate_cache(self.url_repo_name)
612 finally:
629 finally:
@@ -634,10 +651,18 b' class SimpleVCS(object):'
634 """Create a safe config representation."""
651 """Create a safe config representation."""
635 raise NotImplementedError()
652 raise NotImplementedError()
636
653
637 def _prepare_callback_daemon(self, extras):
654 def _should_use_callback_daemon(self, extras, environ, action):
655 return True
656
657 def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
658 direct_calls = vcs_settings.HOOKS_DIRECT_CALLS
659 if not self._should_use_callback_daemon(extras, environ, action):
660 # disable callback daemon for actions that don't require it
661 direct_calls = True
662
638 return prepare_callback_daemon(
663 return prepare_callback_daemon(
639 extras, protocol=vcs_settings.HOOKS_PROTOCOL,
664 extras, protocol=vcs_settings.HOOKS_PROTOCOL,
640 use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
665 use_direct_calls=direct_calls, txn_id=txn_id)
641
666
642
667
643 def _should_check_locking(query_string):
668 def _should_check_locking(query_string):
@@ -514,7 +514,6 b' def repo2db_mapper(initial_repo_list, re'
514 :param remove_obsolete: check for obsolete entries in database
514 :param remove_obsolete: check for obsolete entries in database
515 """
515 """
516 from rhodecode.model.repo import RepoModel
516 from rhodecode.model.repo import RepoModel
517 from rhodecode.model.scm import ScmModel
518 from rhodecode.model.repo_group import RepoGroupModel
517 from rhodecode.model.repo_group import RepoGroupModel
519 from rhodecode.model.settings import SettingsModel
518 from rhodecode.model.settings import SettingsModel
520
519
@@ -566,9 +565,8 b' def repo2db_mapper(initial_repo_list, re'
566
565
567 config = db_repo._config
566 config = db_repo._config
568 config.set('extensions', 'largefiles', '')
567 config.set('extensions', 'largefiles', '')
569 ScmModel().install_hooks(
568 repo = db_repo.scm_instance(config=config)
570 db_repo.scm_instance(config=config),
569 repo.install_hooks()
571 repo_type=db_repo.repo_type)
572
570
573 removed = []
571 removed = []
574 if remove_obsolete:
572 if remove_obsolete:
@@ -175,6 +175,7 b' class BaseRepository(object):'
175 EMPTY_COMMIT_ID = '0' * 40
175 EMPTY_COMMIT_ID = '0' * 40
176
176
177 path = None
177 path = None
178 _remote = None
178
179
179 def __init__(self, repo_path, config=None, create=False, **kwargs):
180 def __init__(self, repo_path, config=None, create=False, **kwargs):
180 """
181 """
@@ -648,6 +649,9 b' class BaseRepository(object):'
648 """
649 """
649 return None
650 return None
650
651
652 def install_hooks(self, force=False):
653 return self._remote.install_hooks(force)
654
651
655
652 class BaseCommit(object):
656 class BaseCommit(object):
653 """
657 """
@@ -743,7 +747,7 b' class BaseCommit(object):'
743
747
744 def _get_refs(self):
748 def _get_refs(self):
745 return {
749 return {
746 'branches': [self.branch],
750 'branches': [self.branch] if self.branch else [],
747 'bookmarks': getattr(self, 'bookmarks', []),
751 'bookmarks': getattr(self, 'bookmarks', []),
748 'tags': self.tags
752 'tags': self.tags
749 }
753 }
@@ -858,7 +858,7 b' class RepoModel(BaseModel):'
858 repo = backend(
858 repo = backend(
859 repo_path, config=config, create=True, src_url=clone_uri)
859 repo_path, config=config, create=True, src_url=clone_uri)
860
860
861 ScmModel().install_hooks(repo, repo_type=repo_type)
861 repo.install_hooks()
862
862
863 log.debug('Created repo %s with %s backend',
863 log.debug('Created repo %s with %s backend',
864 safe_unicode(repo_name), safe_unicode(repo_type))
864 safe_unicode(repo_name), safe_unicode(repo_type))
@@ -807,116 +807,6 b' class ScmModel(BaseModel):'
807
807
808 return choices, hist_l
808 return choices, hist_l
809
809
810 def install_git_hook(self, repo, force_create=False):
811 """
812 Creates a rhodecode hook inside a git repository
813
814 :param repo: Instance of VCS repo
815 :param force_create: Create even if same name hook exists
816 """
817
818 loc = os.path.join(repo.path, 'hooks')
819 if not repo.bare:
820 loc = os.path.join(repo.path, '.git', 'hooks')
821 if not os.path.isdir(loc):
822 os.makedirs(loc, mode=0777)
823
824 tmpl_post = pkg_resources.resource_string(
825 'rhodecode', '/'.join(
826 ('config', 'hook_templates', 'git_post_receive.py.tmpl')))
827 tmpl_pre = pkg_resources.resource_string(
828 'rhodecode', '/'.join(
829 ('config', 'hook_templates', 'git_pre_receive.py.tmpl')))
830
831 for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:
832 _hook_file = os.path.join(loc, '%s-receive' % h_type)
833 log.debug('Installing git hook in repo %s', repo)
834 _rhodecode_hook = _check_rhodecode_hook(_hook_file)
835
836 if _rhodecode_hook or force_create:
837 log.debug('writing %s hook file !', h_type)
838 try:
839 with open(_hook_file, 'wb') as f:
840 tmpl = tmpl.replace('_TMPL_', rhodecode.__version__)
841 tmpl = tmpl.replace('_ENV_', sys.executable)
842 f.write(tmpl)
843 os.chmod(_hook_file, 0755)
844 except IOError:
845 log.exception('error writing hook file %s', _hook_file)
846 else:
847 log.debug('skipping writing hook file')
848
849 def install_svn_hooks(self, repo, force_create=False):
850 """
851 Creates rhodecode hooks inside a svn repository
852
853 :param repo: Instance of VCS repo
854 :param force_create: Create even if same name hook exists
855 """
856 hooks_path = os.path.join(repo.path, 'hooks')
857 if not os.path.isdir(hooks_path):
858 os.makedirs(hooks_path)
859 post_commit_tmpl = pkg_resources.resource_string(
860 'rhodecode', '/'.join(
861 ('config', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
862 pre_commit_template = pkg_resources.resource_string(
863 'rhodecode', '/'.join(
864 ('config', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
865 templates = {
866 'post-commit': post_commit_tmpl,
867 'pre-commit': pre_commit_template
868 }
869 for filename in templates:
870 _hook_file = os.path.join(hooks_path, filename)
871 _rhodecode_hook = _check_rhodecode_hook(_hook_file)
872 if _rhodecode_hook or force_create:
873 log.debug('writing %s hook file !', filename)
874 template = templates[filename]
875 try:
876 with open(_hook_file, 'wb') as f:
877 template = template.replace(
878 '_TMPL_', rhodecode.__version__)
879 template = template.replace('_ENV_', sys.executable)
880 f.write(template)
881 os.chmod(_hook_file, 0755)
882 except IOError:
883 log.exception('error writing hook file %s', filename)
884 else:
885 log.debug('skipping writing hook file')
886
887 def install_hooks(self, repo, repo_type):
888 if repo_type == 'git':
889 self.install_git_hook(repo)
890 elif repo_type == 'svn':
891 self.install_svn_hooks(repo)
892
893 def get_server_info(self, environ=None):
810 def get_server_info(self, environ=None):
894 server_info = get_system_info(environ)
811 server_info = get_system_info(environ)
895 return server_info
812 return server_info
896
897
898 def _check_rhodecode_hook(hook_path):
899 """
900 Check if the hook was created by RhodeCode
901 """
902 if not os.path.exists(hook_path):
903 return True
904
905 log.debug('hook exists, checking if it is from rhodecode')
906 hook_content = _read_hook(hook_path)
907 matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content)
908 if matches:
909 try:
910 version = matches.groups()[0]
911 log.debug('got %s, it is rhodecode', version)
912 return True
913 except Exception:
914 log.exception("Exception while reading the hook version.")
915
916 return False
917
918
919 def _read_hook(hook_path):
920 with open(hook_path, 'rb') as f:
921 content = f.read()
922 return content
@@ -377,34 +377,6 b' class TestGenerateVcsResponse(object):'
377 list(result)
377 list(result)
378 assert self.was_cache_invalidated()
378 assert self.was_cache_invalidated()
379
379
380 @mock.patch('rhodecode.lib.middleware.simplevcs.HTTPLockedRC')
381 def test_handles_locking_exception(self, http_locked_rc):
382 result = self.call_controller_with_response_body(
383 self.raise_result_iter(vcs_kind='repo_locked'))
384 assert not http_locked_rc.called
385 # Consume the result
386 list(result)
387 assert http_locked_rc.called
388
389 @mock.patch('rhodecode.lib.middleware.simplevcs.HTTPRequirementError')
390 def test_handles_requirement_exception(self, http_requirement):
391 result = self.call_controller_with_response_body(
392 self.raise_result_iter(vcs_kind='requirement'))
393 assert not http_requirement.called
394 # Consume the result
395 list(result)
396 assert http_requirement.called
397
398 @mock.patch('rhodecode.lib.middleware.simplevcs.HTTPLockedRC')
399 def test_handles_locking_exception_in_app_call(self, http_locked_rc):
400 app_factory_patcher = mock.patch.object(
401 StubVCSController, '_create_wsgi_app')
402 with app_factory_patcher as app_factory:
403 app_factory().side_effect = self.vcs_exception()
404 result = self.call_controller_with_response_body(['a'])
405 list(result)
406 assert http_locked_rc.called
407
408 def test_raises_unknown_exceptions(self):
380 def test_raises_unknown_exceptions(self):
409 result = self.call_controller_with_response_body(
381 result = self.call_controller_with_response_body(
410 self.raise_result_iter(vcs_kind='unknown'))
382 self.raise_result_iter(vcs_kind='unknown'))
@@ -412,7 +384,7 b' class TestGenerateVcsResponse(object):'
412 list(result)
384 list(result)
413
385
414 def test_prepare_callback_daemon_is_called(self):
386 def test_prepare_callback_daemon_is_called(self):
415 def side_effect(extras):
387 def side_effect(extras, environ, action, txn_id=None):
416 return DummyHooksCallbackDaemon(), extras
388 return DummyHooksCallbackDaemon(), extras
417
389
418 prepare_patcher = mock.patch.object(
390 prepare_patcher = mock.patch.object(
@@ -489,10 +461,11 b' class TestPrepareHooksDaemon(object):'
489 return_value=(daemon, expected_extras))
461 return_value=(daemon, expected_extras))
490 with prepare_patcher as prepare_mock:
462 with prepare_patcher as prepare_mock:
491 callback_daemon, extras = controller._prepare_callback_daemon(
463 callback_daemon, extras = controller._prepare_callback_daemon(
492 expected_extras.copy())
464 expected_extras.copy(), {}, 'push')
493 prepare_mock.assert_called_once_with(
465 prepare_mock.assert_called_once_with(
494 expected_extras,
466 expected_extras,
495 protocol=app_settings['vcs.hooks.protocol'],
467 protocol=app_settings['vcs.hooks.protocol'],
468 txn_id=None,
496 use_direct_calls=app_settings['vcs.hooks.direct_calls'])
469 use_direct_calls=app_settings['vcs.hooks.direct_calls'])
497
470
498 assert callback_daemon == daemon
471 assert callback_daemon == daemon
@@ -179,10 +179,12 b' class TestHttpHooksCallbackDaemon(object'
179 daemon = hooks_daemon.HttpHooksCallbackDaemon()
179 daemon = hooks_daemon.HttpHooksCallbackDaemon()
180 assert daemon._daemon == tcp_server
180 assert daemon._daemon == tcp_server
181
181
182 _, port = tcp_server.server_address
183 expected_uri = '{}:{}'.format(daemon.IP_ADDRESS, port)
184 msg = 'Preparing HTTP callback daemon at `{}` and ' \
185 'registering hook object'.format(expected_uri)
182 assert_message_in_log(
186 assert_message_in_log(
183 caplog.records,
187 caplog.records, msg, levelno=logging.DEBUG, module='hooks_daemon')
184 'Preparing HTTP callback daemon and registering hook object',
185 levelno=logging.DEBUG, module='hooks_daemon')
186
188
187 def test_prepare_inits_hooks_uri_and_logs_it(
189 def test_prepare_inits_hooks_uri_and_logs_it(
188 self, tcp_server, caplog):
190 self, tcp_server, caplog):
@@ -193,8 +195,10 b' class TestHttpHooksCallbackDaemon(object'
193 expected_uri = '{}:{}'.format(daemon.IP_ADDRESS, port)
195 expected_uri = '{}:{}'.format(daemon.IP_ADDRESS, port)
194 assert daemon.hooks_uri == expected_uri
196 assert daemon.hooks_uri == expected_uri
195
197
198 msg = 'Preparing HTTP callback daemon at `{}` and ' \
199 'registering hook object'.format(expected_uri)
196 assert_message_in_log(
200 assert_message_in_log(
197 caplog.records, 'Hooks uri is: {}'.format(expected_uri),
201 caplog.records, msg,
198 levelno=logging.DEBUG, module='hooks_daemon')
202 levelno=logging.DEBUG, module='hooks_daemon')
199
203
200 def test_run_creates_a_thread(self, tcp_server):
204 def test_run_creates_a_thread(self, tcp_server):
@@ -263,7 +267,8 b' class TestPrepareHooksDaemon(object):'
263 expected_extras.copy(), protocol=protocol, use_direct_calls=True)
267 expected_extras.copy(), protocol=protocol, use_direct_calls=True)
264 assert isinstance(callback, hooks_daemon.DummyHooksCallbackDaemon)
268 assert isinstance(callback, hooks_daemon.DummyHooksCallbackDaemon)
265 expected_extras['hooks_module'] = 'rhodecode.lib.hooks_daemon'
269 expected_extras['hooks_module'] = 'rhodecode.lib.hooks_daemon'
266 assert extras == expected_extras
270 expected_extras['time'] = extras['time']
271 assert 'extra1' in extras
267
272
268 @pytest.mark.parametrize('protocol, expected_class', (
273 @pytest.mark.parametrize('protocol, expected_class', (
269 ('http', hooks_daemon.HttpHooksCallbackDaemon),
274 ('http', hooks_daemon.HttpHooksCallbackDaemon),
@@ -272,12 +277,15 b' class TestPrepareHooksDaemon(object):'
272 self, protocol, expected_class):
277 self, protocol, expected_class):
273 expected_extras = {
278 expected_extras = {
274 'extra1': 'value1',
279 'extra1': 'value1',
280 'txn_id': 'txnid2',
275 'hooks_protocol': protocol.lower()
281 'hooks_protocol': protocol.lower()
276 }
282 }
277 callback, extras = hooks_daemon.prepare_callback_daemon(
283 callback, extras = hooks_daemon.prepare_callback_daemon(
278 expected_extras.copy(), protocol=protocol, use_direct_calls=False)
284 expected_extras.copy(), protocol=protocol, use_direct_calls=False,
285 txn_id='txnid2')
279 assert isinstance(callback, expected_class)
286 assert isinstance(callback, expected_class)
280 hooks_uri = extras.pop('hooks_uri')
287 extras.pop('hooks_uri')
288 expected_extras['time'] = extras['time']
281 assert extras == expected_extras
289 assert extras == expected_extras
282
290
283 @pytest.mark.parametrize('protocol', (
291 @pytest.mark.parametrize('protocol', (
@@ -245,9 +245,6 b' def test_repo2db_mapper_enables_largefil'
245 repo = backend.create_repo()
245 repo = backend.create_repo()
246 repo_list = {repo.repo_name: 'test'}
246 repo_list = {repo.repo_name: 'test'}
247 with mock.patch('rhodecode.model.db.Repository.scm_instance') as scm_mock:
247 with mock.patch('rhodecode.model.db.Repository.scm_instance') as scm_mock:
248 with mock.patch.multiple('rhodecode.model.scm.ScmModel',
249 install_git_hook=mock.DEFAULT,
250 install_svn_hooks=mock.DEFAULT):
251 utils.repo2db_mapper(repo_list, remove_obsolete=False)
248 utils.repo2db_mapper(repo_list, remove_obsolete=False)
252 _, kwargs = scm_mock.call_args
249 _, kwargs = scm_mock.call_args
253 assert kwargs['config'].get('extensions', 'largefiles') == ''
250 assert kwargs['config'].get('extensions', 'largefiles') == ''
@@ -257,10 +254,7 b' def test_repo2db_mapper_enables_largefil'
257 def test_repo2db_mapper_installs_hooks_for_repos_in_db(backend):
254 def test_repo2db_mapper_installs_hooks_for_repos_in_db(backend):
258 repo = backend.create_repo()
255 repo = backend.create_repo()
259 repo_list = {repo.repo_name: 'test'}
256 repo_list = {repo.repo_name: 'test'}
260 with mock.patch.object(ScmModel, 'install_hooks') as install_hooks_mock:
261 utils.repo2db_mapper(repo_list, remove_obsolete=False)
257 utils.repo2db_mapper(repo_list, remove_obsolete=False)
262 install_hooks_mock.assert_called_once_with(
263 repo.scm_instance(), repo_type=backend.alias)
264
258
265
259
266 @pytest.mark.backends("git", "svn")
260 @pytest.mark.backends("git", "svn")
@@ -269,11 +263,7 b' def test_repo2db_mapper_installs_hooks_f'
269 RepoModel().delete(repo, fs_remove=False)
263 RepoModel().delete(repo, fs_remove=False)
270 meta.Session().commit()
264 meta.Session().commit()
271 repo_list = {repo.repo_name: repo.scm_instance()}
265 repo_list = {repo.repo_name: repo.scm_instance()}
272 with mock.patch.object(ScmModel, 'install_hooks') as install_hooks_mock:
273 utils.repo2db_mapper(repo_list, remove_obsolete=False)
266 utils.repo2db_mapper(repo_list, remove_obsolete=False)
274 assert install_hooks_mock.call_count == 1
275 install_hooks_args, _ = install_hooks_mock.call_args
276 assert install_hooks_args[0].name == repo.repo_name
277
267
278
268
279 class TestPasswordChanged(object):
269 class TestPasswordChanged(object):
@@ -18,10 +18,11 b''
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import os
22 import mock
23 import pytest
21 import tempfile
24 import tempfile
22
25
23 import mock
24 import pytest
25
26
26 from rhodecode.lib.exceptions import AttachedForksError
27 from rhodecode.lib.exceptions import AttachedForksError
27 from rhodecode.lib.utils import make_db_config
28 from rhodecode.lib.utils import make_db_config
@@ -119,22 +120,22 b' class TestRepoModel(object):'
119
120
120 @pytest.mark.backends("git", "svn")
121 @pytest.mark.backends("git", "svn")
121 def test_create_filesystem_repo_installs_hooks(self, tmpdir, backend):
122 def test_create_filesystem_repo_installs_hooks(self, tmpdir, backend):
122 hook_methods = {
123 'git': 'install_git_hook',
124 'svn': 'install_svn_hooks'
125 }
126 repo = backend.create_repo()
123 repo = backend.create_repo()
127 repo_name = repo.repo_name
124 repo_name = repo.repo_name
128 model = RepoModel()
125 model = RepoModel()
129 repo_location = tempfile.mkdtemp()
126 repo_location = tempfile.mkdtemp()
130 model.repos_path = repo_location
127 model.repos_path = repo_location
131 method = hook_methods[backend.alias]
128 repo = model._create_filesystem_repo(
132 with mock.patch.object(ScmModel, method) as hooks_mock:
133 model._create_filesystem_repo(
134 repo_name, backend.alias, repo_group='', clone_uri=None)
129 repo_name, backend.alias, repo_group='', clone_uri=None)
135 assert hooks_mock.call_count == 1
130
136 hook_args, hook_kwargs = hooks_mock.call_args
131 hooks = {
137 assert hook_args[0].name == repo_name
132 'svn': ('pre-commit', 'post-commit'),
133 'git': ('pre-receive', 'post-receive'),
134 }
135 for hook in hooks[backend.alias]:
136 with open(os.path.join(repo.path, 'hooks', hook)) as f:
137 data = f.read()
138 assert 'RC_HOOK_VER' in data
138
139
139 @pytest.mark.parametrize("use_global_config, repo_name_passed", [
140 @pytest.mark.parametrize("use_global_config, repo_name_passed", [
140 (True, False),
141 (True, False),
@@ -194,143 +194,3 b' def test_get_non_unicode_reference(backe'
194 u'tag:Ad\xc4\xb1n\xc4\xb1']
194 u'tag:Ad\xc4\xb1n\xc4\xb1']
195
195
196 assert choices == valid_choices
196 assert choices == valid_choices
197
198
199 class TestInstallSvnHooks(object):
200 HOOK_FILES = ('pre-commit', 'post-commit')
201
202 def test_new_hooks_are_created(self, backend_svn):
203 model = scm.ScmModel()
204 repo = backend_svn.create_repo()
205 vcs_repo = repo.scm_instance()
206 model.install_svn_hooks(vcs_repo)
207
208 hooks_path = os.path.join(vcs_repo.path, 'hooks')
209 assert os.path.isdir(hooks_path)
210 for file_name in self.HOOK_FILES:
211 file_path = os.path.join(hooks_path, file_name)
212 self._check_hook_file_mode(file_path)
213 self._check_hook_file_content(file_path)
214
215 def test_rc_hooks_are_replaced(self, backend_svn):
216 model = scm.ScmModel()
217 repo = backend_svn.create_repo()
218 vcs_repo = repo.scm_instance()
219 hooks_path = os.path.join(vcs_repo.path, 'hooks')
220 file_paths = [os.path.join(hooks_path, f) for f in self.HOOK_FILES]
221
222 for file_path in file_paths:
223 self._create_fake_hook(
224 file_path, content="RC_HOOK_VER = 'abcde'\n")
225
226 model.install_svn_hooks(vcs_repo)
227
228 for file_path in file_paths:
229 self._check_hook_file_content(file_path)
230
231 def test_non_rc_hooks_are_not_replaced_without_force_create(
232 self, backend_svn):
233 model = scm.ScmModel()
234 repo = backend_svn.create_repo()
235 vcs_repo = repo.scm_instance()
236 hooks_path = os.path.join(vcs_repo.path, 'hooks')
237 file_paths = [os.path.join(hooks_path, f) for f in self.HOOK_FILES]
238 non_rc_content = "exit 0\n"
239
240 for file_path in file_paths:
241 self._create_fake_hook(file_path, content=non_rc_content)
242
243 model.install_svn_hooks(vcs_repo)
244
245 for file_path in file_paths:
246 with open(file_path, 'rt') as hook_file:
247 content = hook_file.read()
248 assert content == non_rc_content
249
250 def test_non_rc_hooks_are_replaced_with_force_create(self, backend_svn):
251 model = scm.ScmModel()
252 repo = backend_svn.create_repo()
253 vcs_repo = repo.scm_instance()
254 hooks_path = os.path.join(vcs_repo.path, 'hooks')
255 file_paths = [os.path.join(hooks_path, f) for f in self.HOOK_FILES]
256 non_rc_content = "exit 0\n"
257
258 for file_path in file_paths:
259 self._create_fake_hook(file_path, content=non_rc_content)
260
261 model.install_svn_hooks(vcs_repo, force_create=True)
262
263 for file_path in file_paths:
264 self._check_hook_file_content(file_path)
265
266 def _check_hook_file_mode(self, file_path):
267 assert os.path.exists(file_path)
268 stat_info = os.stat(file_path)
269
270 file_mode = stat.S_IMODE(stat_info.st_mode)
271 expected_mode = int('755', 8)
272 assert expected_mode == file_mode
273
274 def _check_hook_file_content(self, file_path):
275 with open(file_path, 'rt') as hook_file:
276 content = hook_file.read()
277
278 expected_env = '#!{}'.format(sys.executable)
279 expected_rc_version = "\nRC_HOOK_VER = '{}'\n".format(
280 rhodecode.__version__)
281 assert content.strip().startswith(expected_env)
282 assert expected_rc_version in content
283
284 def _create_fake_hook(self, file_path, content):
285 with open(file_path, 'w') as hook_file:
286 hook_file.write(content)
287
288
289 class TestCheckRhodecodeHook(object):
290
291 @patch('os.path.exists', Mock(return_value=False))
292 def test_returns_true_when_no_hook_found(self):
293 result = scm._check_rhodecode_hook('/tmp/fake_hook_file.py')
294 assert result
295
296 @pytest.mark.parametrize("file_content, expected_result", [
297 ("RC_HOOK_VER = '3.3.3'\n", True),
298 ("RC_HOOK = '3.3.3'\n", False),
299 ], ids=no_newline_id_generator)
300 @patch('os.path.exists', Mock(return_value=True))
301 def test_signatures(self, file_content, expected_result):
302 hook_content_patcher = patch.object(
303 scm, '_read_hook', return_value=file_content)
304 with hook_content_patcher:
305 result = scm._check_rhodecode_hook('/tmp/fake_hook_file.py')
306
307 assert result is expected_result
308
309
310 class TestInstallHooks(object):
311 def test_hooks_are_installed_for_git_repo(self, backend_git):
312 repo = backend_git.create_repo()
313 model = scm.ScmModel()
314 scm_repo = repo.scm_instance()
315 with patch.object(model, 'install_git_hook') as hooks_mock:
316 model.install_hooks(scm_repo, repo_type='git')
317 hooks_mock.assert_called_once_with(scm_repo)
318
319 def test_hooks_are_installed_for_svn_repo(self, backend_svn):
320 repo = backend_svn.create_repo()
321 scm_repo = repo.scm_instance()
322 model = scm.ScmModel()
323 with patch.object(scm.ScmModel, 'install_svn_hooks') as hooks_mock:
324 model.install_hooks(scm_repo, repo_type='svn')
325 hooks_mock.assert_called_once_with(scm_repo)
326
327 @pytest.mark.parametrize('hook_method', [
328 'install_svn_hooks',
329 'install_git_hook'])
330 def test_mercurial_doesnt_trigger_hooks(self, backend_hg, hook_method):
331 repo = backend_hg.create_repo()
332 scm_repo = repo.scm_instance()
333 model = scm.ScmModel()
334 with patch.object(scm.ScmModel, hook_method) as hooks_mock:
335 model.install_hooks(scm_repo, repo_type='hg')
336 assert hooks_mock.call_count == 0
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now