##// END OF EJS Templates
migration-tests: improved error reporting
marcink -
r1293:5d5d616f default
parent child Browse files
Show More
@@ -1,273 +1,275 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2017 RhodeCode GmbH
3 # Copyright (C) 2010-2017 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 from subprocess32 import Popen, PIPE
21 from subprocess32 import Popen, PIPE
22 import os
22 import os
23 import shutil
23 import shutil
24 import sys
24 import sys
25 import tempfile
25 import tempfile
26
26
27 import pytest
27 import pytest
28 from sqlalchemy.engine import url
28 from sqlalchemy.engine import url
29
29
30 from rhodecode.tests.fixture import TestINI
30 from rhodecode.tests.fixture import TestINI
31
31
32
32
33 def _get_dbs_from_metafunc(metafunc):
33 def _get_dbs_from_metafunc(metafunc):
34 if hasattr(metafunc.function, 'dbs'):
34 if hasattr(metafunc.function, 'dbs'):
35 # Supported backends by this test function, created from
35 # Supported backends by this test function, created from
36 # pytest.mark.dbs
36 # pytest.mark.dbs
37 backends = metafunc.function.dbs.args
37 backends = metafunc.function.dbs.args
38 else:
38 else:
39 backends = metafunc.config.getoption('--dbs')
39 backends = metafunc.config.getoption('--dbs')
40 return backends
40 return backends
41
41
42
42
43 def pytest_generate_tests(metafunc):
43 def pytest_generate_tests(metafunc):
44 # Support test generation based on --dbs parameter
44 # Support test generation based on --dbs parameter
45 if 'db_backend' in metafunc.fixturenames:
45 if 'db_backend' in metafunc.fixturenames:
46 requested_backends = set(metafunc.config.getoption('--dbs'))
46 requested_backends = set(metafunc.config.getoption('--dbs'))
47 backends = _get_dbs_from_metafunc(metafunc)
47 backends = _get_dbs_from_metafunc(metafunc)
48 backends = requested_backends.intersection(backends)
48 backends = requested_backends.intersection(backends)
49 # TODO: johbo: Disabling a backend did not work out with
49 # TODO: johbo: Disabling a backend did not work out with
50 # parametrization, find better way to achieve this.
50 # parametrization, find better way to achieve this.
51 if not backends:
51 if not backends:
52 metafunc.function._skip = True
52 metafunc.function._skip = True
53 metafunc.parametrize('db_backend_name', backends)
53 metafunc.parametrize('db_backend_name', backends)
54
54
55
55
56 def pytest_collection_modifyitems(session, config, items):
56 def pytest_collection_modifyitems(session, config, items):
57 remaining = [
57 remaining = [
58 i for i in items if not getattr(i.obj, '_skip', False)]
58 i for i in items if not getattr(i.obj, '_skip', False)]
59 items[:] = remaining
59 items[:] = remaining
60
60
61
61
62 @pytest.fixture
62 @pytest.fixture
63 def db_backend(
63 def db_backend(
64 request, db_backend_name, pylons_config, tmpdir_factory):
64 request, db_backend_name, pylons_config, tmpdir_factory):
65 basetemp = tmpdir_factory.getbasetemp().strpath
65 basetemp = tmpdir_factory.getbasetemp().strpath
66 klass = _get_backend(db_backend_name)
66 klass = _get_backend(db_backend_name)
67
67
68 option_name = '--{}-connection-string'.format(db_backend_name)
68 option_name = '--{}-connection-string'.format(db_backend_name)
69 connection_string = request.config.getoption(option_name) or None
69 connection_string = request.config.getoption(option_name) or None
70
70
71 return klass(
71 return klass(
72 config_file=pylons_config, basetemp=basetemp,
72 config_file=pylons_config, basetemp=basetemp,
73 connection_string=connection_string)
73 connection_string=connection_string)
74
74
75
75
76 def _get_backend(backend_type):
76 def _get_backend(backend_type):
77 return {
77 return {
78 'sqlite': SQLiteDBBackend,
78 'sqlite': SQLiteDBBackend,
79 'postgres': PostgresDBBackend,
79 'postgres': PostgresDBBackend,
80 'mysql': MySQLDBBackend,
80 'mysql': MySQLDBBackend,
81 '': EmptyDBBackend
81 '': EmptyDBBackend
82 }[backend_type]
82 }[backend_type]
83
83
84
84
85 class DBBackend(object):
85 class DBBackend(object):
86 _store = os.path.dirname(os.path.abspath(__file__))
86 _store = os.path.dirname(os.path.abspath(__file__))
87 _type = None
87 _type = None
88 _base_ini_config = [{'app:main': {'vcs.start_server': 'false'}}]
88 _base_ini_config = [{'app:main': {'vcs.start_server': 'false'}}]
89 _db_url = [{'app:main': {'sqlalchemy.db1.url': ''}}]
89 _db_url = [{'app:main': {'sqlalchemy.db1.url': ''}}]
90 _base_db_name = 'rhodecode_test_db_backend'
90 _base_db_name = 'rhodecode_test_db_backend'
91
91
92 def __init__(
92 def __init__(
93 self, config_file, db_name=None, basetemp=None,
93 self, config_file, db_name=None, basetemp=None,
94 connection_string=None):
94 connection_string=None):
95 self.fixture_store = os.path.join(self._store, self._type)
95 self.fixture_store = os.path.join(self._store, self._type)
96 self.db_name = db_name or self._base_db_name
96 self.db_name = db_name or self._base_db_name
97 self._base_ini_file = config_file
97 self._base_ini_file = config_file
98 self.stderr = ''
98 self.stderr = ''
99 self.stdout = ''
99 self.stdout = ''
100 self._basetemp = basetemp or tempfile.gettempdir()
100 self._basetemp = basetemp or tempfile.gettempdir()
101 self._repos_location = os.path.join(self._basetemp, 'rc_test_repos')
101 self._repos_location = os.path.join(self._basetemp, 'rc_test_repos')
102 self.connection_string = connection_string
102 self.connection_string = connection_string
103
103
104 @property
104 @property
105 def connection_string(self):
105 def connection_string(self):
106 return self._connection_string
106 return self._connection_string
107
107
108 @connection_string.setter
108 @connection_string.setter
109 def connection_string(self, new_connection_string):
109 def connection_string(self, new_connection_string):
110 if not new_connection_string:
110 if not new_connection_string:
111 new_connection_string = self.get_default_connection_string()
111 new_connection_string = self.get_default_connection_string()
112 else:
112 else:
113 new_connection_string = new_connection_string.format(
113 new_connection_string = new_connection_string.format(
114 db_name=self.db_name)
114 db_name=self.db_name)
115 url_parts = url.make_url(new_connection_string)
115 url_parts = url.make_url(new_connection_string)
116 self._connection_string = new_connection_string
116 self._connection_string = new_connection_string
117 self.user = url_parts.username
117 self.user = url_parts.username
118 self.password = url_parts.password
118 self.password = url_parts.password
119 self.host = url_parts.host
119 self.host = url_parts.host
120
120
121 def get_default_connection_string(self):
121 def get_default_connection_string(self):
122 raise NotImplementedError('default connection_string is required.')
122 raise NotImplementedError('default connection_string is required.')
123
123
124 def execute(self, cmd, env=None, *args):
124 def execute(self, cmd, env=None, *args):
125 """
125 """
126 Runs command on the system with given ``args``.
126 Runs command on the system with given ``args``.
127 """
127 """
128
128
129 command = cmd + ' ' + ' '.join(args)
129 command = cmd + ' ' + ' '.join(args)
130 sys.stdout.write(command)
130 sys.stdout.write(command)
131
131
132 # Tell Python to use UTF-8 encoding out stdout
132 # Tell Python to use UTF-8 encoding out stdout
133 _env = os.environ.copy()
133 _env = os.environ.copy()
134 _env['PYTHONIOENCODING'] = 'UTF-8'
134 _env['PYTHONIOENCODING'] = 'UTF-8'
135 if env:
135 if env:
136 _env.update(env)
136 _env.update(env)
137 self.p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, env=_env)
137 self.p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, env=_env)
138 self.stdout, self.stderr = self.p.communicate()
138 self.stdout, self.stderr = self.p.communicate()
139 sys.stdout.write('COMMAND:'+command+'\n')
139 sys.stdout.write('COMMAND:'+command+'\n')
140 sys.stdout.write(self.stdout)
140 sys.stdout.write(self.stdout)
141 return self.stdout, self.stderr
141 return self.stdout, self.stderr
142
142
143 def assert_returncode_success(self):
143 def assert_returncode_success(self):
144 assert self.p.returncode == 0, self.stderr
144 if not self.p.returncode == 0:
145 print(self.stderr)
146 raise AssertionError('non 0 retcode:{}'.format(self.p.returncode))
145
147
146 def setup_rhodecode_db(self, ini_params=None, env=None):
148 def setup_rhodecode_db(self, ini_params=None, env=None):
147 if not ini_params:
149 if not ini_params:
148 ini_params = self._base_ini_config
150 ini_params = self._base_ini_config
149
151
150 ini_params.extend(self._db_url)
152 ini_params.extend(self._db_url)
151 with TestINI(self._base_ini_file, ini_params,
153 with TestINI(self._base_ini_file, ini_params,
152 self._type, destroy=True) as _ini_file:
154 self._type, destroy=True) as _ini_file:
153 if not os.path.isdir(self._repos_location):
155 if not os.path.isdir(self._repos_location):
154 os.makedirs(self._repos_location)
156 os.makedirs(self._repos_location)
155 self.execute(
157 self.execute(
156 "paster setup-rhodecode {0} --user=marcink "
158 "paster setup-rhodecode {0} --user=marcink "
157 "--email=marcin@rhodeocode.com --password={1} "
159 "--email=marcin@rhodeocode.com --password={1} "
158 "--repos={2} --force-yes".format(
160 "--repos={2} --force-yes".format(
159 _ini_file, 'qweqwe', self._repos_location), env=env)
161 _ini_file, 'qweqwe', self._repos_location), env=env)
160
162
161 def upgrade_database(self, ini_params=None):
163 def upgrade_database(self, ini_params=None):
162 if not ini_params:
164 if not ini_params:
163 ini_params = self._base_ini_config
165 ini_params = self._base_ini_config
164 ini_params.extend(self._db_url)
166 ini_params.extend(self._db_url)
165
167
166 test_ini = TestINI(
168 test_ini = TestINI(
167 self._base_ini_file, ini_params, self._type, destroy=True)
169 self._base_ini_file, ini_params, self._type, destroy=True)
168 with test_ini as ini_file:
170 with test_ini as ini_file:
169 if not os.path.isdir(self._repos_location):
171 if not os.path.isdir(self._repos_location):
170 os.makedirs(self._repos_location)
172 os.makedirs(self._repos_location)
171 self.execute(
173 self.execute(
172 "paster upgrade-db {} --force-yes".format(ini_file))
174 "paster upgrade-db {} --force-yes".format(ini_file))
173
175
174 def setup_db(self):
176 def setup_db(self):
175 raise NotImplementedError
177 raise NotImplementedError
176
178
177 def teardown_db(self):
179 def teardown_db(self):
178 raise NotImplementedError
180 raise NotImplementedError
179
181
180 def import_dump(self, dumpname):
182 def import_dump(self, dumpname):
181 raise NotImplementedError
183 raise NotImplementedError
182
184
183
185
184 class EmptyDBBackend(DBBackend):
186 class EmptyDBBackend(DBBackend):
185 _type = ''
187 _type = ''
186
188
187 def setup_db(self):
189 def setup_db(self):
188 pass
190 pass
189
191
190 def teardown_db(self):
192 def teardown_db(self):
191 pass
193 pass
192
194
193 def import_dump(self, dumpname):
195 def import_dump(self, dumpname):
194 pass
196 pass
195
197
196 def assert_returncode_success(self):
198 def assert_returncode_success(self):
197 assert True
199 assert True
198
200
199
201
200 class SQLiteDBBackend(DBBackend):
202 class SQLiteDBBackend(DBBackend):
201 _type = 'sqlite'
203 _type = 'sqlite'
202
204
203 def get_default_connection_string(self):
205 def get_default_connection_string(self):
204 return 'sqlite:///{}/{}.sqlite'.format(self._basetemp, self.db_name)
206 return 'sqlite:///{}/{}.sqlite'.format(self._basetemp, self.db_name)
205
207
206 def setup_db(self):
208 def setup_db(self):
207 # dump schema for tests
209 # dump schema for tests
208 # cp -v $TEST_DB_NAME
210 # cp -v $TEST_DB_NAME
209 self._db_url = [{'app:main': {
211 self._db_url = [{'app:main': {
210 'sqlalchemy.db1.url': self.connection_string}}]
212 'sqlalchemy.db1.url': self.connection_string}}]
211
213
212 def import_dump(self, dumpname):
214 def import_dump(self, dumpname):
213 dump = os.path.join(self.fixture_store, dumpname)
215 dump = os.path.join(self.fixture_store, dumpname)
214 shutil.copy(
216 shutil.copy(
215 dump,
217 dump,
216 os.path.join(self._basetemp, '{0.db_name}.sqlite'.format(self)))
218 os.path.join(self._basetemp, '{0.db_name}.sqlite'.format(self)))
217
219
218 def teardown_db(self):
220 def teardown_db(self):
219 self.execute("rm -rf {}.sqlite".format(
221 self.execute("rm -rf {}.sqlite".format(
220 os.path.join(self._basetemp, self.db_name)))
222 os.path.join(self._basetemp, self.db_name)))
221
223
222
224
223 class MySQLDBBackend(DBBackend):
225 class MySQLDBBackend(DBBackend):
224 _type = 'mysql'
226 _type = 'mysql'
225
227
226 def get_default_connection_string(self):
228 def get_default_connection_string(self):
227 return 'mysql://root:qweqwe@127.0.0.1/{}'.format(self.db_name)
229 return 'mysql://root:qweqwe@127.0.0.1/{}'.format(self.db_name)
228
230
229 def setup_db(self):
231 def setup_db(self):
230 # dump schema for tests
232 # dump schema for tests
231 # mysqldump -uroot -pqweqwe $TEST_DB_NAME
233 # mysqldump -uroot -pqweqwe $TEST_DB_NAME
232 self._db_url = [{'app:main': {
234 self._db_url = [{'app:main': {
233 'sqlalchemy.db1.url': self.connection_string}}]
235 'sqlalchemy.db1.url': self.connection_string}}]
234 self.execute("mysql -v -u{} -p{} -e 'create database '{}';'".format(
236 self.execute("mysql -v -u{} -p{} -e 'create database '{}';'".format(
235 self.user, self.password, self.db_name))
237 self.user, self.password, self.db_name))
236
238
237 def import_dump(self, dumpname):
239 def import_dump(self, dumpname):
238 dump = os.path.join(self.fixture_store, dumpname)
240 dump = os.path.join(self.fixture_store, dumpname)
239 self.execute("mysql -u{} -p{} {} < {}".format(
241 self.execute("mysql -u{} -p{} {} < {}".format(
240 self.user, self.password, self.db_name, dump))
242 self.user, self.password, self.db_name, dump))
241
243
242 def teardown_db(self):
244 def teardown_db(self):
243 self.execute("mysql -v -u{} -p{} -e 'drop database '{}';'".format(
245 self.execute("mysql -v -u{} -p{} -e 'drop database '{}';'".format(
244 self.user, self.password, self.db_name))
246 self.user, self.password, self.db_name))
245
247
246
248
247 class PostgresDBBackend(DBBackend):
249 class PostgresDBBackend(DBBackend):
248 _type = 'postgres'
250 _type = 'postgres'
249
251
250 def get_default_connection_string(self):
252 def get_default_connection_string(self):
251 return 'postgresql://postgres:qweqwe@localhost/{}'.format(self.db_name)
253 return 'postgresql://postgres:qweqwe@localhost/{}'.format(self.db_name)
252
254
253 def setup_db(self):
255 def setup_db(self):
254 # dump schema for tests
256 # dump schema for tests
255 # pg_dump -U postgres -h localhost $TEST_DB_NAME
257 # pg_dump -U postgres -h localhost $TEST_DB_NAME
256 self._db_url = [{'app:main': {
258 self._db_url = [{'app:main': {
257 'sqlalchemy.db1.url':
259 'sqlalchemy.db1.url':
258 self.connection_string}}]
260 self.connection_string}}]
259 self.execute("PGPASSWORD={} psql -U {} -h localhost "
261 self.execute("PGPASSWORD={} psql -U {} -h localhost "
260 "-c 'create database '{}';'".format(
262 "-c 'create database '{}';'".format(
261 self.password, self.user, self.db_name))
263 self.password, self.user, self.db_name))
262
264
263 def teardown_db(self):
265 def teardown_db(self):
264 self.execute("PGPASSWORD={} psql -U {} -h localhost "
266 self.execute("PGPASSWORD={} psql -U {} -h localhost "
265 "-c 'drop database if exists '{}';'".format(
267 "-c 'drop database if exists '{}';'".format(
266 self.password, self.user, self.db_name))
268 self.password, self.user, self.db_name))
267
269
268 def import_dump(self, dumpname):
270 def import_dump(self, dumpname):
269 dump = os.path.join(self.fixture_store, dumpname)
271 dump = os.path.join(self.fixture_store, dumpname)
270 self.execute(
272 self.execute(
271 "PGPASSWORD={} psql -U {} -h localhost -d {} -1 "
273 "PGPASSWORD={} psql -U {} -h localhost -d {} -1 "
272 "-f {}".format(
274 "-f {}".format(
273 self.password, self.user, self.db_name, dump))
275 self.password, self.user, self.db_name, dump))
General Comments 0
You need to be logged in to leave comments. Login now