##// END OF EJS Templates
dbmigrate: python3 fixes
super-admin -
r5163:86c1ad25 default
parent child Browse files
Show More
@@ -1,27 +1,27 b''
1 1 """
2 2 Configuration parser module.
3 3 """
4 4
5 import configparser
5 from configparser import ConfigParser
6 6
7 7 from rhodecode.lib.dbmigrate.migrate.versioning.config import *
8 8 from rhodecode.lib.dbmigrate.migrate.versioning import pathed
9 9
10 10
11 11 class Parser(ConfigParser):
12 12 """A project configuration file."""
13 13
14 14 def to_dict(self, sections=None):
15 15 """It's easier to access config values like dictionaries"""
16 16 return self._sections
17 17
18 18
19 19 class Config(pathed.Pathed, Parser):
20 20 """Configuration class."""
21 21
22 22 def __init__(self, path, *p, **k):
23 23 """Confirm the config file exists; read it."""
24 24 self.require_found(path)
25 25 pathed.Pathed.__init__(self, path)
26 26 Parser.__init__(self, *p, **k)
27 27 self.read(path)
@@ -1,299 +1,299 b''
1 1 """
2 2 Schema differencing support.
3 3 """
4 4
5 5 import logging
6 6 import sqlalchemy
7 7
8 8 from sqlalchemy.types import Float
9 9
10 10 log = logging.getLogger(__name__)
11 11
12 12
13 13 def getDiffOfModelAgainstDatabase(metadata, engine, excludeTables=None):
14 14 """
15 15 Return differences of model against database.
16 16
17 17 :return: object which will evaluate to :keyword:`True` if there \
18 18 are differences else :keyword:`False`.
19 19 """
20 20 db_metadata = sqlalchemy.MetaData(engine)
21 21 db_metadata.reflect()
22 22
23 23 # sqlite will include a dynamically generated 'sqlite_sequence' table if
24 24 # there are autoincrement sequences in the database; this should not be
25 25 # compared.
26 26 if engine.dialect.name == 'sqlite':
27 27 if 'sqlite_sequence' in db_metadata.tables:
28 28 db_metadata.remove(db_metadata.tables['sqlite_sequence'])
29 29
30 30 return SchemaDiff(metadata, db_metadata,
31 31 labelA='model',
32 32 labelB='database',
33 33 excludeTables=excludeTables)
34 34
35 35
36 36 def getDiffOfModelAgainstModel(metadataA, metadataB, excludeTables=None):
37 37 """
38 38 Return differences of model against another model.
39 39
40 40 :return: object which will evaluate to :keyword:`True` if there \
41 41 are differences else :keyword:`False`.
42 42 """
43 43 return SchemaDiff(metadataA, metadataB, excludeTables=excludeTables)
44 44
45 45
46 46 class ColDiff(object):
47 47 """
48 48 Container for differences in one :class:`~sqlalchemy.schema.Column`
49 49 between two :class:`~sqlalchemy.schema.Table` instances, ``A``
50 50 and ``B``.
51 51
52 52 .. attribute:: col_A
53 53
54 54 The :class:`~sqlalchemy.schema.Column` object for A.
55 55
56 56 .. attribute:: col_B
57 57
58 58 The :class:`~sqlalchemy.schema.Column` object for B.
59 59
60 60 .. attribute:: type_A
61 61
62 62 The most generic type of the :class:`~sqlalchemy.schema.Column`
63 63 object in A.
64 64
65 65 .. attribute:: type_B
66 66
67 67 The most generic type of the :class:`~sqlalchemy.schema.Column`
68 68 object in A.
69 69
70 70 """
71 71
72 72 diff = False
73 73
74 74 def __init__(self,col_A,col_B):
75 75 self.col_A = col_A
76 76 self.col_B = col_B
77 77
78 78 self.type_A = col_A.type
79 79 self.type_B = col_B.type
80 80
81 81 self.affinity_A = self.type_A._type_affinity
82 82 self.affinity_B = self.type_B._type_affinity
83 83
84 84 if self.affinity_A is not self.affinity_B:
85 85 self.diff = True
86 86 return
87 87
88 88 if isinstance(self.type_A,Float) or isinstance(self.type_B,Float):
89 89 if not (isinstance(self.type_A,Float) and isinstance(self.type_B,Float)):
90 90 self.diff=True
91 91 return
92 92
93 93 for attr in ('precision','scale','length'):
94 94 A = getattr(self.type_A,attr,None)
95 95 B = getattr(self.type_B,attr,None)
96 96 if not (A is None or B is None) and A!=B:
97 97 self.diff=True
98 98 return
99 99
100 def __bool__(self):
100 def __nonzero__(self):
101 101 return self.diff
102 102
103 103 __bool__ = __nonzero__
104 104
105 105
106 106 class TableDiff(object):
107 107 """
108 108 Container for differences in one :class:`~sqlalchemy.schema.Table`
109 109 between two :class:`~sqlalchemy.schema.MetaData` instances, ``A``
110 110 and ``B``.
111 111
112 112 .. attribute:: columns_missing_from_A
113 113
114 114 A sequence of column names that were found in B but weren't in
115 115 A.
116 116
117 117 .. attribute:: columns_missing_from_B
118 118
119 119 A sequence of column names that were found in A but weren't in
120 120 B.
121 121
122 122 .. attribute:: columns_different
123 123
124 124 A dictionary containing information about columns that were
125 125 found to be different.
126 126 It maps column names to a :class:`ColDiff` objects describing the
127 127 differences found.
128 128 """
129 129 __slots__ = (
130 130 'columns_missing_from_A',
131 131 'columns_missing_from_B',
132 132 'columns_different',
133 133 )
134 134
135 def __bool__(self):
135 def __nonzero__(self):
136 136 return bool(
137 137 self.columns_missing_from_A or
138 138 self.columns_missing_from_B or
139 139 self.columns_different
140 140 )
141 141
142 142 __bool__ = __nonzero__
143 143
144 144 class SchemaDiff(object):
145 145 """
146 146 Compute the difference between two :class:`~sqlalchemy.schema.MetaData`
147 147 objects.
148 148
149 149 The string representation of a :class:`SchemaDiff` will summarise
150 150 the changes found between the two
151 151 :class:`~sqlalchemy.schema.MetaData` objects.
152 152
153 153 The length of a :class:`SchemaDiff` will give the number of
154 154 changes found, enabling it to be used much like a boolean in
155 155 expressions.
156 156
157 157 :param metadataA:
158 158 First :class:`~sqlalchemy.schema.MetaData` to compare.
159 159
160 160 :param metadataB:
161 161 Second :class:`~sqlalchemy.schema.MetaData` to compare.
162 162
163 163 :param labelA:
164 164 The label to use in messages about the first
165 165 :class:`~sqlalchemy.schema.MetaData`.
166 166
167 167 :param labelB:
168 168 The label to use in messages about the second
169 169 :class:`~sqlalchemy.schema.MetaData`.
170 170
171 171 :param excludeTables:
172 172 A sequence of table names to exclude.
173 173
174 174 .. attribute:: tables_missing_from_A
175 175
176 176 A sequence of table names that were found in B but weren't in
177 177 A.
178 178
179 179 .. attribute:: tables_missing_from_B
180 180
181 181 A sequence of table names that were found in A but weren't in
182 182 B.
183 183
184 184 .. attribute:: tables_different
185 185
186 186 A dictionary containing information about tables that were found
187 187 to be different.
188 188 It maps table names to a :class:`TableDiff` objects describing the
189 189 differences found.
190 190 """
191 191
192 192 def __init__(self,
193 193 metadataA, metadataB,
194 194 labelA='metadataA',
195 195 labelB='metadataB',
196 196 excludeTables=None):
197 197
198 198 self.metadataA, self.metadataB = metadataA, metadataB
199 199 self.labelA, self.labelB = labelA, labelB
200 200 self.label_width = max(len(labelA),len(labelB))
201 201 excludeTables = set(excludeTables or [])
202 202
203 203 A_table_names = set(metadataA.tables.keys())
204 204 B_table_names = set(metadataB.tables.keys())
205 205
206 206 self.tables_missing_from_A = sorted(
207 207 B_table_names - A_table_names - excludeTables
208 208 )
209 209 self.tables_missing_from_B = sorted(
210 210 A_table_names - B_table_names - excludeTables
211 211 )
212 212
213 213 self.tables_different = {}
214 214 for table_name in A_table_names.intersection(B_table_names):
215 215
216 216 td = TableDiff()
217 217
218 218 A_table = metadataA.tables[table_name]
219 219 B_table = metadataB.tables[table_name]
220 220
221 221 A_column_names = set(A_table.columns.keys())
222 222 B_column_names = set(B_table.columns.keys())
223 223
224 224 td.columns_missing_from_A = sorted(
225 225 B_column_names - A_column_names
226 226 )
227 227
228 228 td.columns_missing_from_B = sorted(
229 229 A_column_names - B_column_names
230 230 )
231 231
232 232 td.columns_different = {}
233 233
234 234 for col_name in A_column_names.intersection(B_column_names):
235 235
236 236 cd = ColDiff(
237 237 A_table.columns.get(col_name),
238 238 B_table.columns.get(col_name)
239 239 )
240 240
241 241 if cd:
242 242 td.columns_different[col_name]=cd
243 243
244 244 # XXX - index and constraint differences should
245 245 # be checked for here
246 246
247 247 if td:
248 248 self.tables_different[table_name]=td
249 249
250 250 def __str__(self):
251 251 """ Summarize differences. """
252 252 out = []
253 253 column_template =' %%%is: %%r' % self.label_width
254 254
255 255 for names,label in (
256 256 (self.tables_missing_from_A,self.labelA),
257 257 (self.tables_missing_from_B,self.labelB),
258 258 ):
259 259 if names:
260 260 out.append(
261 261 ' tables missing from %s: %s' % (
262 262 label,', '.join(sorted(names))
263 263 )
264 264 )
265 265
266 266 for name,td in sorted(self.tables_different.items()):
267 267 out.append(
268 268 ' table with differences: %s' % name
269 269 )
270 270 for names,label in (
271 271 (td.columns_missing_from_A,self.labelA),
272 272 (td.columns_missing_from_B,self.labelB),
273 273 ):
274 274 if names:
275 275 out.append(
276 276 ' %s missing these columns: %s' % (
277 277 label,', '.join(sorted(names))
278 278 )
279 279 )
280 280 for name,cd in list(td.columns_different.items()):
281 281 out.append(' column with differences: %s' % name)
282 282 out.append(column_template % (self.labelA,cd.col_A))
283 283 out.append(column_template % (self.labelB,cd.col_B))
284 284
285 285 if out:
286 286 out.insert(0, 'Schema diffs:')
287 287 return '\n'.join(out)
288 288 else:
289 289 return 'No schema diffs'
290 290
291 291 def __len__(self):
292 292 """
293 293 Used in bool evaluation, return of 0 means no diffs.
294 294 """
295 295 return (
296 296 len(self.tables_missing_from_A) +
297 297 len(self.tables_missing_from_B) +
298 298 len(self.tables_different)
299 299 )
@@ -1,263 +1,269 b''
1 1 #!/usr/bin/env python
2 2
3 3
4 4 import os
5 5 import re
6 6 import shutil
7 7 import logging
8 8
9 9 from rhodecode.lib.dbmigrate.migrate import exceptions
10 10 from rhodecode.lib.dbmigrate.migrate.versioning import pathed, script
11 11 from datetime import datetime
12 12
13 13
14 14 log = logging.getLogger(__name__)
15 15
16 16 class VerNum(object):
17 17 """A version number that behaves like a string and int at the same time"""
18 18
19 19 _instances = {}
20 20
21 21 def __new__(cls, value):
22 22 val = str(value)
23 23 if val not in cls._instances:
24 24 cls._instances[val] = super(VerNum, cls).__new__(cls)
25 25 ret = cls._instances[val]
26 26 return ret
27 27
28 28 def __init__(self,value):
29 29 self.value = str(int(value))
30 30 if self < 0:
31 31 raise ValueError("Version number cannot be negative")
32 32
33 33 def __add__(self, value):
34 34 ret = int(self) + int(value)
35 35 return VerNum(ret)
36 36
37 37 def __sub__(self, value):
38 38 return self + (int(value) * -1)
39 39
40 40 def __eq__(self, value):
41 41 return int(self) == int(value)
42 42
43 43 def __ne__(self, value):
44 44 return int(self) != int(value)
45 45
46 46 def __lt__(self, value):
47 47 return int(self) < int(value)
48 48
49 49 def __gt__(self, value):
50 50 return int(self) > int(value)
51 51
52 52 def __ge__(self, value):
53 53 return int(self) >= int(value)
54 54
55 55 def __le__(self, value):
56 56 return int(self) <= int(value)
57 57
58 58 def __repr__(self):
59 59 return "<VerNum(%s)>" % self.value
60 60
61 61 def __str__(self):
62 62 return str(self.value)
63 63
64 64 def __int__(self):
65 65 return int(self.value)
66 66
67 def __index__(self):
68 return int(self.value)
69
70 def __hash__(self):
71 return hash(self.value)
72
67 73
68 74 class Collection(pathed.Pathed):
69 75 """A collection of versioning scripts in a repository"""
70 76
71 77 FILENAME_WITH_VERSION = re.compile(r'^(\d{3,}).*')
72 78
73 79 def __init__(self, path):
74 80 """Collect current version scripts in repository
75 81 and store them in self.versions
76 82 """
77 83 super(Collection, self).__init__(path)
78 84
79 85 # Create temporary list of files, allowing skipped version numbers.
80 86 files = os.listdir(path)
81 87 if '1' in files:
82 88 # deprecation
83 89 raise Exception('It looks like you have a repository in the old '
84 90 'format (with directories for each version). '
85 91 'Please convert repository before proceeding.')
86 92
87 93 tempVersions = {}
88 94 for filename in files:
89 95 match = self.FILENAME_WITH_VERSION.match(filename)
90 96 if match:
91 97 num = int(match.group(1))
92 98 tempVersions.setdefault(num, []).append(filename)
93 99 else:
94 100 pass # Must be a helper file or something, let's ignore it.
95 101
96 102 # Create the versions member where the keys
97 103 # are VerNum's and the values are Version's.
98 104 self.versions = {}
99 105 for num, files in list(tempVersions.items()):
100 106 self.versions[VerNum(num)] = Version(num, path, files)
101 107
102 108 @property
103 109 def latest(self):
104 110 """:returns: Latest version in Collection"""
105 111 return max([VerNum(0)] + list(self.versions.keys()))
106 112
107 113 def _next_ver_num(self, use_timestamp_numbering):
108 114 if use_timestamp_numbering is True:
109 115 return VerNum(int(datetime.utcnow().strftime('%Y%m%d%H%M%S')))
110 116 else:
111 117 return self.latest + 1
112 118
113 119 def create_new_python_version(self, description, **k):
114 120 """Create Python files for new version"""
115 121 ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
116 122 extra = str_to_filename(description)
117 123
118 124 if extra:
119 125 if extra == '_':
120 126 extra = ''
121 127 elif not extra.startswith('_'):
122 128 extra = '_%s' % extra
123 129
124 130 filename = '%03d%s.py' % (ver, extra)
125 131 filepath = self._version_path(filename)
126 132
127 133 script.PythonScript.create(filepath, **k)
128 134 self.versions[ver] = Version(ver, self.path, [filename])
129 135
130 136 def create_new_sql_version(self, database, description, **k):
131 137 """Create SQL files for new version"""
132 138 ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
133 139 self.versions[ver] = Version(ver, self.path, [])
134 140
135 141 extra = str_to_filename(description)
136 142
137 143 if extra:
138 144 if extra == '_':
139 145 extra = ''
140 146 elif not extra.startswith('_'):
141 147 extra = '_%s' % extra
142 148
143 149 # Create new files.
144 150 for op in ('upgrade', 'downgrade'):
145 151 filename = '%03d%s_%s_%s.sql' % (ver, extra, database, op)
146 152 filepath = self._version_path(filename)
147 153 script.SqlScript.create(filepath, **k)
148 154 self.versions[ver].add_script(filepath)
149 155
150 156 def version(self, vernum=None):
151 157 """Returns latest Version if vernum is not given.
152 158 Otherwise, returns wanted version"""
153 159 if vernum is None:
154 160 vernum = self.latest
155 161 return self.versions[VerNum(vernum)]
156 162
157 163 @classmethod
158 164 def clear(cls):
159 165 super(Collection, cls).clear()
160 166
161 167 def _version_path(self, ver):
162 168 """Returns path of file in versions repository"""
163 169 return os.path.join(self.path, str(ver))
164 170
165 171
166 172 class Version(object):
167 173 """A single version in a collection
168 174 :param vernum: Version Number
169 175 :param path: Path to script files
170 176 :param filelist: List of scripts
171 177 :type vernum: int, VerNum
172 178 :type path: string
173 179 :type filelist: list
174 180 """
175 181
176 182 def __init__(self, vernum, path, filelist):
177 183 self.version = VerNum(vernum)
178 184
179 185 # Collect scripts in this folder
180 186 self.sql = {}
181 187 self.python = None
182 188
183 189 for script in filelist:
184 190 self.add_script(os.path.join(path, script))
185 191
186 192 def script(self, database=None, operation=None):
187 193 """Returns SQL or Python Script"""
188 194 for db in (database, 'default'):
189 195 # Try to return a .sql script first
190 196 try:
191 197 return self.sql[db][operation]
192 198 except KeyError:
193 199 continue # No .sql script exists
194 200
195 201 # TODO: maybe add force Python parameter?
196 202 ret = self.python
197 203
198 204 assert ret is not None, \
199 205 "There is no script for %d version" % self.version
200 206 return ret
201 207
202 208 def add_script(self, path):
203 209 """Add script to Collection/Version"""
204 210 if path.endswith(Extensions.py):
205 211 self._add_script_py(path)
206 212 elif path.endswith(Extensions.sql):
207 213 self._add_script_sql(path)
208 214
209 215 SQL_FILENAME = re.compile(r'^.*\.sql')
210 216
211 217 def _add_script_sql(self, path):
212 218 basename = os.path.basename(path)
213 219 match = self.SQL_FILENAME.match(basename)
214 220
215 221 if match:
216 222 basename = basename.replace('.sql', '')
217 223 parts = basename.split('_')
218 224 if len(parts) < 3:
219 225 raise exceptions.ScriptError(
220 226 "Invalid SQL script name %s " % basename + \
221 227 "(needs to be ###_description_database_operation.sql)")
222 228 version = parts[0]
223 229 op = parts[-1]
224 230 # NOTE(mriedem): check for ibm_db_sa as the database in the name
225 231 if 'ibm_db_sa' in basename:
226 232 if len(parts) == 6:
227 233 dbms = '_'.join(parts[-4: -1])
228 234 else:
229 235 raise exceptions.ScriptError(
230 236 "Invalid ibm_db_sa SQL script name '%s'; "
231 237 "(needs to be "
232 238 "###_description_ibm_db_sa_operation.sql)" % basename)
233 239 else:
234 240 dbms = parts[-2]
235 241 else:
236 242 raise exceptions.ScriptError(
237 243 "Invalid SQL script name %s " % basename + \
238 244 "(needs to be ###_description_database_operation.sql)")
239 245
240 246 # File the script into a dictionary
241 247 self.sql.setdefault(dbms, {})[op] = script.SqlScript(path)
242 248
243 249 def _add_script_py(self, path):
244 250 if self.python is not None:
245 251 raise exceptions.ScriptError('You can only have one Python script '
246 252 'per version, but you have: %s and %s' % (self.python, path))
247 253 self.python = script.PythonScript(path)
248 254
249 255
250 256 class Extensions:
251 257 """A namespace for file extensions"""
252 258 py = 'py'
253 259 sql = 'sql'
254 260
255 261 def str_to_filename(s):
256 262 """Replaces spaces, (double and single) quotes
257 263 and double underscores to underscores
258 264 """
259 265
260 266 s = s.replace(' ', '_').replace('"', '_').replace("'", '_').replace(".", "_")
261 267 while '__' in s:
262 268 s = s.replace('__', '_')
263 269 return s
General Comments 0
You need to be logged in to leave comments. Login now