##// END OF EJS Templates
accept that repos are read-only - very convenient for testing....
marcink -
r3980:3648a2b2 default
parent child Browse files
Show More
@@ -1,721 +1,729 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 rhodecode.lib.db_manage
3 rhodecode.lib.db_manage
4 ~~~~~~~~~~~~~~~~~~~~~~~
4 ~~~~~~~~~~~~~~~~~~~~~~~
5
5
6 Database creation, and setup module for RhodeCode. Used for creation
6 Database creation, and setup module for RhodeCode. Used for creation
7 of database as well as for migration operations
7 of database as well as for migration operations
8
8
9 :created_on: Apr 10, 2010
9 :created_on: Apr 10, 2010
10 :author: marcink
10 :author: marcink
11 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
11 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
12 :license: GPLv3, see COPYING for more details.
12 :license: GPLv3, see COPYING for more details.
13 """
13 """
14 # This program is free software: you can redistribute it and/or modify
14 # This program is free software: you can redistribute it and/or modify
15 # it under the terms of the GNU General Public License as published by
15 # it under the terms of the GNU General Public License as published by
16 # the Free Software Foundation, either version 3 of the License, or
16 # the Free Software Foundation, either version 3 of the License, or
17 # (at your option) any later version.
17 # (at your option) any later version.
18 #
18 #
19 # This program is distributed in the hope that it will be useful,
19 # This program is distributed in the hope that it will be useful,
20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 # GNU General Public License for more details.
22 # GNU General Public License for more details.
23 #
23 #
24 # You should have received a copy of the GNU General Public License
24 # You should have received a copy of the GNU General Public License
25 # along with this program. If not, see <http://www.gnu.org/licenses/>.
25 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26
26
27 import os
27 import os
28 import sys
28 import sys
29 import time
29 import time
30 import uuid
30 import uuid
31 import logging
31 import logging
32 from os.path import dirname as dn, join as jn
32 from os.path import dirname as dn, join as jn
33 import datetime
33 import datetime
34
34
35 from rhodecode import __dbversion__, __py_version__
35 from rhodecode import __dbversion__, __py_version__
36
36
37 from rhodecode.model.user import UserModel
37 from rhodecode.model.user import UserModel
38 from rhodecode.lib.utils import ask_ok
38 from rhodecode.lib.utils import ask_ok
39 from rhodecode.model import init_model
39 from rhodecode.model import init_model
40 from rhodecode.model.db import User, Permission, RhodeCodeUi, \
40 from rhodecode.model.db import User, Permission, RhodeCodeUi, \
41 RhodeCodeSetting, UserToPerm, DbMigrateVersion, RepoGroup, \
41 RhodeCodeSetting, UserToPerm, DbMigrateVersion, RepoGroup, \
42 UserRepoGroupToPerm, CacheInvalidation, UserGroup
42 UserRepoGroupToPerm, CacheInvalidation, UserGroup
43
43
44 from sqlalchemy.engine import create_engine
44 from sqlalchemy.engine import create_engine
45 from rhodecode.model.repos_group import ReposGroupModel
45 from rhodecode.model.repos_group import ReposGroupModel
46 #from rhodecode.model import meta
46 #from rhodecode.model import meta
47 from rhodecode.model.meta import Session, Base
47 from rhodecode.model.meta import Session, Base
48 from rhodecode.model.repo import RepoModel
48 from rhodecode.model.repo import RepoModel
49 from rhodecode.model.permission import PermissionModel
49 from rhodecode.model.permission import PermissionModel
50 from rhodecode.model.users_group import UserGroupModel
50 from rhodecode.model.users_group import UserGroupModel
51
51
52
52
53 log = logging.getLogger(__name__)
53 log = logging.getLogger(__name__)
54
54
55
55
56 def notify(msg):
56 def notify(msg):
57 """
57 """
58 Notification for migrations messages
58 Notification for migrations messages
59 """
59 """
60 ml = len(msg) + (4 * 2)
60 ml = len(msg) + (4 * 2)
61 print('\n%s\n*** %s ***\n%s' % ('*' * ml, msg, '*' * ml)).upper()
61 print('\n%s\n*** %s ***\n%s' % ('*' * ml, msg, '*' * ml)).upper()
62
62
63
63
64 class UpgradeSteps(object):
64 class UpgradeSteps(object):
65 """
65 """
66 Those steps follow schema versions so for example schema
66 Those steps follow schema versions so for example schema
67 for example schema with seq 002 == step_2 and so on.
67 for example schema with seq 002 == step_2 and so on.
68 """
68 """
69
69
70 def __init__(self, klass):
70 def __init__(self, klass):
71 self.klass = klass
71 self.klass = klass
72
72
73 def step_1(self):
73 def step_1(self):
74 pass
74 pass
75
75
76 def step_2(self):
76 def step_2(self):
77 notify('Patching repo paths for newer version of RhodeCode')
77 notify('Patching repo paths for newer version of RhodeCode')
78 self.klass.fix_repo_paths()
78 self.klass.fix_repo_paths()
79
79
80 notify('Patching default user of RhodeCode')
80 notify('Patching default user of RhodeCode')
81 self.klass.fix_default_user()
81 self.klass.fix_default_user()
82
82
83 log.info('Changing ui settings')
83 log.info('Changing ui settings')
84 self.klass.create_ui_settings()
84 self.klass.create_ui_settings()
85
85
86 def step_3(self):
86 def step_3(self):
87 notify('Adding additional settings into RhodeCode db')
87 notify('Adding additional settings into RhodeCode db')
88 self.klass.fix_settings()
88 self.klass.fix_settings()
89 notify('Adding ldap defaults')
89 notify('Adding ldap defaults')
90 self.klass.create_ldap_options(skip_existing=True)
90 self.klass.create_ldap_options(skip_existing=True)
91
91
92 def step_4(self):
92 def step_4(self):
93 notify('create permissions and fix groups')
93 notify('create permissions and fix groups')
94 self.klass.create_permissions()
94 self.klass.create_permissions()
95 self.klass.fixup_groups()
95 self.klass.fixup_groups()
96
96
97 def step_5(self):
97 def step_5(self):
98 pass
98 pass
99
99
100 def step_6(self):
100 def step_6(self):
101
101
102 notify('re-checking permissions')
102 notify('re-checking permissions')
103 self.klass.create_permissions()
103 self.klass.create_permissions()
104
104
105 notify('installing new UI options')
105 notify('installing new UI options')
106 sett4 = RhodeCodeSetting('show_public_icon', True)
106 sett4 = RhodeCodeSetting('show_public_icon', True)
107 Session().add(sett4)
107 Session().add(sett4)
108 sett5 = RhodeCodeSetting('show_private_icon', True)
108 sett5 = RhodeCodeSetting('show_private_icon', True)
109 Session().add(sett5)
109 Session().add(sett5)
110 sett6 = RhodeCodeSetting('stylify_metatags', False)
110 sett6 = RhodeCodeSetting('stylify_metatags', False)
111 Session().add(sett6)
111 Session().add(sett6)
112
112
113 notify('fixing old PULL hook')
113 notify('fixing old PULL hook')
114 _pull = RhodeCodeUi.get_by_key('preoutgoing.pull_logger')
114 _pull = RhodeCodeUi.get_by_key('preoutgoing.pull_logger')
115 if _pull:
115 if _pull:
116 _pull.ui_key = RhodeCodeUi.HOOK_PULL
116 _pull.ui_key = RhodeCodeUi.HOOK_PULL
117 Session().add(_pull)
117 Session().add(_pull)
118
118
119 notify('fixing old PUSH hook')
119 notify('fixing old PUSH hook')
120 _push = RhodeCodeUi.get_by_key('pretxnchangegroup.push_logger')
120 _push = RhodeCodeUi.get_by_key('pretxnchangegroup.push_logger')
121 if _push:
121 if _push:
122 _push.ui_key = RhodeCodeUi.HOOK_PUSH
122 _push.ui_key = RhodeCodeUi.HOOK_PUSH
123 Session().add(_push)
123 Session().add(_push)
124
124
125 notify('installing new pre-push hook')
125 notify('installing new pre-push hook')
126 hooks4 = RhodeCodeUi()
126 hooks4 = RhodeCodeUi()
127 hooks4.ui_section = 'hooks'
127 hooks4.ui_section = 'hooks'
128 hooks4.ui_key = RhodeCodeUi.HOOK_PRE_PUSH
128 hooks4.ui_key = RhodeCodeUi.HOOK_PRE_PUSH
129 hooks4.ui_value = 'python:rhodecode.lib.hooks.pre_push'
129 hooks4.ui_value = 'python:rhodecode.lib.hooks.pre_push'
130 Session().add(hooks4)
130 Session().add(hooks4)
131
131
132 notify('installing new pre-pull hook')
132 notify('installing new pre-pull hook')
133 hooks6 = RhodeCodeUi()
133 hooks6 = RhodeCodeUi()
134 hooks6.ui_section = 'hooks'
134 hooks6.ui_section = 'hooks'
135 hooks6.ui_key = RhodeCodeUi.HOOK_PRE_PULL
135 hooks6.ui_key = RhodeCodeUi.HOOK_PRE_PULL
136 hooks6.ui_value = 'python:rhodecode.lib.hooks.pre_pull'
136 hooks6.ui_value = 'python:rhodecode.lib.hooks.pre_pull'
137 Session().add(hooks6)
137 Session().add(hooks6)
138
138
139 notify('installing hgsubversion option')
139 notify('installing hgsubversion option')
140 # enable hgsubversion disabled by default
140 # enable hgsubversion disabled by default
141 hgsubversion = RhodeCodeUi()
141 hgsubversion = RhodeCodeUi()
142 hgsubversion.ui_section = 'extensions'
142 hgsubversion.ui_section = 'extensions'
143 hgsubversion.ui_key = 'hgsubversion'
143 hgsubversion.ui_key = 'hgsubversion'
144 hgsubversion.ui_value = ''
144 hgsubversion.ui_value = ''
145 hgsubversion.ui_active = False
145 hgsubversion.ui_active = False
146 Session().add(hgsubversion)
146 Session().add(hgsubversion)
147
147
148 notify('installing hg git option')
148 notify('installing hg git option')
149 # enable hggit disabled by default
149 # enable hggit disabled by default
150 hggit = RhodeCodeUi()
150 hggit = RhodeCodeUi()
151 hggit.ui_section = 'extensions'
151 hggit.ui_section = 'extensions'
152 hggit.ui_key = 'hggit'
152 hggit.ui_key = 'hggit'
153 hggit.ui_value = ''
153 hggit.ui_value = ''
154 hggit.ui_active = False
154 hggit.ui_active = False
155 Session().add(hggit)
155 Session().add(hggit)
156
156
157 notify('re-check default permissions')
157 notify('re-check default permissions')
158 default_user = User.get_by_username(User.DEFAULT_USER)
158 default_user = User.get_by_username(User.DEFAULT_USER)
159 perm = Permission.get_by_key('hg.fork.repository')
159 perm = Permission.get_by_key('hg.fork.repository')
160 reg_perm = UserToPerm()
160 reg_perm = UserToPerm()
161 reg_perm.user = default_user
161 reg_perm.user = default_user
162 reg_perm.permission = perm
162 reg_perm.permission = perm
163 Session().add(reg_perm)
163 Session().add(reg_perm)
164
164
165 def step_7(self):
165 def step_7(self):
166 perm_fixes = self.klass.reset_permissions(User.DEFAULT_USER)
166 perm_fixes = self.klass.reset_permissions(User.DEFAULT_USER)
167 Session().commit()
167 Session().commit()
168 if perm_fixes:
168 if perm_fixes:
169 notify('There was an inconsistent state of permissions '
169 notify('There was an inconsistent state of permissions '
170 'detected for default user. Permissions are now '
170 'detected for default user. Permissions are now '
171 'reset to the default value for default user. '
171 'reset to the default value for default user. '
172 'Please validate and check default permissions '
172 'Please validate and check default permissions '
173 'in admin panel')
173 'in admin panel')
174
174
175 def step_8(self):
175 def step_8(self):
176 self.klass.create_permissions()
176 self.klass.create_permissions()
177 self.klass.populate_default_permissions()
177 self.klass.populate_default_permissions()
178 self.klass.create_default_options(skip_existing=True)
178 self.klass.create_default_options(skip_existing=True)
179 Session().commit()
179 Session().commit()
180
180
181 def step_9(self):
181 def step_9(self):
182 pass
182 pass
183
183
184 def step_10(self):
184 def step_10(self):
185 pass
185 pass
186
186
187 def step_11(self):
187 def step_11(self):
188 self.klass.update_repo_info()
188 self.klass.update_repo_info()
189
189
190 def step_12(self):
190 def step_12(self):
191 self.klass.create_permissions()
191 self.klass.create_permissions()
192 Session().commit()
192 Session().commit()
193
193
194 self.klass.populate_default_permissions()
194 self.klass.populate_default_permissions()
195 Session().commit()
195 Session().commit()
196
196
197 #fix all usergroups
197 #fix all usergroups
198 ug_model = UserGroupModel()
198 ug_model = UserGroupModel()
199 for ug in UserGroup.get_all():
199 for ug in UserGroup.get_all():
200 perm_obj = ug_model._create_default_perms(ug)
200 perm_obj = ug_model._create_default_perms(ug)
201 Session().add(perm_obj)
201 Session().add(perm_obj)
202 Session().commit()
202 Session().commit()
203
203
204 adm = User.get_first_admin()
204 adm = User.get_first_admin()
205 # fix owners of UserGroup
205 # fix owners of UserGroup
206 for ug in Session().query(UserGroup).all():
206 for ug in Session().query(UserGroup).all():
207 ug.user_id = adm.user_id
207 ug.user_id = adm.user_id
208 Session().add(ug)
208 Session().add(ug)
209 Session().commit()
209 Session().commit()
210
210
211 # fix owners of RepoGroup
211 # fix owners of RepoGroup
212 for ug in Session().query(RepoGroup).all():
212 for ug in Session().query(RepoGroup).all():
213 ug.user_id = adm.user_id
213 ug.user_id = adm.user_id
214 Session().add(ug)
214 Session().add(ug)
215 Session().commit()
215 Session().commit()
216
216
217 def step_13(self):
217 def step_13(self):
218 pass
218 pass
219
219
220 def step_14(self):
220 def step_14(self):
221 # fix nullable columns on last_update
221 # fix nullable columns on last_update
222 for r in RepoModel().get_all():
222 for r in RepoModel().get_all():
223 if r.updated_on is None:
223 if r.updated_on is None:
224 r.updated_on = datetime.datetime.fromtimestamp(0)
224 r.updated_on = datetime.datetime.fromtimestamp(0)
225 Session().add(r)
225 Session().add(r)
226 Session().commit()
226 Session().commit()
227
227
228 class DbManage(object):
228 class DbManage(object):
229 def __init__(self, log_sql, dbconf, root, tests=False, cli_args={}):
229 def __init__(self, log_sql, dbconf, root, tests=False, cli_args={}):
230 self.dbname = dbconf.split('/')[-1]
230 self.dbname = dbconf.split('/')[-1]
231 self.tests = tests
231 self.tests = tests
232 self.root = root
232 self.root = root
233 self.dburi = dbconf
233 self.dburi = dbconf
234 self.log_sql = log_sql
234 self.log_sql = log_sql
235 self.db_exists = False
235 self.db_exists = False
236 self.cli_args = cli_args
236 self.cli_args = cli_args
237 self.init_db()
237 self.init_db()
238
238
239 force_ask = self.cli_args.get('force_ask')
239 force_ask = self.cli_args.get('force_ask')
240 if force_ask is not None:
240 if force_ask is not None:
241 global ask_ok
241 global ask_ok
242 ask_ok = lambda *args, **kwargs: force_ask
242 ask_ok = lambda *args, **kwargs: force_ask
243
243
244 def init_db(self):
244 def init_db(self):
245 engine = create_engine(self.dburi, echo=self.log_sql)
245 engine = create_engine(self.dburi, echo=self.log_sql)
246 init_model(engine)
246 init_model(engine)
247 self.sa = Session()
247 self.sa = Session()
248
248
249 def create_tables(self, override=False):
249 def create_tables(self, override=False):
250 """
250 """
251 Create a auth database
251 Create a auth database
252 """
252 """
253
253
254 log.info("Any existing database is going to be destroyed")
254 log.info("Any existing database is going to be destroyed")
255 if self.tests:
255 if self.tests:
256 destroy = True
256 destroy = True
257 else:
257 else:
258 destroy = ask_ok('Are you sure to destroy old database ? [y/n]')
258 destroy = ask_ok('Are you sure to destroy old database ? [y/n]')
259 if not destroy:
259 if not destroy:
260 sys.exit('Nothing tables created')
260 sys.exit('Nothing tables created')
261 if destroy:
261 if destroy:
262 Base.metadata.drop_all()
262 Base.metadata.drop_all()
263
263
264 checkfirst = not override
264 checkfirst = not override
265 Base.metadata.create_all(checkfirst=checkfirst)
265 Base.metadata.create_all(checkfirst=checkfirst)
266 log.info('Created tables for %s' % self.dbname)
266 log.info('Created tables for %s' % self.dbname)
267
267
268 def set_db_version(self):
268 def set_db_version(self):
269 ver = DbMigrateVersion()
269 ver = DbMigrateVersion()
270 ver.version = __dbversion__
270 ver.version = __dbversion__
271 ver.repository_id = 'rhodecode_db_migrations'
271 ver.repository_id = 'rhodecode_db_migrations'
272 ver.repository_path = 'versions'
272 ver.repository_path = 'versions'
273 self.sa.add(ver)
273 self.sa.add(ver)
274 log.info('db version set to: %s' % __dbversion__)
274 log.info('db version set to: %s' % __dbversion__)
275
275
276 def upgrade(self):
276 def upgrade(self):
277 """
277 """
278 Upgrades given database schema to given revision following
278 Upgrades given database schema to given revision following
279 all needed steps, to perform the upgrade
279 all needed steps, to perform the upgrade
280
280
281 """
281 """
282
282
283 from rhodecode.lib.dbmigrate.migrate.versioning import api
283 from rhodecode.lib.dbmigrate.migrate.versioning import api
284 from rhodecode.lib.dbmigrate.migrate.exceptions import \
284 from rhodecode.lib.dbmigrate.migrate.exceptions import \
285 DatabaseNotControlledError
285 DatabaseNotControlledError
286
286
287 if 'sqlite' in self.dburi:
287 if 'sqlite' in self.dburi:
288 print (
288 print (
289 '********************** WARNING **********************\n'
289 '********************** WARNING **********************\n'
290 'Make sure your version of sqlite is at least 3.7.X. \n'
290 'Make sure your version of sqlite is at least 3.7.X. \n'
291 'Earlier versions are known to fail on some migrations\n'
291 'Earlier versions are known to fail on some migrations\n'
292 '*****************************************************\n'
292 '*****************************************************\n'
293 )
293 )
294 upgrade = ask_ok('You are about to perform database upgrade, make '
294 upgrade = ask_ok('You are about to perform database upgrade, make '
295 'sure You backed up your database before. '
295 'sure You backed up your database before. '
296 'Continue ? [y/n]')
296 'Continue ? [y/n]')
297 if not upgrade:
297 if not upgrade:
298 sys.exit('No upgrade performed')
298 sys.exit('No upgrade performed')
299
299
300 repository_path = jn(dn(dn(dn(os.path.realpath(__file__)))),
300 repository_path = jn(dn(dn(dn(os.path.realpath(__file__)))),
301 'rhodecode/lib/dbmigrate')
301 'rhodecode/lib/dbmigrate')
302 db_uri = self.dburi
302 db_uri = self.dburi
303
303
304 try:
304 try:
305 curr_version = api.db_version(db_uri, repository_path)
305 curr_version = api.db_version(db_uri, repository_path)
306 msg = ('Found current database under version'
306 msg = ('Found current database under version'
307 ' control with version %s' % curr_version)
307 ' control with version %s' % curr_version)
308
308
309 except (RuntimeError, DatabaseNotControlledError):
309 except (RuntimeError, DatabaseNotControlledError):
310 curr_version = 1
310 curr_version = 1
311 msg = ('Current database is not under version control. Setting'
311 msg = ('Current database is not under version control. Setting'
312 ' as version %s' % curr_version)
312 ' as version %s' % curr_version)
313 api.version_control(db_uri, repository_path, curr_version)
313 api.version_control(db_uri, repository_path, curr_version)
314
314
315 notify(msg)
315 notify(msg)
316
316
317 if curr_version == __dbversion__:
317 if curr_version == __dbversion__:
318 sys.exit('This database is already at the newest version')
318 sys.exit('This database is already at the newest version')
319
319
320 # clear cache keys
320 # clear cache keys
321 log.info("Clearing cache keys now...")
321 log.info("Clearing cache keys now...")
322 CacheInvalidation.clear_cache()
322 CacheInvalidation.clear_cache()
323
323
324 upgrade_steps = range(curr_version + 1, __dbversion__ + 1)
324 upgrade_steps = range(curr_version + 1, __dbversion__ + 1)
325 notify('attempting to do database upgrade from '
325 notify('attempting to do database upgrade from '
326 'version %s to version %s' % (curr_version, __dbversion__))
326 'version %s to version %s' % (curr_version, __dbversion__))
327
327
328 # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
328 # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
329 _step = None
329 _step = None
330 for step in upgrade_steps:
330 for step in upgrade_steps:
331 notify('performing upgrade step %s' % step)
331 notify('performing upgrade step %s' % step)
332 time.sleep(2)
332 time.sleep(2)
333
333
334 api.upgrade(db_uri, repository_path, step)
334 api.upgrade(db_uri, repository_path, step)
335 notify('schema upgrade for step %s completed' % (step,))
335 notify('schema upgrade for step %s completed' % (step,))
336
336
337 fixture = 'step_%s' % step
337 fixture = 'step_%s' % step
338 notify('performing fixture step %s' % fixture)
338 notify('performing fixture step %s' % fixture)
339 getattr(UpgradeSteps(self), fixture)()
339 getattr(UpgradeSteps(self), fixture)()
340 self.sa.commit()
340 self.sa.commit()
341 notify('fixture %s completed' % (fixture,))
341 notify('fixture %s completed' % (fixture,))
342 _step = step
342 _step = step
343
343
344 notify('upgrade to version %s successful' % _step)
344 notify('upgrade to version %s successful' % _step)
345
345
346 def fix_repo_paths(self):
346 def fix_repo_paths(self):
347 """
347 """
348 Fixes a old rhodecode version path into new one without a '*'
348 Fixes a old rhodecode version path into new one without a '*'
349 """
349 """
350
350
351 paths = self.sa.query(RhodeCodeUi)\
351 paths = self.sa.query(RhodeCodeUi)\
352 .filter(RhodeCodeUi.ui_key == '/')\
352 .filter(RhodeCodeUi.ui_key == '/')\
353 .scalar()
353 .scalar()
354
354
355 paths.ui_value = paths.ui_value.replace('*', '')
355 paths.ui_value = paths.ui_value.replace('*', '')
356
356
357 try:
357 try:
358 self.sa.add(paths)
358 self.sa.add(paths)
359 self.sa.commit()
359 self.sa.commit()
360 except Exception:
360 except Exception:
361 self.sa.rollback()
361 self.sa.rollback()
362 raise
362 raise
363
363
364 def fix_default_user(self):
364 def fix_default_user(self):
365 """
365 """
366 Fixes a old default user with some 'nicer' default values,
366 Fixes a old default user with some 'nicer' default values,
367 used mostly for anonymous access
367 used mostly for anonymous access
368 """
368 """
369 def_user = self.sa.query(User)\
369 def_user = self.sa.query(User)\
370 .filter(User.username == 'default')\
370 .filter(User.username == 'default')\
371 .one()
371 .one()
372
372
373 def_user.name = 'Anonymous'
373 def_user.name = 'Anonymous'
374 def_user.lastname = 'User'
374 def_user.lastname = 'User'
375 def_user.email = 'anonymous@rhodecode.org'
375 def_user.email = 'anonymous@rhodecode.org'
376
376
377 try:
377 try:
378 self.sa.add(def_user)
378 self.sa.add(def_user)
379 self.sa.commit()
379 self.sa.commit()
380 except Exception:
380 except Exception:
381 self.sa.rollback()
381 self.sa.rollback()
382 raise
382 raise
383
383
384 def fix_settings(self):
384 def fix_settings(self):
385 """
385 """
386 Fixes rhodecode settings adds ga_code key for google analytics
386 Fixes rhodecode settings adds ga_code key for google analytics
387 """
387 """
388
388
389 hgsettings3 = RhodeCodeSetting('ga_code', '')
389 hgsettings3 = RhodeCodeSetting('ga_code', '')
390
390
391 try:
391 try:
392 self.sa.add(hgsettings3)
392 self.sa.add(hgsettings3)
393 self.sa.commit()
393 self.sa.commit()
394 except Exception:
394 except Exception:
395 self.sa.rollback()
395 self.sa.rollback()
396 raise
396 raise
397
397
398 def admin_prompt(self, second=False):
398 def admin_prompt(self, second=False):
399 if not self.tests:
399 if not self.tests:
400 import getpass
400 import getpass
401
401
402 # defaults
402 # defaults
403 defaults = self.cli_args
403 defaults = self.cli_args
404 username = defaults.get('username')
404 username = defaults.get('username')
405 password = defaults.get('password')
405 password = defaults.get('password')
406 email = defaults.get('email')
406 email = defaults.get('email')
407
407
408 def get_password():
408 def get_password():
409 password = getpass.getpass('Specify admin password '
409 password = getpass.getpass('Specify admin password '
410 '(min 6 chars):')
410 '(min 6 chars):')
411 confirm = getpass.getpass('Confirm password:')
411 confirm = getpass.getpass('Confirm password:')
412
412
413 if password != confirm:
413 if password != confirm:
414 log.error('passwords mismatch')
414 log.error('passwords mismatch')
415 return False
415 return False
416 if len(password) < 6:
416 if len(password) < 6:
417 log.error('password is to short use at least 6 characters')
417 log.error('password is to short use at least 6 characters')
418 return False
418 return False
419
419
420 return password
420 return password
421 if username is None:
421 if username is None:
422 username = raw_input('Specify admin username:')
422 username = raw_input('Specify admin username:')
423 if password is None:
423 if password is None:
424 password = get_password()
424 password = get_password()
425 if not password:
425 if not password:
426 #second try
426 #second try
427 password = get_password()
427 password = get_password()
428 if not password:
428 if not password:
429 sys.exit()
429 sys.exit()
430 if email is None:
430 if email is None:
431 email = raw_input('Specify admin email:')
431 email = raw_input('Specify admin email:')
432 self.create_user(username, password, email, True)
432 self.create_user(username, password, email, True)
433 else:
433 else:
434 log.info('creating admin and regular test users')
434 log.info('creating admin and regular test users')
435 from rhodecode.tests import TEST_USER_ADMIN_LOGIN, \
435 from rhodecode.tests import TEST_USER_ADMIN_LOGIN, \
436 TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, \
436 TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, \
437 TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, \
437 TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, \
438 TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR2_LOGIN, \
438 TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR2_LOGIN, \
439 TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL
439 TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL
440
440
441 self.create_user(TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
441 self.create_user(TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
442 TEST_USER_ADMIN_EMAIL, True)
442 TEST_USER_ADMIN_EMAIL, True)
443
443
444 self.create_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
444 self.create_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
445 TEST_USER_REGULAR_EMAIL, False)
445 TEST_USER_REGULAR_EMAIL, False)
446
446
447 self.create_user(TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS,
447 self.create_user(TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS,
448 TEST_USER_REGULAR2_EMAIL, False)
448 TEST_USER_REGULAR2_EMAIL, False)
449
449
450 def create_ui_settings(self):
450 def create_ui_settings(self):
451 """
451 """
452 Creates ui settings, fills out hooks
452 Creates ui settings, fills out hooks
453 and disables dotencode
453 and disables dotencode
454 """
454 """
455
455
456 #HOOKS
456 #HOOKS
457 hooks1_key = RhodeCodeUi.HOOK_UPDATE
457 hooks1_key = RhodeCodeUi.HOOK_UPDATE
458 hooks1_ = self.sa.query(RhodeCodeUi)\
458 hooks1_ = self.sa.query(RhodeCodeUi)\
459 .filter(RhodeCodeUi.ui_key == hooks1_key).scalar()
459 .filter(RhodeCodeUi.ui_key == hooks1_key).scalar()
460
460
461 hooks1 = RhodeCodeUi() if hooks1_ is None else hooks1_
461 hooks1 = RhodeCodeUi() if hooks1_ is None else hooks1_
462 hooks1.ui_section = 'hooks'
462 hooks1.ui_section = 'hooks'
463 hooks1.ui_key = hooks1_key
463 hooks1.ui_key = hooks1_key
464 hooks1.ui_value = 'hg update >&2'
464 hooks1.ui_value = 'hg update >&2'
465 hooks1.ui_active = False
465 hooks1.ui_active = False
466 self.sa.add(hooks1)
466 self.sa.add(hooks1)
467
467
468 hooks2_key = RhodeCodeUi.HOOK_REPO_SIZE
468 hooks2_key = RhodeCodeUi.HOOK_REPO_SIZE
469 hooks2_ = self.sa.query(RhodeCodeUi)\
469 hooks2_ = self.sa.query(RhodeCodeUi)\
470 .filter(RhodeCodeUi.ui_key == hooks2_key).scalar()
470 .filter(RhodeCodeUi.ui_key == hooks2_key).scalar()
471 hooks2 = RhodeCodeUi() if hooks2_ is None else hooks2_
471 hooks2 = RhodeCodeUi() if hooks2_ is None else hooks2_
472 hooks2.ui_section = 'hooks'
472 hooks2.ui_section = 'hooks'
473 hooks2.ui_key = hooks2_key
473 hooks2.ui_key = hooks2_key
474 hooks2.ui_value = 'python:rhodecode.lib.hooks.repo_size'
474 hooks2.ui_value = 'python:rhodecode.lib.hooks.repo_size'
475 self.sa.add(hooks2)
475 self.sa.add(hooks2)
476
476
477 hooks3 = RhodeCodeUi()
477 hooks3 = RhodeCodeUi()
478 hooks3.ui_section = 'hooks'
478 hooks3.ui_section = 'hooks'
479 hooks3.ui_key = RhodeCodeUi.HOOK_PUSH
479 hooks3.ui_key = RhodeCodeUi.HOOK_PUSH
480 hooks3.ui_value = 'python:rhodecode.lib.hooks.log_push_action'
480 hooks3.ui_value = 'python:rhodecode.lib.hooks.log_push_action'
481 self.sa.add(hooks3)
481 self.sa.add(hooks3)
482
482
483 hooks4 = RhodeCodeUi()
483 hooks4 = RhodeCodeUi()
484 hooks4.ui_section = 'hooks'
484 hooks4.ui_section = 'hooks'
485 hooks4.ui_key = RhodeCodeUi.HOOK_PRE_PUSH
485 hooks4.ui_key = RhodeCodeUi.HOOK_PRE_PUSH
486 hooks4.ui_value = 'python:rhodecode.lib.hooks.pre_push'
486 hooks4.ui_value = 'python:rhodecode.lib.hooks.pre_push'
487 self.sa.add(hooks4)
487 self.sa.add(hooks4)
488
488
489 hooks5 = RhodeCodeUi()
489 hooks5 = RhodeCodeUi()
490 hooks5.ui_section = 'hooks'
490 hooks5.ui_section = 'hooks'
491 hooks5.ui_key = RhodeCodeUi.HOOK_PULL
491 hooks5.ui_key = RhodeCodeUi.HOOK_PULL
492 hooks5.ui_value = 'python:rhodecode.lib.hooks.log_pull_action'
492 hooks5.ui_value = 'python:rhodecode.lib.hooks.log_pull_action'
493 self.sa.add(hooks5)
493 self.sa.add(hooks5)
494
494
495 hooks6 = RhodeCodeUi()
495 hooks6 = RhodeCodeUi()
496 hooks6.ui_section = 'hooks'
496 hooks6.ui_section = 'hooks'
497 hooks6.ui_key = RhodeCodeUi.HOOK_PRE_PULL
497 hooks6.ui_key = RhodeCodeUi.HOOK_PRE_PULL
498 hooks6.ui_value = 'python:rhodecode.lib.hooks.pre_pull'
498 hooks6.ui_value = 'python:rhodecode.lib.hooks.pre_pull'
499 self.sa.add(hooks6)
499 self.sa.add(hooks6)
500
500
501 # enable largefiles
501 # enable largefiles
502 largefiles = RhodeCodeUi()
502 largefiles = RhodeCodeUi()
503 largefiles.ui_section = 'extensions'
503 largefiles.ui_section = 'extensions'
504 largefiles.ui_key = 'largefiles'
504 largefiles.ui_key = 'largefiles'
505 largefiles.ui_value = ''
505 largefiles.ui_value = ''
506 self.sa.add(largefiles)
506 self.sa.add(largefiles)
507
507
508 # enable hgsubversion disabled by default
508 # enable hgsubversion disabled by default
509 hgsubversion = RhodeCodeUi()
509 hgsubversion = RhodeCodeUi()
510 hgsubversion.ui_section = 'extensions'
510 hgsubversion.ui_section = 'extensions'
511 hgsubversion.ui_key = 'hgsubversion'
511 hgsubversion.ui_key = 'hgsubversion'
512 hgsubversion.ui_value = ''
512 hgsubversion.ui_value = ''
513 hgsubversion.ui_active = False
513 hgsubversion.ui_active = False
514 self.sa.add(hgsubversion)
514 self.sa.add(hgsubversion)
515
515
516 # enable hggit disabled by default
516 # enable hggit disabled by default
517 hggit = RhodeCodeUi()
517 hggit = RhodeCodeUi()
518 hggit.ui_section = 'extensions'
518 hggit.ui_section = 'extensions'
519 hggit.ui_key = 'hggit'
519 hggit.ui_key = 'hggit'
520 hggit.ui_value = ''
520 hggit.ui_value = ''
521 hggit.ui_active = False
521 hggit.ui_active = False
522 self.sa.add(hggit)
522 self.sa.add(hggit)
523
523
524 def create_ldap_options(self, skip_existing=False):
524 def create_ldap_options(self, skip_existing=False):
525 """Creates ldap settings"""
525 """Creates ldap settings"""
526
526
527 for k, v in [('ldap_active', 'false'), ('ldap_host', ''),
527 for k, v in [('ldap_active', 'false'), ('ldap_host', ''),
528 ('ldap_port', '389'), ('ldap_tls_kind', 'PLAIN'),
528 ('ldap_port', '389'), ('ldap_tls_kind', 'PLAIN'),
529 ('ldap_tls_reqcert', ''), ('ldap_dn_user', ''),
529 ('ldap_tls_reqcert', ''), ('ldap_dn_user', ''),
530 ('ldap_dn_pass', ''), ('ldap_base_dn', ''),
530 ('ldap_dn_pass', ''), ('ldap_base_dn', ''),
531 ('ldap_filter', ''), ('ldap_search_scope', ''),
531 ('ldap_filter', ''), ('ldap_search_scope', ''),
532 ('ldap_attr_login', ''), ('ldap_attr_firstname', ''),
532 ('ldap_attr_login', ''), ('ldap_attr_firstname', ''),
533 ('ldap_attr_lastname', ''), ('ldap_attr_email', '')]:
533 ('ldap_attr_lastname', ''), ('ldap_attr_email', '')]:
534
534
535 if skip_existing and RhodeCodeSetting.get_by_name(k) is not None:
535 if skip_existing and RhodeCodeSetting.get_by_name(k) is not None:
536 log.debug('Skipping option %s' % k)
536 log.debug('Skipping option %s' % k)
537 continue
537 continue
538 setting = RhodeCodeSetting(k, v)
538 setting = RhodeCodeSetting(k, v)
539 self.sa.add(setting)
539 self.sa.add(setting)
540
540
541 def create_default_options(self, skip_existing=False):
541 def create_default_options(self, skip_existing=False):
542 """Creates default settings"""
542 """Creates default settings"""
543
543
544 for k, v in [
544 for k, v in [
545 ('default_repo_enable_locking', False),
545 ('default_repo_enable_locking', False),
546 ('default_repo_enable_downloads', False),
546 ('default_repo_enable_downloads', False),
547 ('default_repo_enable_statistics', False),
547 ('default_repo_enable_statistics', False),
548 ('default_repo_private', False),
548 ('default_repo_private', False),
549 ('default_repo_type', 'hg')]:
549 ('default_repo_type', 'hg')]:
550
550
551 if skip_existing and RhodeCodeSetting.get_by_name(k) is not None:
551 if skip_existing and RhodeCodeSetting.get_by_name(k) is not None:
552 log.debug('Skipping option %s' % k)
552 log.debug('Skipping option %s' % k)
553 continue
553 continue
554 setting = RhodeCodeSetting(k, v)
554 setting = RhodeCodeSetting(k, v)
555 self.sa.add(setting)
555 self.sa.add(setting)
556
556
557 def fixup_groups(self):
557 def fixup_groups(self):
558 def_usr = User.get_default_user()
558 def_usr = User.get_default_user()
559 for g in RepoGroup.query().all():
559 for g in RepoGroup.query().all():
560 g.group_name = g.get_new_name(g.name)
560 g.group_name = g.get_new_name(g.name)
561 self.sa.add(g)
561 self.sa.add(g)
562 # get default perm
562 # get default perm
563 default = UserRepoGroupToPerm.query()\
563 default = UserRepoGroupToPerm.query()\
564 .filter(UserRepoGroupToPerm.group == g)\
564 .filter(UserRepoGroupToPerm.group == g)\
565 .filter(UserRepoGroupToPerm.user == def_usr)\
565 .filter(UserRepoGroupToPerm.user == def_usr)\
566 .scalar()
566 .scalar()
567
567
568 if default is None:
568 if default is None:
569 log.debug('missing default permission for group %s adding' % g)
569 log.debug('missing default permission for group %s adding' % g)
570 perm_obj = ReposGroupModel()._create_default_perms(g)
570 perm_obj = ReposGroupModel()._create_default_perms(g)
571 self.sa.add(perm_obj)
571 self.sa.add(perm_obj)
572
572
573 def reset_permissions(self, username):
573 def reset_permissions(self, username):
574 """
574 """
575 Resets permissions to default state, usefull when old systems had
575 Resets permissions to default state, usefull when old systems had
576 bad permissions, we must clean them up
576 bad permissions, we must clean them up
577
577
578 :param username:
578 :param username:
579 """
579 """
580 default_user = User.get_by_username(username)
580 default_user = User.get_by_username(username)
581 if not default_user:
581 if not default_user:
582 return
582 return
583
583
584 u2p = UserToPerm.query()\
584 u2p = UserToPerm.query()\
585 .filter(UserToPerm.user == default_user).all()
585 .filter(UserToPerm.user == default_user).all()
586 fixed = False
586 fixed = False
587 if len(u2p) != len(Permission.DEFAULT_USER_PERMISSIONS):
587 if len(u2p) != len(Permission.DEFAULT_USER_PERMISSIONS):
588 for p in u2p:
588 for p in u2p:
589 Session().delete(p)
589 Session().delete(p)
590 fixed = True
590 fixed = True
591 self.populate_default_permissions()
591 self.populate_default_permissions()
592 return fixed
592 return fixed
593
593
594 def update_repo_info(self):
594 def update_repo_info(self):
595 RepoModel.update_repoinfo()
595 RepoModel.update_repoinfo()
596
596
597 def config_prompt(self, test_repo_path='', retries=3):
597 def config_prompt(self, test_repo_path='', retries=3):
598 defaults = self.cli_args
598 defaults = self.cli_args
599 _path = defaults.get('repos_location')
599 _path = defaults.get('repos_location')
600 if retries == 3:
600 if retries == 3:
601 log.info('Setting up repositories config')
601 log.info('Setting up repositories config')
602
602
603 if _path is not None:
603 if _path is not None:
604 path = _path
604 path = _path
605 elif not self.tests and not test_repo_path:
605 elif not self.tests and not test_repo_path:
606 path = raw_input(
606 path = raw_input(
607 'Enter a valid absolute path to store repositories. '
607 'Enter a valid absolute path to store repositories. '
608 'All repositories in that path will be added automatically:'
608 'All repositories in that path will be added automatically:'
609 )
609 )
610 else:
610 else:
611 path = test_repo_path
611 path = test_repo_path
612 path_ok = True
612 path_ok = True
613
613
614 # check proper dir
614 # check proper dir
615 if not os.path.isdir(path):
615 if not os.path.isdir(path):
616 path_ok = False
616 path_ok = False
617 log.error('Given path %s is not a valid directory' % path)
617 log.error('Given path %s is not a valid directory' % (path,))
618
618
619 elif not os.path.isabs(path):
619 elif not os.path.isabs(path):
620 path_ok = False
620 path_ok = False
621 log.error('Given path %s is not an absolute path' % path)
621 log.error('Given path %s is not an absolute path' % (path,))
622
623 # check if path is at least readable.
624 if not os.access(path, os.R_OK):
625 path_ok = False
626 log.error('Given path %s is not readable' % (path,))
622
627
623 # check write access
628 # check write access, warn user about non writeable paths
624 elif not os.access(path, os.W_OK) and path_ok:
629 elif not os.access(path, os.W_OK) and path_ok:
625 path_ok = False
630 log.warn('No write permission to given path %s' % (path,))
626 log.error('No write permission to given path %s' % path)
631 if not ask_ok('Given path %s is not writeable, do you want to '
632 'continue with read only mode ? [y/n]' % (path,)):
633 log.error('Canceled by user')
634 sys.exit(-1)
627
635
628 if retries == 0:
636 if retries == 0:
629 sys.exit('max retries reached')
637 sys.exit('max retries reached')
630 if not path_ok:
638 if not path_ok:
631 retries -= 1
639 retries -= 1
632 return self.config_prompt(test_repo_path, retries)
640 return self.config_prompt(test_repo_path, retries)
633
641
634 real_path = os.path.normpath(os.path.realpath(path))
642 real_path = os.path.normpath(os.path.realpath(path))
635
643
636 if real_path != os.path.normpath(path):
644 if real_path != os.path.normpath(path):
637 if not ask_ok(('Path looks like a symlink, Rhodecode will store '
645 if not ask_ok(('Path looks like a symlink, Rhodecode will store '
638 'given path as %s ? [y/n]') % (real_path)):
646 'given path as %s ? [y/n]') % (real_path,)):
639 log.error('Canceled by user')
647 log.error('Canceled by user')
640 sys.exit(-1)
648 sys.exit(-1)
641
649
642 return real_path
650 return real_path
643
651
644 def create_settings(self, path):
652 def create_settings(self, path):
645
653
646 self.create_ui_settings()
654 self.create_ui_settings()
647
655
648 ui_config = [
656 ui_config = [
649 ('web', 'push_ssl', 'false'),
657 ('web', 'push_ssl', 'false'),
650 ('web', 'allow_archive', 'gz zip bz2'),
658 ('web', 'allow_archive', 'gz zip bz2'),
651 ('web', 'allow_push', '*'),
659 ('web', 'allow_push', '*'),
652 ('web', 'baseurl', '/'),
660 ('web', 'baseurl', '/'),
653 ('paths', '/', path),
661 ('paths', '/', path),
654 #('phases', 'publish', 'false')
662 #('phases', 'publish', 'false')
655 ]
663 ]
656 for section, key, value in ui_config:
664 for section, key, value in ui_config:
657 ui_conf = RhodeCodeUi()
665 ui_conf = RhodeCodeUi()
658 setattr(ui_conf, 'ui_section', section)
666 setattr(ui_conf, 'ui_section', section)
659 setattr(ui_conf, 'ui_key', key)
667 setattr(ui_conf, 'ui_key', key)
660 setattr(ui_conf, 'ui_value', value)
668 setattr(ui_conf, 'ui_value', value)
661 self.sa.add(ui_conf)
669 self.sa.add(ui_conf)
662
670
663 settings = [
671 settings = [
664 ('realm', 'RhodeCode authentication', unicode),
672 ('realm', 'RhodeCode authentication', unicode),
665 ('title', 'RhodeCode', unicode),
673 ('title', 'RhodeCode', unicode),
666 ('ga_code', '', unicode),
674 ('ga_code', '', unicode),
667 ('show_public_icon', True, bool),
675 ('show_public_icon', True, bool),
668 ('show_private_icon', True, bool),
676 ('show_private_icon', True, bool),
669 ('stylify_metatags', False, bool),
677 ('stylify_metatags', False, bool),
670 ('dashboard_items', 100, int),
678 ('dashboard_items', 100, int),
671 ('show_version', True, bool)
679 ('show_version', True, bool)
672 ]
680 ]
673 for key, val, type_ in settings:
681 for key, val, type_ in settings:
674 sett = RhodeCodeSetting(key, val)
682 sett = RhodeCodeSetting(key, val)
675 self.sa.add(sett)
683 self.sa.add(sett)
676
684
677 self.create_ldap_options()
685 self.create_ldap_options()
678 self.create_default_options()
686 self.create_default_options()
679
687
680 log.info('created ui config')
688 log.info('created ui config')
681
689
682 def create_user(self, username, password, email='', admin=False):
690 def create_user(self, username, password, email='', admin=False):
683 log.info('creating user %s' % username)
691 log.info('creating user %s' % username)
684 UserModel().create_or_update(username, password, email,
692 UserModel().create_or_update(username, password, email,
685 firstname='RhodeCode', lastname='Admin',
693 firstname='RhodeCode', lastname='Admin',
686 active=True, admin=admin)
694 active=True, admin=admin)
687
695
688 def create_default_user(self):
696 def create_default_user(self):
689 log.info('creating default user')
697 log.info('creating default user')
690 # create default user for handling default permissions.
698 # create default user for handling default permissions.
691 UserModel().create_or_update(username='default',
699 UserModel().create_or_update(username='default',
692 password=str(uuid.uuid1())[:8],
700 password=str(uuid.uuid1())[:8],
693 email='anonymous@rhodecode.org',
701 email='anonymous@rhodecode.org',
694 firstname='Anonymous', lastname='User')
702 firstname='Anonymous', lastname='User')
695
703
696 def create_permissions(self):
704 def create_permissions(self):
697 """
705 """
698 Creates all permissions defined in the system
706 Creates all permissions defined in the system
699 """
707 """
700 # module.(access|create|change|delete)_[name]
708 # module.(access|create|change|delete)_[name]
701 # module.(none|read|write|admin)
709 # module.(none|read|write|admin)
702 log.info('creating permissions')
710 log.info('creating permissions')
703 PermissionModel(self.sa).create_permissions()
711 PermissionModel(self.sa).create_permissions()
704
712
705 def populate_default_permissions(self):
713 def populate_default_permissions(self):
706 """
714 """
707 Populate default permissions. It will create only the default
715 Populate default permissions. It will create only the default
708 permissions that are missing, and not alter already defined ones
716 permissions that are missing, and not alter already defined ones
709 """
717 """
710 log.info('creating default user permissions')
718 log.info('creating default user permissions')
711 PermissionModel(self.sa).create_default_permissions(user=User.DEFAULT_USER)
719 PermissionModel(self.sa).create_default_permissions(user=User.DEFAULT_USER)
712
720
713 @staticmethod
721 @staticmethod
714 def check_waitress():
722 def check_waitress():
715 """
723 """
716 Function executed at the end of setup
724 Function executed at the end of setup
717 """
725 """
718 if not __py_version__ >= (2, 6):
726 if not __py_version__ >= (2, 6):
719 notify('Python2.5 detected, please switch '
727 notify('Python2.5 detected, please switch '
720 'egg:waitress#main -> egg:Paste#http '
728 'egg:waitress#main -> egg:Paste#http '
721 'in your .ini file')
729 'in your .ini file')
@@ -1,815 +1,817 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 rhodecode.lib.utils
3 rhodecode.lib.utils
4 ~~~~~~~~~~~~~~~~~~~
4 ~~~~~~~~~~~~~~~~~~~
5
5
6 Utilities library for RhodeCode
6 Utilities library for RhodeCode
7
7
8 :created_on: Apr 18, 2010
8 :created_on: Apr 18, 2010
9 :author: marcink
9 :author: marcink
10 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
10 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
11 :license: GPLv3, see COPYING for more details.
11 :license: GPLv3, see COPYING for more details.
12 """
12 """
13 # This program is free software: you can redistribute it and/or modify
13 # This program is free software: you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation, either version 3 of the License, or
15 # the Free Software Foundation, either version 3 of the License, or
16 # (at your option) any later version.
16 # (at your option) any later version.
17 #
17 #
18 # This program is distributed in the hope that it will be useful,
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
21 # GNU General Public License for more details.
22 #
22 #
23 # You should have received a copy of the GNU General Public License
23 # You should have received a copy of the GNU General Public License
24 # along with this program. If not, see <http://www.gnu.org/licenses/>.
24 # along with this program. If not, see <http://www.gnu.org/licenses/>.
25
25
26 import os
26 import os
27 import re
27 import re
28 import logging
28 import logging
29 import datetime
29 import datetime
30 import traceback
30 import traceback
31 import paste
31 import paste
32 import beaker
32 import beaker
33 import tarfile
33 import tarfile
34 import shutil
34 import shutil
35 import decorator
35 import decorator
36 import warnings
36 import warnings
37 from os.path import abspath
37 from os.path import abspath
38 from os.path import dirname as dn, join as jn
38 from os.path import dirname as dn, join as jn
39
39
40 from paste.script.command import Command, BadCommand
40 from paste.script.command import Command, BadCommand
41
41
42 from webhelpers.text import collapse, remove_formatting, strip_tags
42 from webhelpers.text import collapse, remove_formatting, strip_tags
43
43
44 from rhodecode.lib.vcs import get_backend
44 from rhodecode.lib.vcs import get_backend
45 from rhodecode.lib.vcs.backends.base import BaseChangeset
45 from rhodecode.lib.vcs.backends.base import BaseChangeset
46 from rhodecode.lib.vcs.utils.lazy import LazyProperty
46 from rhodecode.lib.vcs.utils.lazy import LazyProperty
47 from rhodecode.lib.vcs.utils.hgcompat import ui, config
47 from rhodecode.lib.vcs.utils.hgcompat import ui, config
48 from rhodecode.lib.vcs.utils.helpers import get_scm
48 from rhodecode.lib.vcs.utils.helpers import get_scm
49 from rhodecode.lib.vcs.exceptions import VCSError
49 from rhodecode.lib.vcs.exceptions import VCSError
50
50
51 from rhodecode.lib.caching_query import FromCache
51 from rhodecode.lib.caching_query import FromCache
52
52
53 from rhodecode.model import meta
53 from rhodecode.model import meta
54 from rhodecode.model.db import Repository, User, RhodeCodeUi, \
54 from rhodecode.model.db import Repository, User, RhodeCodeUi, \
55 UserLog, RepoGroup, RhodeCodeSetting, CacheInvalidation, UserGroup
55 UserLog, RepoGroup, RhodeCodeSetting, CacheInvalidation, UserGroup
56 from rhodecode.model.meta import Session
56 from rhodecode.model.meta import Session
57 from rhodecode.model.repos_group import ReposGroupModel
57 from rhodecode.model.repos_group import ReposGroupModel
58 from rhodecode.lib.utils2 import safe_str, safe_unicode
58 from rhodecode.lib.utils2 import safe_str, safe_unicode
59 from rhodecode.lib.vcs.utils.fakemod import create_module
59 from rhodecode.lib.vcs.utils.fakemod import create_module
60 from rhodecode.model.users_group import UserGroupModel
60 from rhodecode.model.users_group import UserGroupModel
61
61
62 log = logging.getLogger(__name__)
62 log = logging.getLogger(__name__)
63
63
64 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
64 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
65
65
66
66
67 def recursive_replace(str_, replace=' '):
67 def recursive_replace(str_, replace=' '):
68 """
68 """
69 Recursive replace of given sign to just one instance
69 Recursive replace of given sign to just one instance
70
70
71 :param str_: given string
71 :param str_: given string
72 :param replace: char to find and replace multiple instances
72 :param replace: char to find and replace multiple instances
73
73
74 Examples::
74 Examples::
75 >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
75 >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
76 'Mighty-Mighty-Bo-sstones'
76 'Mighty-Mighty-Bo-sstones'
77 """
77 """
78
78
79 if str_.find(replace * 2) == -1:
79 if str_.find(replace * 2) == -1:
80 return str_
80 return str_
81 else:
81 else:
82 str_ = str_.replace(replace * 2, replace)
82 str_ = str_.replace(replace * 2, replace)
83 return recursive_replace(str_, replace)
83 return recursive_replace(str_, replace)
84
84
85
85
86 def repo_name_slug(value):
86 def repo_name_slug(value):
87 """
87 """
88 Return slug of name of repository
88 Return slug of name of repository
89 This function is called on each creation/modification
89 This function is called on each creation/modification
90 of repository to prevent bad names in repo
90 of repository to prevent bad names in repo
91 """
91 """
92
92
93 slug = remove_formatting(value)
93 slug = remove_formatting(value)
94 slug = strip_tags(slug)
94 slug = strip_tags(slug)
95
95
96 for c in """`?=[]\;'"<>,/~!@#$%^&*()+{}|: """:
96 for c in """`?=[]\;'"<>,/~!@#$%^&*()+{}|: """:
97 slug = slug.replace(c, '-')
97 slug = slug.replace(c, '-')
98 slug = recursive_replace(slug, '-')
98 slug = recursive_replace(slug, '-')
99 slug = collapse(slug, '-')
99 slug = collapse(slug, '-')
100 return slug
100 return slug
101
101
102
102
103 #==============================================================================
103 #==============================================================================
104 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
104 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
105 #==============================================================================
105 #==============================================================================
106 def get_repo_slug(request):
106 def get_repo_slug(request):
107 _repo = request.environ['pylons.routes_dict'].get('repo_name')
107 _repo = request.environ['pylons.routes_dict'].get('repo_name')
108 if _repo:
108 if _repo:
109 _repo = _repo.rstrip('/')
109 _repo = _repo.rstrip('/')
110 return _repo
110 return _repo
111
111
112
112
113 def get_repos_group_slug(request):
113 def get_repos_group_slug(request):
114 _group = request.environ['pylons.routes_dict'].get('group_name')
114 _group = request.environ['pylons.routes_dict'].get('group_name')
115 if _group:
115 if _group:
116 _group = _group.rstrip('/')
116 _group = _group.rstrip('/')
117 return _group
117 return _group
118
118
119
119
120 def get_user_group_slug(request):
120 def get_user_group_slug(request):
121 _group = request.environ['pylons.routes_dict'].get('id')
121 _group = request.environ['pylons.routes_dict'].get('id')
122 try:
122 try:
123 _group = UserGroup.get(_group)
123 _group = UserGroup.get(_group)
124 if _group:
124 if _group:
125 _group = _group.users_group_name
125 _group = _group.users_group_name
126 except Exception:
126 except Exception:
127 log.debug(traceback.format_exc())
127 log.debug(traceback.format_exc())
128 #catch all failures here
128 #catch all failures here
129 pass
129 pass
130
130
131 return _group
131 return _group
132
132
133
133
134 def action_logger(user, action, repo, ipaddr='', sa=None, commit=False):
134 def action_logger(user, action, repo, ipaddr='', sa=None, commit=False):
135 """
135 """
136 Action logger for various actions made by users
136 Action logger for various actions made by users
137
137
138 :param user: user that made this action, can be a unique username string or
138 :param user: user that made this action, can be a unique username string or
139 object containing user_id attribute
139 object containing user_id attribute
140 :param action: action to log, should be on of predefined unique actions for
140 :param action: action to log, should be on of predefined unique actions for
141 easy translations
141 easy translations
142 :param repo: string name of repository or object containing repo_id,
142 :param repo: string name of repository or object containing repo_id,
143 that action was made on
143 that action was made on
144 :param ipaddr: optional ip address from what the action was made
144 :param ipaddr: optional ip address from what the action was made
145 :param sa: optional sqlalchemy session
145 :param sa: optional sqlalchemy session
146
146
147 """
147 """
148
148
149 if not sa:
149 if not sa:
150 sa = meta.Session()
150 sa = meta.Session()
151
151
152 try:
152 try:
153 if hasattr(user, 'user_id'):
153 if hasattr(user, 'user_id'):
154 user_obj = User.get(user.user_id)
154 user_obj = User.get(user.user_id)
155 elif isinstance(user, basestring):
155 elif isinstance(user, basestring):
156 user_obj = User.get_by_username(user)
156 user_obj = User.get_by_username(user)
157 else:
157 else:
158 raise Exception('You have to provide a user object or a username')
158 raise Exception('You have to provide a user object or a username')
159
159
160 if hasattr(repo, 'repo_id'):
160 if hasattr(repo, 'repo_id'):
161 repo_obj = Repository.get(repo.repo_id)
161 repo_obj = Repository.get(repo.repo_id)
162 repo_name = repo_obj.repo_name
162 repo_name = repo_obj.repo_name
163 elif isinstance(repo, basestring):
163 elif isinstance(repo, basestring):
164 repo_name = repo.lstrip('/')
164 repo_name = repo.lstrip('/')
165 repo_obj = Repository.get_by_repo_name(repo_name)
165 repo_obj = Repository.get_by_repo_name(repo_name)
166 else:
166 else:
167 repo_obj = None
167 repo_obj = None
168 repo_name = ''
168 repo_name = ''
169
169
170 user_log = UserLog()
170 user_log = UserLog()
171 user_log.user_id = user_obj.user_id
171 user_log.user_id = user_obj.user_id
172 user_log.username = user_obj.username
172 user_log.username = user_obj.username
173 user_log.action = safe_unicode(action)
173 user_log.action = safe_unicode(action)
174
174
175 user_log.repository = repo_obj
175 user_log.repository = repo_obj
176 user_log.repository_name = repo_name
176 user_log.repository_name = repo_name
177
177
178 user_log.action_date = datetime.datetime.now()
178 user_log.action_date = datetime.datetime.now()
179 user_log.user_ip = ipaddr
179 user_log.user_ip = ipaddr
180 sa.add(user_log)
180 sa.add(user_log)
181
181
182 log.info('Logging action:%s on %s by user:%s ip:%s' %
182 log.info('Logging action:%s on %s by user:%s ip:%s' %
183 (action, safe_unicode(repo), user_obj, ipaddr))
183 (action, safe_unicode(repo), user_obj, ipaddr))
184 if commit:
184 if commit:
185 sa.commit()
185 sa.commit()
186 except Exception:
186 except Exception:
187 log.error(traceback.format_exc())
187 log.error(traceback.format_exc())
188 raise
188 raise
189
189
190
190
191 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
191 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
192 """
192 """
193 Scans given path for repos and return (name,(type,path)) tuple
193 Scans given path for repos and return (name,(type,path)) tuple
194
194
195 :param path: path to scan for repositories
195 :param path: path to scan for repositories
196 :param recursive: recursive search and return names with subdirs in front
196 :param recursive: recursive search and return names with subdirs in front
197 """
197 """
198
198
199 # remove ending slash for better results
199 # remove ending slash for better results
200 path = path.rstrip(os.sep)
200 path = path.rstrip(os.sep)
201 log.debug('now scanning in %s location recursive:%s...' % (path, recursive))
201 log.debug('now scanning in %s location recursive:%s...' % (path, recursive))
202
202
203 def _get_repos(p):
203 def _get_repos(p):
204 if not os.access(p, os.R_OK) or not os.access(p, os.X_OK):
205 log.warn('ignoring repo path without access: %s', p)
206 return
204 if not os.access(p, os.W_OK):
207 if not os.access(p, os.W_OK):
205 log.warn('ignoring repo path without write access: %s', p)
208 log.warn('repo path without write access: %s', p)
206 return
207 for dirpath in os.listdir(p):
209 for dirpath in os.listdir(p):
208 if os.path.isfile(os.path.join(p, dirpath)):
210 if os.path.isfile(os.path.join(p, dirpath)):
209 continue
211 continue
210 cur_path = os.path.join(p, dirpath)
212 cur_path = os.path.join(p, dirpath)
211
213
212 # skip removed repos
214 # skip removed repos
213 if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
215 if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
214 continue
216 continue
215
217
216 #skip .<somethin> dirs
218 #skip .<somethin> dirs
217 if dirpath.startswith('.'):
219 if dirpath.startswith('.'):
218 continue
220 continue
219
221
220 try:
222 try:
221 scm_info = get_scm(cur_path)
223 scm_info = get_scm(cur_path)
222 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
224 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
223 except VCSError:
225 except VCSError:
224 if not recursive:
226 if not recursive:
225 continue
227 continue
226 #check if this dir containts other repos for recursive scan
228 #check if this dir containts other repos for recursive scan
227 rec_path = os.path.join(p, dirpath)
229 rec_path = os.path.join(p, dirpath)
228 if os.path.isdir(rec_path):
230 if os.path.isdir(rec_path):
229 for inner_scm in _get_repos(rec_path):
231 for inner_scm in _get_repos(rec_path):
230 yield inner_scm
232 yield inner_scm
231
233
232 return _get_repos(path)
234 return _get_repos(path)
233
235
234
236
235 def is_valid_repo(repo_name, base_path, scm=None):
237 def is_valid_repo(repo_name, base_path, scm=None):
236 """
238 """
237 Returns True if given path is a valid repository False otherwise.
239 Returns True if given path is a valid repository False otherwise.
238 If scm param is given also compare if given scm is the same as expected
240 If scm param is given also compare if given scm is the same as expected
239 from scm parameter
241 from scm parameter
240
242
241 :param repo_name:
243 :param repo_name:
242 :param base_path:
244 :param base_path:
243 :param scm:
245 :param scm:
244
246
245 :return True: if given path is a valid repository
247 :return True: if given path is a valid repository
246 """
248 """
247 full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
249 full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
248
250
249 try:
251 try:
250 scm_ = get_scm(full_path)
252 scm_ = get_scm(full_path)
251 if scm:
253 if scm:
252 return scm_[0] == scm
254 return scm_[0] == scm
253 return True
255 return True
254 except VCSError:
256 except VCSError:
255 return False
257 return False
256
258
257
259
258 def is_valid_repos_group(repos_group_name, base_path, skip_path_check=False):
260 def is_valid_repos_group(repos_group_name, base_path, skip_path_check=False):
259 """
261 """
260 Returns True if given path is a repository group False otherwise
262 Returns True if given path is a repository group False otherwise
261
263
262 :param repo_name:
264 :param repo_name:
263 :param base_path:
265 :param base_path:
264 """
266 """
265 full_path = os.path.join(safe_str(base_path), safe_str(repos_group_name))
267 full_path = os.path.join(safe_str(base_path), safe_str(repos_group_name))
266
268
267 # check if it's not a repo
269 # check if it's not a repo
268 if is_valid_repo(repos_group_name, base_path):
270 if is_valid_repo(repos_group_name, base_path):
269 return False
271 return False
270
272
271 try:
273 try:
272 # we need to check bare git repos at higher level
274 # we need to check bare git repos at higher level
273 # since we might match branches/hooks/info/objects or possible
275 # since we might match branches/hooks/info/objects or possible
274 # other things inside bare git repo
276 # other things inside bare git repo
275 get_scm(os.path.dirname(full_path))
277 get_scm(os.path.dirname(full_path))
276 return False
278 return False
277 except VCSError:
279 except VCSError:
278 pass
280 pass
279
281
280 # check if it's a valid path
282 # check if it's a valid path
281 if skip_path_check or os.path.isdir(full_path):
283 if skip_path_check or os.path.isdir(full_path):
282 return True
284 return True
283
285
284 return False
286 return False
285
287
286
288
287 def ask_ok(prompt, retries=4, complaint='Yes or no please!'):
289 def ask_ok(prompt, retries=4, complaint='Yes or no please!'):
288 while True:
290 while True:
289 ok = raw_input(prompt)
291 ok = raw_input(prompt)
290 if ok in ('y', 'ye', 'yes'):
292 if ok in ('y', 'ye', 'yes'):
291 return True
293 return True
292 if ok in ('n', 'no', 'nop', 'nope'):
294 if ok in ('n', 'no', 'nop', 'nope'):
293 return False
295 return False
294 retries = retries - 1
296 retries = retries - 1
295 if retries < 0:
297 if retries < 0:
296 raise IOError
298 raise IOError
297 print complaint
299 print complaint
298
300
299 #propagated from mercurial documentation
301 #propagated from mercurial documentation
300 ui_sections = ['alias', 'auth',
302 ui_sections = ['alias', 'auth',
301 'decode/encode', 'defaults',
303 'decode/encode', 'defaults',
302 'diff', 'email',
304 'diff', 'email',
303 'extensions', 'format',
305 'extensions', 'format',
304 'merge-patterns', 'merge-tools',
306 'merge-patterns', 'merge-tools',
305 'hooks', 'http_proxy',
307 'hooks', 'http_proxy',
306 'smtp', 'patch',
308 'smtp', 'patch',
307 'paths', 'profiling',
309 'paths', 'profiling',
308 'server', 'trusted',
310 'server', 'trusted',
309 'ui', 'web', ]
311 'ui', 'web', ]
310
312
311
313
312 def make_ui(read_from='file', path=None, checkpaths=True, clear_session=True):
314 def make_ui(read_from='file', path=None, checkpaths=True, clear_session=True):
313 """
315 """
314 A function that will read python rc files or database
316 A function that will read python rc files or database
315 and make an mercurial ui object from read options
317 and make an mercurial ui object from read options
316
318
317 :param path: path to mercurial config file
319 :param path: path to mercurial config file
318 :param checkpaths: check the path
320 :param checkpaths: check the path
319 :param read_from: read from 'file' or 'db'
321 :param read_from: read from 'file' or 'db'
320 """
322 """
321
323
322 baseui = ui.ui()
324 baseui = ui.ui()
323
325
324 # clean the baseui object
326 # clean the baseui object
325 baseui._ocfg = config.config()
327 baseui._ocfg = config.config()
326 baseui._ucfg = config.config()
328 baseui._ucfg = config.config()
327 baseui._tcfg = config.config()
329 baseui._tcfg = config.config()
328
330
329 if read_from == 'file':
331 if read_from == 'file':
330 if not os.path.isfile(path):
332 if not os.path.isfile(path):
331 log.debug('hgrc file is not present at %s, skipping...' % path)
333 log.debug('hgrc file is not present at %s, skipping...' % path)
332 return False
334 return False
333 log.debug('reading hgrc from %s' % path)
335 log.debug('reading hgrc from %s' % path)
334 cfg = config.config()
336 cfg = config.config()
335 cfg.read(path)
337 cfg.read(path)
336 for section in ui_sections:
338 for section in ui_sections:
337 for k, v in cfg.items(section):
339 for k, v in cfg.items(section):
338 log.debug('settings ui from file: [%s] %s=%s' % (section, k, v))
340 log.debug('settings ui from file: [%s] %s=%s' % (section, k, v))
339 baseui.setconfig(safe_str(section), safe_str(k), safe_str(v))
341 baseui.setconfig(safe_str(section), safe_str(k), safe_str(v))
340
342
341 elif read_from == 'db':
343 elif read_from == 'db':
342 sa = meta.Session()
344 sa = meta.Session()
343 ret = sa.query(RhodeCodeUi)\
345 ret = sa.query(RhodeCodeUi)\
344 .options(FromCache("sql_cache_short", "get_hg_ui_settings"))\
346 .options(FromCache("sql_cache_short", "get_hg_ui_settings"))\
345 .all()
347 .all()
346
348
347 hg_ui = ret
349 hg_ui = ret
348 for ui_ in hg_ui:
350 for ui_ in hg_ui:
349 if ui_.ui_active:
351 if ui_.ui_active:
350 log.debug('settings ui from db: [%s] %s=%s', ui_.ui_section,
352 log.debug('settings ui from db: [%s] %s=%s', ui_.ui_section,
351 ui_.ui_key, ui_.ui_value)
353 ui_.ui_key, ui_.ui_value)
352 baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),
354 baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),
353 safe_str(ui_.ui_value))
355 safe_str(ui_.ui_value))
354 if ui_.ui_key == 'push_ssl':
356 if ui_.ui_key == 'push_ssl':
355 # force set push_ssl requirement to False, rhodecode
357 # force set push_ssl requirement to False, rhodecode
356 # handles that
358 # handles that
357 baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),
359 baseui.setconfig(safe_str(ui_.ui_section), safe_str(ui_.ui_key),
358 False)
360 False)
359 if clear_session:
361 if clear_session:
360 meta.Session.remove()
362 meta.Session.remove()
361 return baseui
363 return baseui
362
364
363
365
364 def set_rhodecode_config(config):
366 def set_rhodecode_config(config):
365 """
367 """
366 Updates pylons config with new settings from database
368 Updates pylons config with new settings from database
367
369
368 :param config:
370 :param config:
369 """
371 """
370 hgsettings = RhodeCodeSetting.get_app_settings()
372 hgsettings = RhodeCodeSetting.get_app_settings()
371
373
372 for k, v in hgsettings.items():
374 for k, v in hgsettings.items():
373 config[k] = v
375 config[k] = v
374
376
375
377
376 def set_vcs_config(config):
378 def set_vcs_config(config):
377 """
379 """
378 Patch VCS config with some RhodeCode specific stuff
380 Patch VCS config with some RhodeCode specific stuff
379
381
380 :param config: rhodecode.CONFIG
382 :param config: rhodecode.CONFIG
381 """
383 """
382 import rhodecode
384 import rhodecode
383 from rhodecode.lib.vcs import conf
385 from rhodecode.lib.vcs import conf
384 from rhodecode.lib.utils2 import aslist
386 from rhodecode.lib.utils2 import aslist
385 conf.settings.BACKENDS = {
387 conf.settings.BACKENDS = {
386 'hg': 'rhodecode.lib.vcs.backends.hg.MercurialRepository',
388 'hg': 'rhodecode.lib.vcs.backends.hg.MercurialRepository',
387 'git': 'rhodecode.lib.vcs.backends.git.GitRepository',
389 'git': 'rhodecode.lib.vcs.backends.git.GitRepository',
388 }
390 }
389
391
390 conf.settings.GIT_EXECUTABLE_PATH = config.get('git_path', 'git')
392 conf.settings.GIT_EXECUTABLE_PATH = config.get('git_path', 'git')
391 conf.settings.GIT_REV_FILTER = config.get('git_rev_filter', '--all').strip()
393 conf.settings.GIT_REV_FILTER = config.get('git_rev_filter', '--all').strip()
392 conf.settings.DEFAULT_ENCODINGS = aslist(config.get('default_encoding',
394 conf.settings.DEFAULT_ENCODINGS = aslist(config.get('default_encoding',
393 'utf8'), sep=',')
395 'utf8'), sep=',')
394
396
395
397
396 def map_groups(path):
398 def map_groups(path):
397 """
399 """
398 Given a full path to a repository, create all nested groups that this
400 Given a full path to a repository, create all nested groups that this
399 repo is inside. This function creates parent-child relationships between
401 repo is inside. This function creates parent-child relationships between
400 groups and creates default perms for all new groups.
402 groups and creates default perms for all new groups.
401
403
402 :param paths: full path to repository
404 :param paths: full path to repository
403 """
405 """
404 sa = meta.Session()
406 sa = meta.Session()
405 groups = path.split(Repository.url_sep())
407 groups = path.split(Repository.url_sep())
406 parent = None
408 parent = None
407 group = None
409 group = None
408
410
409 # last element is repo in nested groups structure
411 # last element is repo in nested groups structure
410 groups = groups[:-1]
412 groups = groups[:-1]
411 rgm = ReposGroupModel(sa)
413 rgm = ReposGroupModel(sa)
412 owner = User.get_first_admin()
414 owner = User.get_first_admin()
413 for lvl, group_name in enumerate(groups):
415 for lvl, group_name in enumerate(groups):
414 group_name = '/'.join(groups[:lvl] + [group_name])
416 group_name = '/'.join(groups[:lvl] + [group_name])
415 group = RepoGroup.get_by_group_name(group_name)
417 group = RepoGroup.get_by_group_name(group_name)
416 desc = '%s group' % group_name
418 desc = '%s group' % group_name
417
419
418 # skip folders that are now removed repos
420 # skip folders that are now removed repos
419 if REMOVED_REPO_PAT.match(group_name):
421 if REMOVED_REPO_PAT.match(group_name):
420 break
422 break
421
423
422 if group is None:
424 if group is None:
423 log.debug('creating group level: %s group_name: %s'
425 log.debug('creating group level: %s group_name: %s'
424 % (lvl, group_name))
426 % (lvl, group_name))
425 group = RepoGroup(group_name, parent)
427 group = RepoGroup(group_name, parent)
426 group.group_description = desc
428 group.group_description = desc
427 group.user = owner
429 group.user = owner
428 sa.add(group)
430 sa.add(group)
429 perm_obj = rgm._create_default_perms(group)
431 perm_obj = rgm._create_default_perms(group)
430 sa.add(perm_obj)
432 sa.add(perm_obj)
431 sa.flush()
433 sa.flush()
432
434
433 parent = group
435 parent = group
434 return group
436 return group
435
437
436
438
437 def repo2db_mapper(initial_repo_list, remove_obsolete=False,
439 def repo2db_mapper(initial_repo_list, remove_obsolete=False,
438 install_git_hook=False):
440 install_git_hook=False):
439 """
441 """
440 maps all repos given in initial_repo_list, non existing repositories
442 maps all repos given in initial_repo_list, non existing repositories
441 are created, if remove_obsolete is True it also check for db entries
443 are created, if remove_obsolete is True it also check for db entries
442 that are not in initial_repo_list and removes them.
444 that are not in initial_repo_list and removes them.
443
445
444 :param initial_repo_list: list of repositories found by scanning methods
446 :param initial_repo_list: list of repositories found by scanning methods
445 :param remove_obsolete: check for obsolete entries in database
447 :param remove_obsolete: check for obsolete entries in database
446 :param install_git_hook: if this is True, also check and install githook
448 :param install_git_hook: if this is True, also check and install githook
447 for a repo if missing
449 for a repo if missing
448 """
450 """
449 from rhodecode.model.repo import RepoModel
451 from rhodecode.model.repo import RepoModel
450 from rhodecode.model.scm import ScmModel
452 from rhodecode.model.scm import ScmModel
451 sa = meta.Session()
453 sa = meta.Session()
452 rm = RepoModel()
454 rm = RepoModel()
453 user = User.get_first_admin()
455 user = User.get_first_admin()
454 added = []
456 added = []
455
457
456 ##creation defaults
458 ##creation defaults
457 defs = RhodeCodeSetting.get_default_repo_settings(strip_prefix=True)
459 defs = RhodeCodeSetting.get_default_repo_settings(strip_prefix=True)
458 enable_statistics = defs.get('repo_enable_statistics')
460 enable_statistics = defs.get('repo_enable_statistics')
459 enable_locking = defs.get('repo_enable_locking')
461 enable_locking = defs.get('repo_enable_locking')
460 enable_downloads = defs.get('repo_enable_downloads')
462 enable_downloads = defs.get('repo_enable_downloads')
461 private = defs.get('repo_private')
463 private = defs.get('repo_private')
462
464
463 for name, repo in initial_repo_list.items():
465 for name, repo in initial_repo_list.items():
464 group = map_groups(name)
466 group = map_groups(name)
465 db_repo = rm.get_by_repo_name(name)
467 db_repo = rm.get_by_repo_name(name)
466 # found repo that is on filesystem not in RhodeCode database
468 # found repo that is on filesystem not in RhodeCode database
467 if not db_repo:
469 if not db_repo:
468 log.info('repository %s not found, creating now' % name)
470 log.info('repository %s not found, creating now' % name)
469 added.append(name)
471 added.append(name)
470 desc = (repo.description
472 desc = (repo.description
471 if repo.description != 'unknown'
473 if repo.description != 'unknown'
472 else '%s repository' % name)
474 else '%s repository' % name)
473
475
474 new_repo = rm.create_repo(
476 new_repo = rm.create_repo(
475 repo_name=name,
477 repo_name=name,
476 repo_type=repo.alias,
478 repo_type=repo.alias,
477 description=desc,
479 description=desc,
478 repos_group=getattr(group, 'group_id', None),
480 repos_group=getattr(group, 'group_id', None),
479 owner=user,
481 owner=user,
480 just_db=True,
482 just_db=True,
481 enable_locking=enable_locking,
483 enable_locking=enable_locking,
482 enable_downloads=enable_downloads,
484 enable_downloads=enable_downloads,
483 enable_statistics=enable_statistics,
485 enable_statistics=enable_statistics,
484 private=private
486 private=private
485 )
487 )
486 # we added that repo just now, and make sure it has githook
488 # we added that repo just now, and make sure it has githook
487 # installed
489 # installed
488 if new_repo.repo_type == 'git':
490 if new_repo.repo_type == 'git':
489 ScmModel().install_git_hook(new_repo.scm_instance)
491 ScmModel().install_git_hook(new_repo.scm_instance)
490 new_repo.update_changeset_cache()
492 new_repo.update_changeset_cache()
491 elif install_git_hook:
493 elif install_git_hook:
492 if db_repo.repo_type == 'git':
494 if db_repo.repo_type == 'git':
493 ScmModel().install_git_hook(db_repo.scm_instance)
495 ScmModel().install_git_hook(db_repo.scm_instance)
494
496
495 sa.commit()
497 sa.commit()
496 removed = []
498 removed = []
497 if remove_obsolete:
499 if remove_obsolete:
498 # remove from database those repositories that are not in the filesystem
500 # remove from database those repositories that are not in the filesystem
499 for repo in sa.query(Repository).all():
501 for repo in sa.query(Repository).all():
500 if repo.repo_name not in initial_repo_list.keys():
502 if repo.repo_name not in initial_repo_list.keys():
501 log.debug("Removing non-existing repository found in db `%s`" %
503 log.debug("Removing non-existing repository found in db `%s`" %
502 repo.repo_name)
504 repo.repo_name)
503 try:
505 try:
504 removed.append(repo.repo_name)
506 removed.append(repo.repo_name)
505 RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
507 RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
506 sa.commit()
508 sa.commit()
507 except Exception:
509 except Exception:
508 #don't hold further removals on error
510 #don't hold further removals on error
509 log.error(traceback.format_exc())
511 log.error(traceback.format_exc())
510 sa.rollback()
512 sa.rollback()
511 return added, removed
513 return added, removed
512
514
513
515
514 # set cache regions for beaker so celery can utilise it
516 # set cache regions for beaker so celery can utilise it
515 def add_cache(settings):
517 def add_cache(settings):
516 cache_settings = {'regions': None}
518 cache_settings = {'regions': None}
517 for key in settings.keys():
519 for key in settings.keys():
518 for prefix in ['beaker.cache.', 'cache.']:
520 for prefix in ['beaker.cache.', 'cache.']:
519 if key.startswith(prefix):
521 if key.startswith(prefix):
520 name = key.split(prefix)[1].strip()
522 name = key.split(prefix)[1].strip()
521 cache_settings[name] = settings[key].strip()
523 cache_settings[name] = settings[key].strip()
522 if cache_settings['regions']:
524 if cache_settings['regions']:
523 for region in cache_settings['regions'].split(','):
525 for region in cache_settings['regions'].split(','):
524 region = region.strip()
526 region = region.strip()
525 region_settings = {}
527 region_settings = {}
526 for key, value in cache_settings.items():
528 for key, value in cache_settings.items():
527 if key.startswith(region):
529 if key.startswith(region):
528 region_settings[key.split('.')[1]] = value
530 region_settings[key.split('.')[1]] = value
529 region_settings['expire'] = int(region_settings.get('expire',
531 region_settings['expire'] = int(region_settings.get('expire',
530 60))
532 60))
531 region_settings.setdefault('lock_dir',
533 region_settings.setdefault('lock_dir',
532 cache_settings.get('lock_dir'))
534 cache_settings.get('lock_dir'))
533 region_settings.setdefault('data_dir',
535 region_settings.setdefault('data_dir',
534 cache_settings.get('data_dir'))
536 cache_settings.get('data_dir'))
535
537
536 if 'type' not in region_settings:
538 if 'type' not in region_settings:
537 region_settings['type'] = cache_settings.get('type',
539 region_settings['type'] = cache_settings.get('type',
538 'memory')
540 'memory')
539 beaker.cache.cache_regions[region] = region_settings
541 beaker.cache.cache_regions[region] = region_settings
540
542
541
543
542 def load_rcextensions(root_path):
544 def load_rcextensions(root_path):
543 import rhodecode
545 import rhodecode
544 from rhodecode.config import conf
546 from rhodecode.config import conf
545
547
546 path = os.path.join(root_path, 'rcextensions', '__init__.py')
548 path = os.path.join(root_path, 'rcextensions', '__init__.py')
547 if os.path.isfile(path):
549 if os.path.isfile(path):
548 rcext = create_module('rc', path)
550 rcext = create_module('rc', path)
549 EXT = rhodecode.EXTENSIONS = rcext
551 EXT = rhodecode.EXTENSIONS = rcext
550 log.debug('Found rcextensions now loading %s...' % rcext)
552 log.debug('Found rcextensions now loading %s...' % rcext)
551
553
552 # Additional mappings that are not present in the pygments lexers
554 # Additional mappings that are not present in the pygments lexers
553 conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
555 conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
554
556
555 #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)
557 #OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)
556
558
557 if getattr(EXT, 'INDEX_EXTENSIONS', []):
559 if getattr(EXT, 'INDEX_EXTENSIONS', []):
558 log.debug('settings custom INDEX_EXTENSIONS')
560 log.debug('settings custom INDEX_EXTENSIONS')
559 conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])
561 conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])
560
562
561 #ADDITIONAL MAPPINGS
563 #ADDITIONAL MAPPINGS
562 log.debug('adding extra into INDEX_EXTENSIONS')
564 log.debug('adding extra into INDEX_EXTENSIONS')
563 conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))
565 conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))
564
566
565 # auto check if the module is not missing any data, set to default if is
567 # auto check if the module is not missing any data, set to default if is
566 # this will help autoupdate new feature of rcext module
568 # this will help autoupdate new feature of rcext module
567 from rhodecode.config import rcextensions
569 from rhodecode.config import rcextensions
568 for k in dir(rcextensions):
570 for k in dir(rcextensions):
569 if not k.startswith('_') and not hasattr(EXT, k):
571 if not k.startswith('_') and not hasattr(EXT, k):
570 setattr(EXT, k, getattr(rcextensions, k))
572 setattr(EXT, k, getattr(rcextensions, k))
571
573
572
574
573 def get_custom_lexer(extension):
575 def get_custom_lexer(extension):
574 """
576 """
575 returns a custom lexer if it's defined in rcextensions module, or None
577 returns a custom lexer if it's defined in rcextensions module, or None
576 if there's no custom lexer defined
578 if there's no custom lexer defined
577 """
579 """
578 import rhodecode
580 import rhodecode
579 from pygments import lexers
581 from pygments import lexers
580 #check if we didn't define this extension as other lexer
582 #check if we didn't define this extension as other lexer
581 if rhodecode.EXTENSIONS and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
583 if rhodecode.EXTENSIONS and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
582 _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
584 _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
583 return lexers.get_lexer_by_name(_lexer_name)
585 return lexers.get_lexer_by_name(_lexer_name)
584
586
585
587
586 #==============================================================================
588 #==============================================================================
587 # TEST FUNCTIONS AND CREATORS
589 # TEST FUNCTIONS AND CREATORS
588 #==============================================================================
590 #==============================================================================
589 def create_test_index(repo_location, config, full_index):
591 def create_test_index(repo_location, config, full_index):
590 """
592 """
591 Makes default test index
593 Makes default test index
592
594
593 :param config: test config
595 :param config: test config
594 :param full_index:
596 :param full_index:
595 """
597 """
596
598
597 from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
599 from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon
598 from rhodecode.lib.pidlock import DaemonLock, LockHeld
600 from rhodecode.lib.pidlock import DaemonLock, LockHeld
599
601
600 repo_location = repo_location
602 repo_location = repo_location
601
603
602 index_location = os.path.join(config['app_conf']['index_dir'])
604 index_location = os.path.join(config['app_conf']['index_dir'])
603 if not os.path.exists(index_location):
605 if not os.path.exists(index_location):
604 os.makedirs(index_location)
606 os.makedirs(index_location)
605
607
606 try:
608 try:
607 l = DaemonLock(file_=jn(dn(index_location), 'make_index.lock'))
609 l = DaemonLock(file_=jn(dn(index_location), 'make_index.lock'))
608 WhooshIndexingDaemon(index_location=index_location,
610 WhooshIndexingDaemon(index_location=index_location,
609 repo_location=repo_location)\
611 repo_location=repo_location)\
610 .run(full_index=full_index)
612 .run(full_index=full_index)
611 l.release()
613 l.release()
612 except LockHeld:
614 except LockHeld:
613 pass
615 pass
614
616
615
617
616 def create_test_env(repos_test_path, config):
618 def create_test_env(repos_test_path, config):
617 """
619 """
618 Makes a fresh database and
620 Makes a fresh database and
619 install test repository into tmp dir
621 install test repository into tmp dir
620 """
622 """
621 from rhodecode.lib.db_manage import DbManage
623 from rhodecode.lib.db_manage import DbManage
622 from rhodecode.tests import HG_REPO, GIT_REPO, TESTS_TMP_PATH
624 from rhodecode.tests import HG_REPO, GIT_REPO, TESTS_TMP_PATH
623
625
624 # PART ONE create db
626 # PART ONE create db
625 dbconf = config['sqlalchemy.db1.url']
627 dbconf = config['sqlalchemy.db1.url']
626 log.debug('making test db %s' % dbconf)
628 log.debug('making test db %s' % dbconf)
627
629
628 # create test dir if it doesn't exist
630 # create test dir if it doesn't exist
629 if not os.path.isdir(repos_test_path):
631 if not os.path.isdir(repos_test_path):
630 log.debug('Creating testdir %s' % repos_test_path)
632 log.debug('Creating testdir %s' % repos_test_path)
631 os.makedirs(repos_test_path)
633 os.makedirs(repos_test_path)
632
634
633 dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],
635 dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],
634 tests=True)
636 tests=True)
635 dbmanage.create_tables(override=True)
637 dbmanage.create_tables(override=True)
636 dbmanage.create_settings(dbmanage.config_prompt(repos_test_path))
638 dbmanage.create_settings(dbmanage.config_prompt(repos_test_path))
637 dbmanage.create_default_user()
639 dbmanage.create_default_user()
638 dbmanage.admin_prompt()
640 dbmanage.admin_prompt()
639 dbmanage.create_permissions()
641 dbmanage.create_permissions()
640 dbmanage.populate_default_permissions()
642 dbmanage.populate_default_permissions()
641 Session().commit()
643 Session().commit()
642 # PART TWO make test repo
644 # PART TWO make test repo
643 log.debug('making test vcs repositories')
645 log.debug('making test vcs repositories')
644
646
645 idx_path = config['app_conf']['index_dir']
647 idx_path = config['app_conf']['index_dir']
646 data_path = config['app_conf']['cache_dir']
648 data_path = config['app_conf']['cache_dir']
647
649
648 #clean index and data
650 #clean index and data
649 if idx_path and os.path.exists(idx_path):
651 if idx_path and os.path.exists(idx_path):
650 log.debug('remove %s' % idx_path)
652 log.debug('remove %s' % idx_path)
651 shutil.rmtree(idx_path)
653 shutil.rmtree(idx_path)
652
654
653 if data_path and os.path.exists(data_path):
655 if data_path and os.path.exists(data_path):
654 log.debug('remove %s' % data_path)
656 log.debug('remove %s' % data_path)
655 shutil.rmtree(data_path)
657 shutil.rmtree(data_path)
656
658
657 #CREATE DEFAULT TEST REPOS
659 #CREATE DEFAULT TEST REPOS
658 cur_dir = dn(dn(abspath(__file__)))
660 cur_dir = dn(dn(abspath(__file__)))
659 tar = tarfile.open(jn(cur_dir, 'tests', 'fixtures', "vcs_test_hg.tar.gz"))
661 tar = tarfile.open(jn(cur_dir, 'tests', 'fixtures', "vcs_test_hg.tar.gz"))
660 tar.extractall(jn(TESTS_TMP_PATH, HG_REPO))
662 tar.extractall(jn(TESTS_TMP_PATH, HG_REPO))
661 tar.close()
663 tar.close()
662
664
663 cur_dir = dn(dn(abspath(__file__)))
665 cur_dir = dn(dn(abspath(__file__)))
664 tar = tarfile.open(jn(cur_dir, 'tests', 'fixtures', "vcs_test_git.tar.gz"))
666 tar = tarfile.open(jn(cur_dir, 'tests', 'fixtures', "vcs_test_git.tar.gz"))
665 tar.extractall(jn(TESTS_TMP_PATH, GIT_REPO))
667 tar.extractall(jn(TESTS_TMP_PATH, GIT_REPO))
666 tar.close()
668 tar.close()
667
669
668 #LOAD VCS test stuff
670 #LOAD VCS test stuff
669 from rhodecode.tests.vcs import setup_package
671 from rhodecode.tests.vcs import setup_package
670 setup_package()
672 setup_package()
671
673
672
674
673 #==============================================================================
675 #==============================================================================
674 # PASTER COMMANDS
676 # PASTER COMMANDS
675 #==============================================================================
677 #==============================================================================
676 class BasePasterCommand(Command):
678 class BasePasterCommand(Command):
677 """
679 """
678 Abstract Base Class for paster commands.
680 Abstract Base Class for paster commands.
679
681
680 The celery commands are somewhat aggressive about loading
682 The celery commands are somewhat aggressive about loading
681 celery.conf, and since our module sets the `CELERY_LOADER`
683 celery.conf, and since our module sets the `CELERY_LOADER`
682 environment variable to our loader, we have to bootstrap a bit and
684 environment variable to our loader, we have to bootstrap a bit and
683 make sure we've had a chance to load the pylons config off of the
685 make sure we've had a chance to load the pylons config off of the
684 command line, otherwise everything fails.
686 command line, otherwise everything fails.
685 """
687 """
686 min_args = 1
688 min_args = 1
687 min_args_error = "Please provide a paster config file as an argument."
689 min_args_error = "Please provide a paster config file as an argument."
688 takes_config_file = 1
690 takes_config_file = 1
689 requires_config_file = True
691 requires_config_file = True
690
692
691 def notify_msg(self, msg, log=False):
693 def notify_msg(self, msg, log=False):
692 """Make a notification to user, additionally if logger is passed
694 """Make a notification to user, additionally if logger is passed
693 it logs this action using given logger
695 it logs this action using given logger
694
696
695 :param msg: message that will be printed to user
697 :param msg: message that will be printed to user
696 :param log: logging instance, to use to additionally log this message
698 :param log: logging instance, to use to additionally log this message
697
699
698 """
700 """
699 if log and isinstance(log, logging):
701 if log and isinstance(log, logging):
700 log(msg)
702 log(msg)
701
703
702 def run(self, args):
704 def run(self, args):
703 """
705 """
704 Overrides Command.run
706 Overrides Command.run
705
707
706 Checks for a config file argument and loads it.
708 Checks for a config file argument and loads it.
707 """
709 """
708 if len(args) < self.min_args:
710 if len(args) < self.min_args:
709 raise BadCommand(
711 raise BadCommand(
710 self.min_args_error % {'min_args': self.min_args,
712 self.min_args_error % {'min_args': self.min_args,
711 'actual_args': len(args)})
713 'actual_args': len(args)})
712
714
713 # Decrement because we're going to lob off the first argument.
715 # Decrement because we're going to lob off the first argument.
714 # @@ This is hacky
716 # @@ This is hacky
715 self.min_args -= 1
717 self.min_args -= 1
716 self.bootstrap_config(args[0])
718 self.bootstrap_config(args[0])
717 self.update_parser()
719 self.update_parser()
718 return super(BasePasterCommand, self).run(args[1:])
720 return super(BasePasterCommand, self).run(args[1:])
719
721
720 def update_parser(self):
722 def update_parser(self):
721 """
723 """
722 Abstract method. Allows for the class's parser to be updated
724 Abstract method. Allows for the class's parser to be updated
723 before the superclass's `run` method is called. Necessary to
725 before the superclass's `run` method is called. Necessary to
724 allow options/arguments to be passed through to the underlying
726 allow options/arguments to be passed through to the underlying
725 celery command.
727 celery command.
726 """
728 """
727 raise NotImplementedError("Abstract Method.")
729 raise NotImplementedError("Abstract Method.")
728
730
729 def bootstrap_config(self, conf):
731 def bootstrap_config(self, conf):
730 """
732 """
731 Loads the pylons configuration.
733 Loads the pylons configuration.
732 """
734 """
733 from pylons import config as pylonsconfig
735 from pylons import config as pylonsconfig
734
736
735 self.path_to_ini_file = os.path.realpath(conf)
737 self.path_to_ini_file = os.path.realpath(conf)
736 conf = paste.deploy.appconfig('config:' + self.path_to_ini_file)
738 conf = paste.deploy.appconfig('config:' + self.path_to_ini_file)
737 pylonsconfig.init_app(conf.global_conf, conf.local_conf)
739 pylonsconfig.init_app(conf.global_conf, conf.local_conf)
738
740
739 def _init_session(self):
741 def _init_session(self):
740 """
742 """
741 Inits SqlAlchemy Session
743 Inits SqlAlchemy Session
742 """
744 """
743 logging.config.fileConfig(self.path_to_ini_file)
745 logging.config.fileConfig(self.path_to_ini_file)
744 from pylons import config
746 from pylons import config
745 from rhodecode.model import init_model
747 from rhodecode.model import init_model
746 from rhodecode.lib.utils2 import engine_from_config
748 from rhodecode.lib.utils2 import engine_from_config
747
749
748 #get to remove repos !!
750 #get to remove repos !!
749 add_cache(config)
751 add_cache(config)
750 engine = engine_from_config(config, 'sqlalchemy.db1.')
752 engine = engine_from_config(config, 'sqlalchemy.db1.')
751 init_model(engine)
753 init_model(engine)
752
754
753
755
754 def check_git_version():
756 def check_git_version():
755 """
757 """
756 Checks what version of git is installed in system, and issues a warning
758 Checks what version of git is installed in system, and issues a warning
757 if it's too old for RhodeCode to properly work.
759 if it's too old for RhodeCode to properly work.
758 """
760 """
759 from rhodecode import BACKENDS
761 from rhodecode import BACKENDS
760 from rhodecode.lib.vcs.backends.git.repository import GitRepository
762 from rhodecode.lib.vcs.backends.git.repository import GitRepository
761 from rhodecode.lib.vcs.conf import settings
763 from rhodecode.lib.vcs.conf import settings
762 from distutils.version import StrictVersion
764 from distutils.version import StrictVersion
763
765
764 stdout, stderr = GitRepository._run_git_command('--version', _bare=True,
766 stdout, stderr = GitRepository._run_git_command('--version', _bare=True,
765 _safe=True)
767 _safe=True)
766
768
767 ver = (stdout.split(' ')[-1] or '').strip() or '0.0.0'
769 ver = (stdout.split(' ')[-1] or '').strip() or '0.0.0'
768 if len(ver.split('.')) > 3:
770 if len(ver.split('.')) > 3:
769 #StrictVersion needs to be only 3 element type
771 #StrictVersion needs to be only 3 element type
770 ver = '.'.join(ver.split('.')[:3])
772 ver = '.'.join(ver.split('.')[:3])
771 try:
773 try:
772 _ver = StrictVersion(ver)
774 _ver = StrictVersion(ver)
773 except Exception:
775 except Exception:
774 _ver = StrictVersion('0.0.0')
776 _ver = StrictVersion('0.0.0')
775 stderr = traceback.format_exc()
777 stderr = traceback.format_exc()
776
778
777 req_ver = '1.7.4'
779 req_ver = '1.7.4'
778 to_old_git = False
780 to_old_git = False
779 if _ver < StrictVersion(req_ver):
781 if _ver < StrictVersion(req_ver):
780 to_old_git = True
782 to_old_git = True
781
783
782 if 'git' in BACKENDS:
784 if 'git' in BACKENDS:
783 log.debug('GIT executable: "%s" version detected: %s'
785 log.debug('GIT executable: "%s" version detected: %s'
784 % (settings.GIT_EXECUTABLE_PATH, stdout))
786 % (settings.GIT_EXECUTABLE_PATH, stdout))
785 if stderr:
787 if stderr:
786 log.warning('Unable to detect git version, org error was: %r' % stderr)
788 log.warning('Unable to detect git version, org error was: %r' % stderr)
787 elif to_old_git:
789 elif to_old_git:
788 log.warning('RhodeCode detected git version %s, which is too old '
790 log.warning('RhodeCode detected git version %s, which is too old '
789 'for the system to function properly. Make sure '
791 'for the system to function properly. Make sure '
790 'its version is at least %s' % (ver, req_ver))
792 'its version is at least %s' % (ver, req_ver))
791 return _ver
793 return _ver
792
794
793
795
794 @decorator.decorator
796 @decorator.decorator
795 def jsonify(func, *args, **kwargs):
797 def jsonify(func, *args, **kwargs):
796 """Action decorator that formats output for JSON
798 """Action decorator that formats output for JSON
797
799
798 Given a function that will return content, this decorator will turn
800 Given a function that will return content, this decorator will turn
799 the result into JSON, with a content-type of 'application/json' and
801 the result into JSON, with a content-type of 'application/json' and
800 output it.
802 output it.
801
803
802 """
804 """
803 from pylons.decorators.util import get_pylons
805 from pylons.decorators.util import get_pylons
804 from rhodecode.lib.compat import json
806 from rhodecode.lib.compat import json
805 pylons = get_pylons(args)
807 pylons = get_pylons(args)
806 pylons.response.headers['Content-Type'] = 'application/json; charset=utf-8'
808 pylons.response.headers['Content-Type'] = 'application/json; charset=utf-8'
807 data = func(*args, **kwargs)
809 data = func(*args, **kwargs)
808 if isinstance(data, (list, tuple)):
810 if isinstance(data, (list, tuple)):
809 msg = "JSON responses with Array envelopes are susceptible to " \
811 msg = "JSON responses with Array envelopes are susceptible to " \
810 "cross-site data leak attacks, see " \
812 "cross-site data leak attacks, see " \
811 "http://wiki.pylonshq.com/display/pylonsfaq/Warnings"
813 "http://wiki.pylonshq.com/display/pylonsfaq/Warnings"
812 warnings.warn(msg, Warning, 2)
814 warnings.warn(msg, Warning, 2)
813 log.warning(msg)
815 log.warning(msg)
814 log.debug("Returning JSON wrapped action output")
816 log.debug("Returning JSON wrapped action output")
815 return json.dumps(data, encoding='utf-8')
817 return json.dumps(data, encoding='utf-8')
@@ -1,752 +1,755 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 rhodecode.model.scm
3 rhodecode.model.scm
4 ~~~~~~~~~~~~~~~~~~~
4 ~~~~~~~~~~~~~~~~~~~
5
5
6 Scm model for RhodeCode
6 Scm model for RhodeCode
7
7
8 :created_on: Apr 9, 2010
8 :created_on: Apr 9, 2010
9 :author: marcink
9 :author: marcink
10 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
10 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
11 :license: GPLv3, see COPYING for more details.
11 :license: GPLv3, see COPYING for more details.
12 """
12 """
13 # This program is free software: you can redistribute it and/or modify
13 # This program is free software: you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation, either version 3 of the License, or
15 # the Free Software Foundation, either version 3 of the License, or
16 # (at your option) any later version.
16 # (at your option) any later version.
17 #
17 #
18 # This program is distributed in the hope that it will be useful,
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
21 # GNU General Public License for more details.
22 #
22 #
23 # You should have received a copy of the GNU General Public License
23 # You should have received a copy of the GNU General Public License
24 # along with this program. If not, see <http://www.gnu.org/licenses/>.
24 # along with this program. If not, see <http://www.gnu.org/licenses/>.
25 from __future__ import with_statement
25 from __future__ import with_statement
26 import os
26 import os
27 import re
27 import re
28 import time
28 import time
29 import traceback
29 import traceback
30 import logging
30 import logging
31 import cStringIO
31 import cStringIO
32 import pkg_resources
32 import pkg_resources
33 from os.path import join as jn
33 from os.path import join as jn
34
34
35 from sqlalchemy import func
35 from sqlalchemy import func
36 from pylons.i18n.translation import _
36 from pylons.i18n.translation import _
37
37
38 import rhodecode
38 import rhodecode
39 from rhodecode.lib.vcs import get_backend
39 from rhodecode.lib.vcs import get_backend
40 from rhodecode.lib.vcs.exceptions import RepositoryError
40 from rhodecode.lib.vcs.exceptions import RepositoryError
41 from rhodecode.lib.vcs.utils.lazy import LazyProperty
41 from rhodecode.lib.vcs.utils.lazy import LazyProperty
42 from rhodecode.lib.vcs.nodes import FileNode
42 from rhodecode.lib.vcs.nodes import FileNode
43 from rhodecode.lib.vcs.backends.base import EmptyChangeset
43 from rhodecode.lib.vcs.backends.base import EmptyChangeset
44
44
45 from rhodecode import BACKENDS
45 from rhodecode import BACKENDS
46 from rhodecode.lib import helpers as h
46 from rhodecode.lib import helpers as h
47 from rhodecode.lib.utils2 import safe_str, safe_unicode, get_server_url,\
47 from rhodecode.lib.utils2 import safe_str, safe_unicode, get_server_url,\
48 _set_extras
48 _set_extras
49 from rhodecode.lib.auth import HasRepoPermissionAny, HasReposGroupPermissionAny,\
49 from rhodecode.lib.auth import HasRepoPermissionAny, HasReposGroupPermissionAny,\
50 HasUserGroupPermissionAny
50 HasUserGroupPermissionAny
51 from rhodecode.lib.utils import get_filesystem_repos, make_ui, \
51 from rhodecode.lib.utils import get_filesystem_repos, make_ui, \
52 action_logger
52 action_logger
53 from rhodecode.model import BaseModel
53 from rhodecode.model import BaseModel
54 from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \
54 from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \
55 UserFollowing, UserLog, User, RepoGroup, PullRequest
55 UserFollowing, UserLog, User, RepoGroup, PullRequest
56 from rhodecode.lib.hooks import log_push_action
56 from rhodecode.lib.hooks import log_push_action
57 from rhodecode.lib.exceptions import NonRelativePathError
57 from rhodecode.lib.exceptions import NonRelativePathError
58
58
59 log = logging.getLogger(__name__)
59 log = logging.getLogger(__name__)
60
60
61
61
62 class UserTemp(object):
62 class UserTemp(object):
63 def __init__(self, user_id):
63 def __init__(self, user_id):
64 self.user_id = user_id
64 self.user_id = user_id
65
65
66 def __repr__(self):
66 def __repr__(self):
67 return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
67 return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
68
68
69
69
70 class RepoTemp(object):
70 class RepoTemp(object):
71 def __init__(self, repo_id):
71 def __init__(self, repo_id):
72 self.repo_id = repo_id
72 self.repo_id = repo_id
73
73
74 def __repr__(self):
74 def __repr__(self):
75 return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
75 return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
76
76
77
77
78 class CachedRepoList(object):
78 class CachedRepoList(object):
79 """
79 """
80 Cached repo list, uses in-memory cache after initialization, that is
80 Cached repo list, uses in-memory cache after initialization, that is
81 super fast
81 super fast
82 """
82 """
83
83
84 def __init__(self, db_repo_list, repos_path, order_by=None, perm_set=None):
84 def __init__(self, db_repo_list, repos_path, order_by=None, perm_set=None):
85 self.db_repo_list = db_repo_list
85 self.db_repo_list = db_repo_list
86 self.repos_path = repos_path
86 self.repos_path = repos_path
87 self.order_by = order_by
87 self.order_by = order_by
88 self.reversed = (order_by or '').startswith('-')
88 self.reversed = (order_by or '').startswith('-')
89 if not perm_set:
89 if not perm_set:
90 perm_set = ['repository.read', 'repository.write',
90 perm_set = ['repository.read', 'repository.write',
91 'repository.admin']
91 'repository.admin']
92 self.perm_set = perm_set
92 self.perm_set = perm_set
93
93
94 def __len__(self):
94 def __len__(self):
95 return len(self.db_repo_list)
95 return len(self.db_repo_list)
96
96
97 def __repr__(self):
97 def __repr__(self):
98 return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
98 return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
99
99
100 def __iter__(self):
100 def __iter__(self):
101 # pre-propagated valid_cache_keys to save executing select statements
101 # pre-propagated valid_cache_keys to save executing select statements
102 # for each repo
102 # for each repo
103 valid_cache_keys = CacheInvalidation.get_valid_cache_keys()
103 valid_cache_keys = CacheInvalidation.get_valid_cache_keys()
104
104
105 for dbr in self.db_repo_list:
105 for dbr in self.db_repo_list:
106 scmr = dbr.scm_instance_cached(valid_cache_keys)
106 scmr = dbr.scm_instance_cached(valid_cache_keys)
107 # check permission at this level
107 # check permission at this level
108 if not HasRepoPermissionAny(
108 if not HasRepoPermissionAny(
109 *self.perm_set)(dbr.repo_name, 'get repo check'):
109 *self.perm_set)(dbr.repo_name, 'get repo check'):
110 continue
110 continue
111
111
112 try:
112 try:
113 last_change = scmr.last_change
113 last_change = scmr.last_change
114 tip = h.get_changeset_safe(scmr, 'tip')
114 tip = h.get_changeset_safe(scmr, 'tip')
115 except Exception:
115 except Exception:
116 log.error(
116 log.error(
117 '%s this repository is present in database but it '
117 '%s this repository is present in database but it '
118 'cannot be created as an scm instance, org_exc:%s'
118 'cannot be created as an scm instance, org_exc:%s'
119 % (dbr.repo_name, traceback.format_exc())
119 % (dbr.repo_name, traceback.format_exc())
120 )
120 )
121 continue
121 continue
122
122
123 tmp_d = {}
123 tmp_d = {}
124 tmp_d['name'] = dbr.repo_name
124 tmp_d['name'] = dbr.repo_name
125 tmp_d['name_sort'] = tmp_d['name'].lower()
125 tmp_d['name_sort'] = tmp_d['name'].lower()
126 tmp_d['raw_name'] = tmp_d['name'].lower()
126 tmp_d['raw_name'] = tmp_d['name'].lower()
127 tmp_d['description'] = dbr.description
127 tmp_d['description'] = dbr.description
128 tmp_d['description_sort'] = tmp_d['description'].lower()
128 tmp_d['description_sort'] = tmp_d['description'].lower()
129 tmp_d['last_change'] = last_change
129 tmp_d['last_change'] = last_change
130 tmp_d['last_change_sort'] = time.mktime(last_change.timetuple())
130 tmp_d['last_change_sort'] = time.mktime(last_change.timetuple())
131 tmp_d['tip'] = tip.raw_id
131 tmp_d['tip'] = tip.raw_id
132 tmp_d['tip_sort'] = tip.revision
132 tmp_d['tip_sort'] = tip.revision
133 tmp_d['rev'] = tip.revision
133 tmp_d['rev'] = tip.revision
134 tmp_d['contact'] = dbr.user.full_contact
134 tmp_d['contact'] = dbr.user.full_contact
135 tmp_d['contact_sort'] = tmp_d['contact']
135 tmp_d['contact_sort'] = tmp_d['contact']
136 tmp_d['owner_sort'] = tmp_d['contact']
136 tmp_d['owner_sort'] = tmp_d['contact']
137 tmp_d['repo_archives'] = list(scmr._get_archives())
137 tmp_d['repo_archives'] = list(scmr._get_archives())
138 tmp_d['last_msg'] = tip.message
138 tmp_d['last_msg'] = tip.message
139 tmp_d['author'] = tip.author
139 tmp_d['author'] = tip.author
140 tmp_d['dbrepo'] = dbr.get_dict()
140 tmp_d['dbrepo'] = dbr.get_dict()
141 tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork else {}
141 tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork else {}
142 yield tmp_d
142 yield tmp_d
143
143
144
144
145 class SimpleCachedRepoList(CachedRepoList):
145 class SimpleCachedRepoList(CachedRepoList):
146 """
146 """
147 Lighter version of CachedRepoList without the scm initialisation
147 Lighter version of CachedRepoList without the scm initialisation
148 """
148 """
149
149
150 def __iter__(self):
150 def __iter__(self):
151 for dbr in self.db_repo_list:
151 for dbr in self.db_repo_list:
152 # check permission at this level
152 # check permission at this level
153 if not HasRepoPermissionAny(
153 if not HasRepoPermissionAny(
154 *self.perm_set)(dbr.repo_name, 'get repo check'):
154 *self.perm_set)(dbr.repo_name, 'get repo check'):
155 continue
155 continue
156
156
157 tmp_d = {}
157 tmp_d = {}
158 tmp_d['name'] = dbr.repo_name
158 tmp_d['name'] = dbr.repo_name
159 tmp_d['name_sort'] = tmp_d['name'].lower()
159 tmp_d['name_sort'] = tmp_d['name'].lower()
160 tmp_d['raw_name'] = tmp_d['name'].lower()
160 tmp_d['raw_name'] = tmp_d['name'].lower()
161 tmp_d['description'] = dbr.description
161 tmp_d['description'] = dbr.description
162 tmp_d['description_sort'] = tmp_d['description'].lower()
162 tmp_d['description_sort'] = tmp_d['description'].lower()
163 tmp_d['dbrepo'] = dbr.get_dict()
163 tmp_d['dbrepo'] = dbr.get_dict()
164 tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork else {}
164 tmp_d['dbrepo_fork'] = dbr.fork.get_dict() if dbr.fork else {}
165 yield tmp_d
165 yield tmp_d
166
166
167
167
168 class _PermCheckIterator(object):
168 class _PermCheckIterator(object):
169 def __init__(self, obj_list, obj_attr, perm_set, perm_checker):
169 def __init__(self, obj_list, obj_attr, perm_set, perm_checker):
170 """
170 """
171 Creates iterator from given list of objects, additionally
171 Creates iterator from given list of objects, additionally
172 checking permission for them from perm_set var
172 checking permission for them from perm_set var
173
173
174 :param obj_list: list of db objects
174 :param obj_list: list of db objects
175 :param obj_attr: attribute of object to pass into perm_checker
175 :param obj_attr: attribute of object to pass into perm_checker
176 :param perm_set: list of permissions to check
176 :param perm_set: list of permissions to check
177 :param perm_checker: callable to check permissions against
177 :param perm_checker: callable to check permissions against
178 """
178 """
179 self.obj_list = obj_list
179 self.obj_list = obj_list
180 self.obj_attr = obj_attr
180 self.obj_attr = obj_attr
181 self.perm_set = perm_set
181 self.perm_set = perm_set
182 self.perm_checker = perm_checker
182 self.perm_checker = perm_checker
183
183
184 def __len__(self):
184 def __len__(self):
185 return len(self.obj_list)
185 return len(self.obj_list)
186
186
187 def __repr__(self):
187 def __repr__(self):
188 return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
188 return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
189
189
190 def __iter__(self):
190 def __iter__(self):
191 for db_obj in self.obj_list:
191 for db_obj in self.obj_list:
192 # check permission at this level
192 # check permission at this level
193 name = getattr(db_obj, self.obj_attr, None)
193 name = getattr(db_obj, self.obj_attr, None)
194 if not self.perm_checker(*self.perm_set)(name, self.__class__.__name__):
194 if not self.perm_checker(*self.perm_set)(name, self.__class__.__name__):
195 continue
195 continue
196
196
197 yield db_obj
197 yield db_obj
198
198
199
199
200 class RepoList(_PermCheckIterator):
200 class RepoList(_PermCheckIterator):
201
201
202 def __init__(self, db_repo_list, perm_set=None):
202 def __init__(self, db_repo_list, perm_set=None):
203 if not perm_set:
203 if not perm_set:
204 perm_set = ['repository.read', 'repository.write', 'repository.admin']
204 perm_set = ['repository.read', 'repository.write', 'repository.admin']
205
205
206 super(RepoList, self).__init__(obj_list=db_repo_list,
206 super(RepoList, self).__init__(obj_list=db_repo_list,
207 obj_attr='repo_name', perm_set=perm_set,
207 obj_attr='repo_name', perm_set=perm_set,
208 perm_checker=HasRepoPermissionAny)
208 perm_checker=HasRepoPermissionAny)
209
209
210
210
211 class RepoGroupList(_PermCheckIterator):
211 class RepoGroupList(_PermCheckIterator):
212
212
213 def __init__(self, db_repo_group_list, perm_set=None):
213 def __init__(self, db_repo_group_list, perm_set=None):
214 if not perm_set:
214 if not perm_set:
215 perm_set = ['group.read', 'group.write', 'group.admin']
215 perm_set = ['group.read', 'group.write', 'group.admin']
216
216
217 super(RepoGroupList, self).__init__(obj_list=db_repo_group_list,
217 super(RepoGroupList, self).__init__(obj_list=db_repo_group_list,
218 obj_attr='group_name', perm_set=perm_set,
218 obj_attr='group_name', perm_set=perm_set,
219 perm_checker=HasReposGroupPermissionAny)
219 perm_checker=HasReposGroupPermissionAny)
220
220
221
221
222 class UserGroupList(_PermCheckIterator):
222 class UserGroupList(_PermCheckIterator):
223
223
224 def __init__(self, db_user_group_list, perm_set=None):
224 def __init__(self, db_user_group_list, perm_set=None):
225 if not perm_set:
225 if not perm_set:
226 perm_set = ['usergroup.read', 'usergroup.write', 'usergroup.admin']
226 perm_set = ['usergroup.read', 'usergroup.write', 'usergroup.admin']
227
227
228 super(UserGroupList, self).__init__(obj_list=db_user_group_list,
228 super(UserGroupList, self).__init__(obj_list=db_user_group_list,
229 obj_attr='users_group_name', perm_set=perm_set,
229 obj_attr='users_group_name', perm_set=perm_set,
230 perm_checker=HasUserGroupPermissionAny)
230 perm_checker=HasUserGroupPermissionAny)
231
231
232
232
233 class ScmModel(BaseModel):
233 class ScmModel(BaseModel):
234 """
234 """
235 Generic Scm Model
235 Generic Scm Model
236 """
236 """
237
237
238 def __get_repo(self, instance):
238 def __get_repo(self, instance):
239 cls = Repository
239 cls = Repository
240 if isinstance(instance, cls):
240 if isinstance(instance, cls):
241 return instance
241 return instance
242 elif isinstance(instance, int) or safe_str(instance).isdigit():
242 elif isinstance(instance, int) or safe_str(instance).isdigit():
243 return cls.get(instance)
243 return cls.get(instance)
244 elif isinstance(instance, basestring):
244 elif isinstance(instance, basestring):
245 return cls.get_by_repo_name(instance)
245 return cls.get_by_repo_name(instance)
246 elif instance:
246 elif instance:
247 raise Exception('given object must be int, basestr or Instance'
247 raise Exception('given object must be int, basestr or Instance'
248 ' of %s got %s' % (type(cls), type(instance)))
248 ' of %s got %s' % (type(cls), type(instance)))
249
249
250 @LazyProperty
250 @LazyProperty
251 def repos_path(self):
251 def repos_path(self):
252 """
252 """
253 Get's the repositories root path from database
253 Get's the repositories root path from database
254 """
254 """
255
255
256 q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
256 q = self.sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()
257
257
258 return q.ui_value
258 return q.ui_value
259
259
260 def repo_scan(self, repos_path=None):
260 def repo_scan(self, repos_path=None):
261 """
261 """
262 Listing of repositories in given path. This path should not be a
262 Listing of repositories in given path. This path should not be a
263 repository itself. Return a dictionary of repository objects
263 repository itself. Return a dictionary of repository objects
264
264
265 :param repos_path: path to directory containing repositories
265 :param repos_path: path to directory containing repositories
266 """
266 """
267
267
268 if repos_path is None:
268 if repos_path is None:
269 repos_path = self.repos_path
269 repos_path = self.repos_path
270
270
271 log.info('scanning for repositories in %s' % repos_path)
271 log.info('scanning for repositories in %s' % repos_path)
272
272
273 baseui = make_ui('db')
273 baseui = make_ui('db')
274 repos = {}
274 repos = {}
275
275
276 for name, path in get_filesystem_repos(repos_path, recursive=True):
276 for name, path in get_filesystem_repos(repos_path, recursive=True):
277 # name need to be decomposed and put back together using the /
277 # name need to be decomposed and put back together using the /
278 # since this is internal storage separator for rhodecode
278 # since this is internal storage separator for rhodecode
279 name = Repository.normalize_repo_name(name)
279 name = Repository.normalize_repo_name(name)
280
280
281 try:
281 try:
282 if name in repos:
282 if name in repos:
283 raise RepositoryError('Duplicate repository name %s '
283 raise RepositoryError('Duplicate repository name %s '
284 'found in %s' % (name, path))
284 'found in %s' % (name, path))
285 else:
285 else:
286
286
287 klass = get_backend(path[0])
287 klass = get_backend(path[0])
288
288
289 if path[0] == 'hg' and path[0] in BACKENDS.keys():
289 if path[0] == 'hg' and path[0] in BACKENDS.keys():
290 repos[name] = klass(safe_str(path[1]), baseui=baseui)
290 repos[name] = klass(safe_str(path[1]), baseui=baseui)
291
291
292 if path[0] == 'git' and path[0] in BACKENDS.keys():
292 if path[0] == 'git' and path[0] in BACKENDS.keys():
293 repos[name] = klass(path[1])
293 repos[name] = klass(path[1])
294 except OSError:
294 except OSError:
295 continue
295 continue
296 log.debug('found %s paths with repositories' % (len(repos)))
296 log.debug('found %s paths with repositories' % (len(repos)))
297 return repos
297 return repos
298
298
299 def get_repos(self, all_repos=None, sort_key=None, simple=False):
299 def get_repos(self, all_repos=None, sort_key=None, simple=False):
300 """
300 """
301 Get all repos from db and for each repo create it's
301 Get all repos from db and for each repo create it's
302 backend instance and fill that backed with information from database
302 backend instance and fill that backed with information from database
303
303
304 :param all_repos: list of repository names as strings
304 :param all_repos: list of repository names as strings
305 give specific repositories list, good for filtering
305 give specific repositories list, good for filtering
306
306
307 :param sort_key: initial sorting of repos
307 :param sort_key: initial sorting of repos
308 :param simple: use SimpleCachedList - one without the SCM info
308 :param simple: use SimpleCachedList - one without the SCM info
309 """
309 """
310 if all_repos is None:
310 if all_repos is None:
311 all_repos = self.sa.query(Repository)\
311 all_repos = self.sa.query(Repository)\
312 .filter(Repository.group_id == None)\
312 .filter(Repository.group_id == None)\
313 .order_by(func.lower(Repository.repo_name)).all()
313 .order_by(func.lower(Repository.repo_name)).all()
314 if simple:
314 if simple:
315 repo_iter = SimpleCachedRepoList(all_repos,
315 repo_iter = SimpleCachedRepoList(all_repos,
316 repos_path=self.repos_path,
316 repos_path=self.repos_path,
317 order_by=sort_key)
317 order_by=sort_key)
318 else:
318 else:
319 repo_iter = CachedRepoList(all_repos,
319 repo_iter = CachedRepoList(all_repos,
320 repos_path=self.repos_path,
320 repos_path=self.repos_path,
321 order_by=sort_key)
321 order_by=sort_key)
322
322
323 return repo_iter
323 return repo_iter
324
324
325 def get_repos_groups(self, all_groups=None):
325 def get_repos_groups(self, all_groups=None):
326 if all_groups is None:
326 if all_groups is None:
327 all_groups = RepoGroup.query()\
327 all_groups = RepoGroup.query()\
328 .filter(RepoGroup.group_parent_id == None).all()
328 .filter(RepoGroup.group_parent_id == None).all()
329 return [x for x in RepoGroupList(all_groups)]
329 return [x for x in RepoGroupList(all_groups)]
330
330
331 def mark_for_invalidation(self, repo_name):
331 def mark_for_invalidation(self, repo_name):
332 """
332 """
333 Mark caches of this repo invalid in the database.
333 Mark caches of this repo invalid in the database.
334
334
335 :param repo_name: the repo for which caches should be marked invalid
335 :param repo_name: the repo for which caches should be marked invalid
336 """
336 """
337 CacheInvalidation.set_invalidate(repo_name)
337 CacheInvalidation.set_invalidate(repo_name)
338 repo = Repository.get_by_repo_name(repo_name)
338 repo = Repository.get_by_repo_name(repo_name)
339 if repo:
339 if repo:
340 repo.update_changeset_cache()
340 repo.update_changeset_cache()
341
341
342 def toggle_following_repo(self, follow_repo_id, user_id):
342 def toggle_following_repo(self, follow_repo_id, user_id):
343
343
344 f = self.sa.query(UserFollowing)\
344 f = self.sa.query(UserFollowing)\
345 .filter(UserFollowing.follows_repo_id == follow_repo_id)\
345 .filter(UserFollowing.follows_repo_id == follow_repo_id)\
346 .filter(UserFollowing.user_id == user_id).scalar()
346 .filter(UserFollowing.user_id == user_id).scalar()
347
347
348 if f is not None:
348 if f is not None:
349 try:
349 try:
350 self.sa.delete(f)
350 self.sa.delete(f)
351 action_logger(UserTemp(user_id),
351 action_logger(UserTemp(user_id),
352 'stopped_following_repo',
352 'stopped_following_repo',
353 RepoTemp(follow_repo_id))
353 RepoTemp(follow_repo_id))
354 return
354 return
355 except Exception:
355 except Exception:
356 log.error(traceback.format_exc())
356 log.error(traceback.format_exc())
357 raise
357 raise
358
358
359 try:
359 try:
360 f = UserFollowing()
360 f = UserFollowing()
361 f.user_id = user_id
361 f.user_id = user_id
362 f.follows_repo_id = follow_repo_id
362 f.follows_repo_id = follow_repo_id
363 self.sa.add(f)
363 self.sa.add(f)
364
364
365 action_logger(UserTemp(user_id),
365 action_logger(UserTemp(user_id),
366 'started_following_repo',
366 'started_following_repo',
367 RepoTemp(follow_repo_id))
367 RepoTemp(follow_repo_id))
368 except Exception:
368 except Exception:
369 log.error(traceback.format_exc())
369 log.error(traceback.format_exc())
370 raise
370 raise
371
371
372 def toggle_following_user(self, follow_user_id, user_id):
372 def toggle_following_user(self, follow_user_id, user_id):
373 f = self.sa.query(UserFollowing)\
373 f = self.sa.query(UserFollowing)\
374 .filter(UserFollowing.follows_user_id == follow_user_id)\
374 .filter(UserFollowing.follows_user_id == follow_user_id)\
375 .filter(UserFollowing.user_id == user_id).scalar()
375 .filter(UserFollowing.user_id == user_id).scalar()
376
376
377 if f is not None:
377 if f is not None:
378 try:
378 try:
379 self.sa.delete(f)
379 self.sa.delete(f)
380 return
380 return
381 except Exception:
381 except Exception:
382 log.error(traceback.format_exc())
382 log.error(traceback.format_exc())
383 raise
383 raise
384
384
385 try:
385 try:
386 f = UserFollowing()
386 f = UserFollowing()
387 f.user_id = user_id
387 f.user_id = user_id
388 f.follows_user_id = follow_user_id
388 f.follows_user_id = follow_user_id
389 self.sa.add(f)
389 self.sa.add(f)
390 except Exception:
390 except Exception:
391 log.error(traceback.format_exc())
391 log.error(traceback.format_exc())
392 raise
392 raise
393
393
394 def is_following_repo(self, repo_name, user_id, cache=False):
394 def is_following_repo(self, repo_name, user_id, cache=False):
395 r = self.sa.query(Repository)\
395 r = self.sa.query(Repository)\
396 .filter(Repository.repo_name == repo_name).scalar()
396 .filter(Repository.repo_name == repo_name).scalar()
397
397
398 f = self.sa.query(UserFollowing)\
398 f = self.sa.query(UserFollowing)\
399 .filter(UserFollowing.follows_repository == r)\
399 .filter(UserFollowing.follows_repository == r)\
400 .filter(UserFollowing.user_id == user_id).scalar()
400 .filter(UserFollowing.user_id == user_id).scalar()
401
401
402 return f is not None
402 return f is not None
403
403
404 def is_following_user(self, username, user_id, cache=False):
404 def is_following_user(self, username, user_id, cache=False):
405 u = User.get_by_username(username)
405 u = User.get_by_username(username)
406
406
407 f = self.sa.query(UserFollowing)\
407 f = self.sa.query(UserFollowing)\
408 .filter(UserFollowing.follows_user == u)\
408 .filter(UserFollowing.follows_user == u)\
409 .filter(UserFollowing.user_id == user_id).scalar()
409 .filter(UserFollowing.user_id == user_id).scalar()
410
410
411 return f is not None
411 return f is not None
412
412
413 def get_followers(self, repo):
413 def get_followers(self, repo):
414 repo = self._get_repo(repo)
414 repo = self._get_repo(repo)
415
415
416 return self.sa.query(UserFollowing)\
416 return self.sa.query(UserFollowing)\
417 .filter(UserFollowing.follows_repository == repo).count()
417 .filter(UserFollowing.follows_repository == repo).count()
418
418
419 def get_forks(self, repo):
419 def get_forks(self, repo):
420 repo = self._get_repo(repo)
420 repo = self._get_repo(repo)
421 return self.sa.query(Repository)\
421 return self.sa.query(Repository)\
422 .filter(Repository.fork == repo).count()
422 .filter(Repository.fork == repo).count()
423
423
424 def get_pull_requests(self, repo):
424 def get_pull_requests(self, repo):
425 repo = self._get_repo(repo)
425 repo = self._get_repo(repo)
426 return self.sa.query(PullRequest)\
426 return self.sa.query(PullRequest)\
427 .filter(PullRequest.other_repo == repo)\
427 .filter(PullRequest.other_repo == repo)\
428 .filter(PullRequest.status != PullRequest.STATUS_CLOSED).count()
428 .filter(PullRequest.status != PullRequest.STATUS_CLOSED).count()
429
429
430 def mark_as_fork(self, repo, fork, user):
430 def mark_as_fork(self, repo, fork, user):
431 repo = self.__get_repo(repo)
431 repo = self.__get_repo(repo)
432 fork = self.__get_repo(fork)
432 fork = self.__get_repo(fork)
433 if fork and repo.repo_id == fork.repo_id:
433 if fork and repo.repo_id == fork.repo_id:
434 raise Exception("Cannot set repository as fork of itself")
434 raise Exception("Cannot set repository as fork of itself")
435 repo.fork = fork
435 repo.fork = fork
436 self.sa.add(repo)
436 self.sa.add(repo)
437 return repo
437 return repo
438
438
439 def _handle_rc_scm_extras(self, username, repo_name, repo_alias,
439 def _handle_rc_scm_extras(self, username, repo_name, repo_alias,
440 action=None):
440 action=None):
441 from rhodecode import CONFIG
441 from rhodecode import CONFIG
442 from rhodecode.lib.base import _get_ip_addr
442 from rhodecode.lib.base import _get_ip_addr
443 try:
443 try:
444 from pylons import request
444 from pylons import request
445 environ = request.environ
445 environ = request.environ
446 except TypeError:
446 except TypeError:
447 # we might use this outside of request context, let's fake the
447 # we might use this outside of request context, let's fake the
448 # environ data
448 # environ data
449 from webob import Request
449 from webob import Request
450 environ = Request.blank('').environ
450 environ = Request.blank('').environ
451 extras = {
451 extras = {
452 'ip': _get_ip_addr(environ),
452 'ip': _get_ip_addr(environ),
453 'username': username,
453 'username': username,
454 'action': action or 'push_local',
454 'action': action or 'push_local',
455 'repository': repo_name,
455 'repository': repo_name,
456 'scm': repo_alias,
456 'scm': repo_alias,
457 'config': CONFIG['__file__'],
457 'config': CONFIG['__file__'],
458 'server_url': get_server_url(environ),
458 'server_url': get_server_url(environ),
459 'make_lock': None,
459 'make_lock': None,
460 'locked_by': [None, None]
460 'locked_by': [None, None]
461 }
461 }
462 _set_extras(extras)
462 _set_extras(extras)
463
463
464 def _handle_push(self, repo, username, action, repo_name, revisions):
464 def _handle_push(self, repo, username, action, repo_name, revisions):
465 """
465 """
466 Triggers push action hooks
466 Triggers push action hooks
467
467
468 :param repo: SCM repo
468 :param repo: SCM repo
469 :param username: username who pushes
469 :param username: username who pushes
470 :param action: push/push_loca/push_remote
470 :param action: push/push_loca/push_remote
471 :param repo_name: name of repo
471 :param repo_name: name of repo
472 :param revisions: list of revisions that we pushed
472 :param revisions: list of revisions that we pushed
473 """
473 """
474 self._handle_rc_scm_extras(username, repo_name, repo_alias=repo.alias)
474 self._handle_rc_scm_extras(username, repo_name, repo_alias=repo.alias)
475 _scm_repo = repo._repo
475 _scm_repo = repo._repo
476 # trigger push hook
476 # trigger push hook
477 if repo.alias == 'hg':
477 if repo.alias == 'hg':
478 log_push_action(_scm_repo.ui, _scm_repo, node=revisions[0])
478 log_push_action(_scm_repo.ui, _scm_repo, node=revisions[0])
479 elif repo.alias == 'git':
479 elif repo.alias == 'git':
480 log_push_action(None, _scm_repo, _git_revs=revisions)
480 log_push_action(None, _scm_repo, _git_revs=revisions)
481
481
482 def _get_IMC_module(self, scm_type):
482 def _get_IMC_module(self, scm_type):
483 """
483 """
484 Returns InMemoryCommit class based on scm_type
484 Returns InMemoryCommit class based on scm_type
485
485
486 :param scm_type:
486 :param scm_type:
487 """
487 """
488 if scm_type == 'hg':
488 if scm_type == 'hg':
489 from rhodecode.lib.vcs.backends.hg import MercurialInMemoryChangeset
489 from rhodecode.lib.vcs.backends.hg import MercurialInMemoryChangeset
490 return MercurialInMemoryChangeset
490 return MercurialInMemoryChangeset
491
491
492 if scm_type == 'git':
492 if scm_type == 'git':
493 from rhodecode.lib.vcs.backends.git import GitInMemoryChangeset
493 from rhodecode.lib.vcs.backends.git import GitInMemoryChangeset
494 return GitInMemoryChangeset
494 return GitInMemoryChangeset
495
495
496 raise Exception('Invalid scm_type, must be one of hg,git got %s'
496 raise Exception('Invalid scm_type, must be one of hg,git got %s'
497 % (scm_type,))
497 % (scm_type,))
498
498
499 def pull_changes(self, repo, username):
499 def pull_changes(self, repo, username):
500 dbrepo = self.__get_repo(repo)
500 dbrepo = self.__get_repo(repo)
501 clone_uri = dbrepo.clone_uri
501 clone_uri = dbrepo.clone_uri
502 if not clone_uri:
502 if not clone_uri:
503 raise Exception("This repository doesn't have a clone uri")
503 raise Exception("This repository doesn't have a clone uri")
504
504
505 repo = dbrepo.scm_instance
505 repo = dbrepo.scm_instance
506 repo_name = dbrepo.repo_name
506 repo_name = dbrepo.repo_name
507 try:
507 try:
508 if repo.alias == 'git':
508 if repo.alias == 'git':
509 repo.fetch(clone_uri)
509 repo.fetch(clone_uri)
510 # git doesn't really have something like post-fetch action
510 # git doesn't really have something like post-fetch action
511 # we fake that now. #TODO: extract fetched revisions somehow
511 # we fake that now. #TODO: extract fetched revisions somehow
512 # here
512 # here
513 self._handle_push(repo,
513 self._handle_push(repo,
514 username=username,
514 username=username,
515 action='push_remote',
515 action='push_remote',
516 repo_name=repo_name,
516 repo_name=repo_name,
517 revisions=[])
517 revisions=[])
518 else:
518 else:
519 self._handle_rc_scm_extras(username, dbrepo.repo_name,
519 self._handle_rc_scm_extras(username, dbrepo.repo_name,
520 repo.alias, action='push_remote')
520 repo.alias, action='push_remote')
521 repo.pull(clone_uri)
521 repo.pull(clone_uri)
522
522
523 self.mark_for_invalidation(repo_name)
523 self.mark_for_invalidation(repo_name)
524 except Exception:
524 except Exception:
525 log.error(traceback.format_exc())
525 log.error(traceback.format_exc())
526 raise
526 raise
527
527
528 def commit_change(self, repo, repo_name, cs, user, author, message,
528 def commit_change(self, repo, repo_name, cs, user, author, message,
529 content, f_path):
529 content, f_path):
530 """
530 """
531 Commits changes
531 Commits changes
532
532
533 :param repo: SCM instance
533 :param repo: SCM instance
534
534
535 """
535 """
536 user = self._get_user(user)
536 user = self._get_user(user)
537 IMC = self._get_IMC_module(repo.alias)
537 IMC = self._get_IMC_module(repo.alias)
538
538
539 # decoding here will force that we have proper encoded values
539 # decoding here will force that we have proper encoded values
540 # in any other case this will throw exceptions and deny commit
540 # in any other case this will throw exceptions and deny commit
541 content = safe_str(content)
541 content = safe_str(content)
542 path = safe_str(f_path)
542 path = safe_str(f_path)
543 # message and author needs to be unicode
543 # message and author needs to be unicode
544 # proper backend should then translate that into required type
544 # proper backend should then translate that into required type
545 message = safe_unicode(message)
545 message = safe_unicode(message)
546 author = safe_unicode(author)
546 author = safe_unicode(author)
547 imc = IMC(repo)
547 imc = IMC(repo)
548 imc.change(FileNode(path, content, mode=cs.get_file_mode(f_path)))
548 imc.change(FileNode(path, content, mode=cs.get_file_mode(f_path)))
549 tip = imc.commit(message=message,
549 tip = imc.commit(message=message,
550 author=author,
550 author=author,
551 parents=[cs], branch=cs.branch)
551 parents=[cs], branch=cs.branch)
552
552
553 self.mark_for_invalidation(repo_name)
553 self.mark_for_invalidation(repo_name)
554 self._handle_push(repo,
554 self._handle_push(repo,
555 username=user.username,
555 username=user.username,
556 action='push_local',
556 action='push_local',
557 repo_name=repo_name,
557 repo_name=repo_name,
558 revisions=[tip.raw_id])
558 revisions=[tip.raw_id])
559 return tip
559 return tip
560
560
561 def create_nodes(self, user, repo, message, nodes, parent_cs=None,
561 def create_nodes(self, user, repo, message, nodes, parent_cs=None,
562 author=None, trigger_push_hook=True):
562 author=None, trigger_push_hook=True):
563 """
563 """
564 Commits given multiple nodes into repo
564 Commits given multiple nodes into repo
565
565
566 :param user: RhodeCode User object or user_id, the commiter
566 :param user: RhodeCode User object or user_id, the commiter
567 :param repo: RhodeCode Repository object
567 :param repo: RhodeCode Repository object
568 :param message: commit message
568 :param message: commit message
569 :param nodes: mapping {filename:{'content':content},...}
569 :param nodes: mapping {filename:{'content':content},...}
570 :param parent_cs: parent changeset, can be empty than it's initial commit
570 :param parent_cs: parent changeset, can be empty than it's initial commit
571 :param author: author of commit, cna be different that commiter only for git
571 :param author: author of commit, cna be different that commiter only for git
572 :param trigger_push_hook: trigger push hooks
572 :param trigger_push_hook: trigger push hooks
573
573
574 :returns: new commited changeset
574 :returns: new commited changeset
575 """
575 """
576
576
577 user = self._get_user(user)
577 user = self._get_user(user)
578 scm_instance = repo.scm_instance_no_cache()
578 scm_instance = repo.scm_instance_no_cache()
579
579
580 processed_nodes = []
580 processed_nodes = []
581 for f_path in nodes:
581 for f_path in nodes:
582 if f_path.startswith('/') or f_path.startswith('.') or '../' in f_path:
582 if f_path.startswith('/') or f_path.startswith('.') or '../' in f_path:
583 raise NonRelativePathError('%s is not an relative path' % f_path)
583 raise NonRelativePathError('%s is not an relative path' % f_path)
584 if f_path:
584 if f_path:
585 f_path = os.path.normpath(f_path)
585 f_path = os.path.normpath(f_path)
586 content = nodes[f_path]['content']
586 content = nodes[f_path]['content']
587 f_path = safe_str(f_path)
587 f_path = safe_str(f_path)
588 # decoding here will force that we have proper encoded values
588 # decoding here will force that we have proper encoded values
589 # in any other case this will throw exceptions and deny commit
589 # in any other case this will throw exceptions and deny commit
590 if isinstance(content, (basestring,)):
590 if isinstance(content, (basestring,)):
591 content = safe_str(content)
591 content = safe_str(content)
592 elif isinstance(content, (file, cStringIO.OutputType,)):
592 elif isinstance(content, (file, cStringIO.OutputType,)):
593 content = content.read()
593 content = content.read()
594 else:
594 else:
595 raise Exception('Content is of unrecognized type %s' % (
595 raise Exception('Content is of unrecognized type %s' % (
596 type(content)
596 type(content)
597 ))
597 ))
598 processed_nodes.append((f_path, content))
598 processed_nodes.append((f_path, content))
599
599
600 message = safe_unicode(message)
600 message = safe_unicode(message)
601 commiter = user.full_contact
601 commiter = user.full_contact
602 author = safe_unicode(author) if author else commiter
602 author = safe_unicode(author) if author else commiter
603
603
604 IMC = self._get_IMC_module(scm_instance.alias)
604 IMC = self._get_IMC_module(scm_instance.alias)
605 imc = IMC(scm_instance)
605 imc = IMC(scm_instance)
606
606
607 if not parent_cs:
607 if not parent_cs:
608 parent_cs = EmptyChangeset(alias=scm_instance.alias)
608 parent_cs = EmptyChangeset(alias=scm_instance.alias)
609
609
610 if isinstance(parent_cs, EmptyChangeset):
610 if isinstance(parent_cs, EmptyChangeset):
611 # EmptyChangeset means we we're editing empty repository
611 # EmptyChangeset means we we're editing empty repository
612 parents = None
612 parents = None
613 else:
613 else:
614 parents = [parent_cs]
614 parents = [parent_cs]
615 # add multiple nodes
615 # add multiple nodes
616 for path, content in processed_nodes:
616 for path, content in processed_nodes:
617 imc.add(FileNode(path, content=content))
617 imc.add(FileNode(path, content=content))
618
618
619 tip = imc.commit(message=message,
619 tip = imc.commit(message=message,
620 author=author,
620 author=author,
621 parents=parents,
621 parents=parents,
622 branch=parent_cs.branch)
622 branch=parent_cs.branch)
623
623
624 self.mark_for_invalidation(repo.repo_name)
624 self.mark_for_invalidation(repo.repo_name)
625 if trigger_push_hook:
625 if trigger_push_hook:
626 self._handle_push(scm_instance,
626 self._handle_push(scm_instance,
627 username=user.username,
627 username=user.username,
628 action='push_local',
628 action='push_local',
629 repo_name=repo.repo_name,
629 repo_name=repo.repo_name,
630 revisions=[tip.raw_id])
630 revisions=[tip.raw_id])
631 return tip
631 return tip
632
632
633 def get_nodes(self, repo_name, revision, root_path='/', flat=True):
633 def get_nodes(self, repo_name, revision, root_path='/', flat=True):
634 """
634 """
635 recursive walk in root dir and return a set of all path in that dir
635 recursive walk in root dir and return a set of all path in that dir
636 based on repository walk function
636 based on repository walk function
637
637
638 :param repo_name: name of repository
638 :param repo_name: name of repository
639 :param revision: revision for which to list nodes
639 :param revision: revision for which to list nodes
640 :param root_path: root path to list
640 :param root_path: root path to list
641 :param flat: return as a list, if False returns a dict with decription
641 :param flat: return as a list, if False returns a dict with decription
642
642
643 """
643 """
644 _files = list()
644 _files = list()
645 _dirs = list()
645 _dirs = list()
646 try:
646 try:
647 _repo = self.__get_repo(repo_name)
647 _repo = self.__get_repo(repo_name)
648 changeset = _repo.scm_instance.get_changeset(revision)
648 changeset = _repo.scm_instance.get_changeset(revision)
649 root_path = root_path.lstrip('/')
649 root_path = root_path.lstrip('/')
650 for topnode, dirs, files in changeset.walk(root_path):
650 for topnode, dirs, files in changeset.walk(root_path):
651 for f in files:
651 for f in files:
652 _files.append(f.path if flat else {"name": f.path,
652 _files.append(f.path if flat else {"name": f.path,
653 "type": "file"})
653 "type": "file"})
654 for d in dirs:
654 for d in dirs:
655 _dirs.append(d.path if flat else {"name": d.path,
655 _dirs.append(d.path if flat else {"name": d.path,
656 "type": "dir"})
656 "type": "dir"})
657 except RepositoryError:
657 except RepositoryError:
658 log.debug(traceback.format_exc())
658 log.debug(traceback.format_exc())
659 raise
659 raise
660
660
661 return _dirs, _files
661 return _dirs, _files
662
662
663 def get_unread_journal(self):
663 def get_unread_journal(self):
664 return self.sa.query(UserLog).count()
664 return self.sa.query(UserLog).count()
665
665
666 def get_repo_landing_revs(self, repo=None):
666 def get_repo_landing_revs(self, repo=None):
667 """
667 """
668 Generates select option with tags branches and bookmarks (for hg only)
668 Generates select option with tags branches and bookmarks (for hg only)
669 grouped by type
669 grouped by type
670
670
671 :param repo:
671 :param repo:
672 """
672 """
673
673
674 hist_l = []
674 hist_l = []
675 choices = []
675 choices = []
676 repo = self.__get_repo(repo)
676 repo = self.__get_repo(repo)
677 hist_l.append(['tip', _('latest tip')])
677 hist_l.append(['tip', _('latest tip')])
678 choices.append('tip')
678 choices.append('tip')
679 if not repo:
679 if not repo:
680 return choices, hist_l
680 return choices, hist_l
681
681
682 repo = repo.scm_instance
682 repo = repo.scm_instance
683
683
684 branches_group = ([(k, k) for k, v in
684 branches_group = ([(k, k) for k, v in
685 repo.branches.iteritems()], _("Branches"))
685 repo.branches.iteritems()], _("Branches"))
686 hist_l.append(branches_group)
686 hist_l.append(branches_group)
687 choices.extend([x[0] for x in branches_group[0]])
687 choices.extend([x[0] for x in branches_group[0]])
688
688
689 if repo.alias == 'hg':
689 if repo.alias == 'hg':
690 bookmarks_group = ([(k, k) for k, v in
690 bookmarks_group = ([(k, k) for k, v in
691 repo.bookmarks.iteritems()], _("Bookmarks"))
691 repo.bookmarks.iteritems()], _("Bookmarks"))
692 hist_l.append(bookmarks_group)
692 hist_l.append(bookmarks_group)
693 choices.extend([x[0] for x in bookmarks_group[0]])
693 choices.extend([x[0] for x in bookmarks_group[0]])
694
694
695 tags_group = ([(k, k) for k, v in
695 tags_group = ([(k, k) for k, v in
696 repo.tags.iteritems()], _("Tags"))
696 repo.tags.iteritems()], _("Tags"))
697 hist_l.append(tags_group)
697 hist_l.append(tags_group)
698 choices.extend([x[0] for x in tags_group[0]])
698 choices.extend([x[0] for x in tags_group[0]])
699
699
700 return choices, hist_l
700 return choices, hist_l
701
701
702 def install_git_hook(self, repo, force_create=False):
702 def install_git_hook(self, repo, force_create=False):
703 """
703 """
704 Creates a rhodecode hook inside a git repository
704 Creates a rhodecode hook inside a git repository
705
705
706 :param repo: Instance of VCS repo
706 :param repo: Instance of VCS repo
707 :param force_create: Create even if same name hook exists
707 :param force_create: Create even if same name hook exists
708 """
708 """
709
709
710 loc = jn(repo.path, 'hooks')
710 loc = jn(repo.path, 'hooks')
711 if not repo.bare:
711 if not repo.bare:
712 loc = jn(repo.path, '.git', 'hooks')
712 loc = jn(repo.path, '.git', 'hooks')
713 if not os.path.isdir(loc):
713 if not os.path.isdir(loc):
714 os.makedirs(loc)
714 os.makedirs(loc)
715
715
716 tmpl_post = pkg_resources.resource_string(
716 tmpl_post = pkg_resources.resource_string(
717 'rhodecode', jn('config', 'post_receive_tmpl.py')
717 'rhodecode', jn('config', 'post_receive_tmpl.py')
718 )
718 )
719 tmpl_pre = pkg_resources.resource_string(
719 tmpl_pre = pkg_resources.resource_string(
720 'rhodecode', jn('config', 'pre_receive_tmpl.py')
720 'rhodecode', jn('config', 'pre_receive_tmpl.py')
721 )
721 )
722
722
723 for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:
723 for h_type, tmpl in [('pre', tmpl_pre), ('post', tmpl_post)]:
724 _hook_file = jn(loc, '%s-receive' % h_type)
724 _hook_file = jn(loc, '%s-receive' % h_type)
725 _rhodecode_hook = False
725 _rhodecode_hook = False
726 log.debug('Installing git hook in repo %s' % repo)
726 log.debug('Installing git hook in repo %s' % repo)
727 if os.path.exists(_hook_file):
727 if os.path.exists(_hook_file):
728 # let's take a look at this hook, maybe it's rhodecode ?
728 # let's take a look at this hook, maybe it's rhodecode ?
729 log.debug('hook exists, checking if it is from rhodecode')
729 log.debug('hook exists, checking if it is from rhodecode')
730 with open(_hook_file, 'rb') as f:
730 with open(_hook_file, 'rb') as f:
731 data = f.read()
731 data = f.read()
732 matches = re.compile(r'(?:%s)\s*=\s*(.*)'
732 matches = re.compile(r'(?:%s)\s*=\s*(.*)'
733 % 'RC_HOOK_VER').search(data)
733 % 'RC_HOOK_VER').search(data)
734 if matches:
734 if matches:
735 try:
735 try:
736 ver = matches.groups()[0]
736 ver = matches.groups()[0]
737 log.debug('got %s it is rhodecode' % (ver))
737 log.debug('got %s it is rhodecode' % (ver))
738 _rhodecode_hook = True
738 _rhodecode_hook = True
739 except Exception:
739 except Exception:
740 log.error(traceback.format_exc())
740 log.error(traceback.format_exc())
741 else:
741 else:
742 # there is no hook in this dir, so we want to create one
742 # there is no hook in this dir, so we want to create one
743 _rhodecode_hook = True
743 _rhodecode_hook = True
744
744
745 if _rhodecode_hook or force_create:
745 if _rhodecode_hook or force_create:
746 log.debug('writing %s hook file !' % (h_type,))
746 log.debug('writing %s hook file !' % (h_type,))
747 with open(_hook_file, 'wb') as f:
747 try:
748 tmpl = tmpl.replace('_TMPL_', rhodecode.__version__)
748 with open(_hook_file, 'wb') as f:
749 f.write(tmpl)
749 tmpl = tmpl.replace('_TMPL_', rhodecode.__version__)
750 os.chmod(_hook_file, 0755)
750 f.write(tmpl)
751 os.chmod(_hook_file, 0755)
752 except IOError, e:
753 log.error('error writing %s: %s' % (_hook_file, e))
751 else:
754 else:
752 log.debug('skipping writing hook file')
755 log.debug('skipping writing hook file')
General Comments 0
You need to be logged in to leave comments. Login now