##// END OF EJS Templates
caches: use .refresh() instead of .invalidate()...
marcink -
r2939:913f92bf default
parent child Browse files
Show More
@@ -1,242 +1,238 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2017-2018 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20 import pytz
21 21 import logging
22 22
23 23 from pyramid.view import view_config
24 24 from pyramid.response import Response
25 25 from webhelpers.feedgenerator import Rss201rev2Feed, Atom1Feed
26 26
27 27 from rhodecode.apps._base import RepoAppView
28 28 from rhodecode.lib import audit_logger
29 29 from rhodecode.lib import rc_cache
30 30 from rhodecode.lib import helpers as h
31 31 from rhodecode.lib.auth import (
32 32 LoginRequired, HasRepoPermissionAnyDecorator)
33 33 from rhodecode.lib.diffs import DiffProcessor, LimitedDiffContainer
34 34 from rhodecode.lib.utils2 import str2bool, safe_int, md5_safe
35 35 from rhodecode.model.db import UserApiKeys, CacheKey
36 36
37 37 log = logging.getLogger(__name__)
38 38
39 39
40 40 class RepoFeedView(RepoAppView):
41 41 def load_default_context(self):
42 42 c = self._get_local_tmpl_context()
43 43
44 44
45 45 self._load_defaults()
46 46 return c
47 47
48 48 def _get_config(self):
49 49 import rhodecode
50 50 config = rhodecode.CONFIG
51 51
52 52 return {
53 53 'language': 'en-us',
54 54 'feed_ttl': '5', # TTL of feed,
55 55 'feed_include_diff':
56 56 str2bool(config.get('rss_include_diff', False)),
57 57 'feed_items_per_page':
58 58 safe_int(config.get('rss_items_per_page', 20)),
59 59 'feed_diff_limit':
60 60 # we need to protect from parsing huge diffs here other way
61 61 # we can kill the server
62 62 safe_int(config.get('rss_cut_off_limit', 32 * 1024)),
63 63 }
64 64
65 65 def _load_defaults(self):
66 66 _ = self.request.translate
67 67 config = self._get_config()
68 68 # common values for feeds
69 69 self.description = _('Changes on %s repository')
70 70 self.title = self.title = _('%s %s feed') % (self.db_repo_name, '%s')
71 71 self.language = config["language"]
72 72 self.ttl = config["feed_ttl"]
73 73 self.feed_include_diff = config['feed_include_diff']
74 74 self.feed_diff_limit = config['feed_diff_limit']
75 75 self.feed_items_per_page = config['feed_items_per_page']
76 76
77 77 def _changes(self, commit):
78 78 diff_processor = DiffProcessor(
79 79 commit.diff(), diff_limit=self.feed_diff_limit)
80 80 _parsed = diff_processor.prepare(inline_diff=False)
81 81 limited_diff = isinstance(_parsed, LimitedDiffContainer)
82 82
83 83 return diff_processor, _parsed, limited_diff
84 84
85 85 def _get_title(self, commit):
86 86 return h.shorter(commit.message, 160)
87 87
88 88 def _get_description(self, commit):
89 89 _renderer = self.request.get_partial_renderer(
90 90 'rhodecode:templates/feed/atom_feed_entry.mako')
91 91 diff_processor, parsed_diff, limited_diff = self._changes(commit)
92 92 filtered_parsed_diff, has_hidden_changes = self.path_filter.filter_patchset(parsed_diff)
93 93 return _renderer(
94 94 'body',
95 95 commit=commit,
96 96 parsed_diff=filtered_parsed_diff,
97 97 limited_diff=limited_diff,
98 98 feed_include_diff=self.feed_include_diff,
99 99 diff_processor=diff_processor,
100 100 has_hidden_changes=has_hidden_changes
101 101 )
102 102
103 103 def _set_timezone(self, date, tzinfo=pytz.utc):
104 104 if not getattr(date, "tzinfo", None):
105 105 date.replace(tzinfo=tzinfo)
106 106 return date
107 107
108 108 def _get_commits(self):
109 109 return list(self.rhodecode_vcs_repo[-self.feed_items_per_page:])
110 110
111 111 def uid(self, repo_id, commit_id):
112 112 return '{}:{}'.format(md5_safe(repo_id), md5_safe(commit_id))
113 113
114 114 @LoginRequired(auth_token_access=[UserApiKeys.ROLE_FEED])
115 115 @HasRepoPermissionAnyDecorator(
116 116 'repository.read', 'repository.write', 'repository.admin')
117 117 @view_config(
118 118 route_name='atom_feed_home', request_method='GET',
119 119 renderer=None)
120 120 def atom(self):
121 121 """
122 122 Produce an atom-1.0 feed via feedgenerator module
123 123 """
124 124 self.load_default_context()
125 125
126 126 cache_namespace_uid = 'cache_repo_instance.{}_{}'.format(
127 127 self.db_repo.repo_id, CacheKey.CACHE_TYPE_FEED)
128 128 invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
129 129 repo_id=self.db_repo.repo_id)
130 130
131 131 region = rc_cache.get_or_create_region('cache_repo_longterm',
132 132 cache_namespace_uid)
133 133
134 134 condition = not self.path_filter.is_enabled
135 135
136 136 @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
137 137 condition=condition)
138 138 def generate_atom_feed(repo_id, _repo_name, _feed_type):
139 139 feed = Atom1Feed(
140 140 title=self.title % _repo_name,
141 141 link=h.route_url('repo_summary', repo_name=_repo_name),
142 142 description=self.description % _repo_name,
143 143 language=self.language,
144 144 ttl=self.ttl
145 145 )
146 146
147 147 for commit in reversed(self._get_commits()):
148 148 date = self._set_timezone(commit.date)
149 149 feed.add_item(
150 150 unique_id=self.uid(repo_id, commit.raw_id),
151 151 title=self._get_title(commit),
152 152 author_name=commit.author,
153 153 description=self._get_description(commit),
154 154 link=h.route_url(
155 155 'repo_commit', repo_name=_repo_name,
156 156 commit_id=commit.raw_id),
157 157 pubdate=date,)
158 158
159 159 return feed.mime_type, feed.writeString('utf-8')
160 160
161 161 inv_context_manager = rc_cache.InvalidationContext(
162 162 uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
163 163 with inv_context_manager as invalidation_context:
164 # check for stored invalidation signal, and maybe purge the cache
165 # before computing it again
164 args = (self.db_repo.repo_id, self.db_repo.repo_name, 'atom',)
165 # re-compute and store cache if we get invalidate signal
166 166 if invalidation_context.should_invalidate():
167 generate_atom_feed.invalidate(
168 self.db_repo.repo_id, self.db_repo.repo_name, 'atom')
169
170 mime_type, feed = generate_atom_feed(
171 self.db_repo.repo_id, self.db_repo.repo_name, 'atom')
167 mime_type, feed = generate_atom_feed.refresh(*args)
168 else:
169 mime_type, feed = generate_atom_feed(*args)
172 170
173 171 log.debug('Repo ATOM feed computed in %.3fs',
174 172 inv_context_manager.compute_time)
175 173
176 174 response = Response(feed)
177 175 response.content_type = mime_type
178 176 return response
179 177
180 178 @LoginRequired(auth_token_access=[UserApiKeys.ROLE_FEED])
181 179 @HasRepoPermissionAnyDecorator(
182 180 'repository.read', 'repository.write', 'repository.admin')
183 181 @view_config(
184 182 route_name='rss_feed_home', request_method='GET',
185 183 renderer=None)
186 184 def rss(self):
187 185 """
188 186 Produce an rss2 feed via feedgenerator module
189 187 """
190 188 self.load_default_context()
191 189
192 190 cache_namespace_uid = 'cache_repo_instance.{}_{}'.format(
193 191 self.db_repo.repo_id, CacheKey.CACHE_TYPE_FEED)
194 192 invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
195 193 repo_id=self.db_repo.repo_id)
196 194 region = rc_cache.get_or_create_region('cache_repo_longterm',
197 195 cache_namespace_uid)
198 196
199 197 condition = not self.path_filter.is_enabled
200 198
201 199 @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
202 200 condition=condition)
203 201 def generate_rss_feed(repo_id, _repo_name, _feed_type):
204 202 feed = Rss201rev2Feed(
205 203 title=self.title % _repo_name,
206 204 link=h.route_url('repo_summary', repo_name=_repo_name),
207 205 description=self.description % _repo_name,
208 206 language=self.language,
209 207 ttl=self.ttl
210 208 )
211 209
212 210 for commit in reversed(self._get_commits()):
213 211 date = self._set_timezone(commit.date)
214 212 feed.add_item(
215 213 unique_id=self.uid(repo_id, commit.raw_id),
216 214 title=self._get_title(commit),
217 215 author_name=commit.author,
218 216 description=self._get_description(commit),
219 217 link=h.route_url(
220 218 'repo_commit', repo_name=_repo_name,
221 219 commit_id=commit.raw_id),
222 220 pubdate=date,)
223 221
224 222 return feed.mime_type, feed.writeString('utf-8')
225 223
226 224 inv_context_manager = rc_cache.InvalidationContext(
227 225 uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
228 226 with inv_context_manager as invalidation_context:
229 # check for stored invalidation signal, and maybe purge the cache
230 # before computing it again
227 args = (self.db_repo.repo_id, self.db_repo.repo_name, 'rss',)
228 # re-compute and store cache if we get invalidate signal
231 229 if invalidation_context.should_invalidate():
232 generate_rss_feed.invalidate(
233 self.db_repo.repo_id, self.db_repo.repo_name, 'rss')
234
235 mime_type, feed = generate_rss_feed(
236 self.db_repo.repo_id, self.db_repo.repo_name, 'rss')
230 mime_type, feed = generate_rss_feed.refresh(*args)
231 else:
232 mime_type, feed = generate_rss_feed(*args)
237 233 log.debug(
238 234 'Repo RSS feed computed in %.3fs', inv_context_manager.compute_time)
239 235
240 236 response = Response(feed)
241 237 response.content_type = mime_type
242 238 return response
@@ -1,392 +1,390 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2011-2018 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 import logging
22 22 import string
23 23 import rhodecode
24 24
25 25 from pyramid.view import view_config
26 26
27 27 from rhodecode.controllers import utils
28 28 from rhodecode.apps._base import RepoAppView
29 29 from rhodecode.config.conf import (LANGUAGES_EXTENSIONS_MAP)
30 30 from rhodecode.lib import helpers as h, rc_cache
31 31 from rhodecode.lib.utils2 import safe_str, safe_int
32 32 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
33 33 from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
34 34 from rhodecode.lib.ext_json import json
35 35 from rhodecode.lib.vcs.backends.base import EmptyCommit
36 36 from rhodecode.lib.vcs.exceptions import (
37 37 CommitError, EmptyRepositoryError, CommitDoesNotExistError)
38 38 from rhodecode.model.db import Statistics, CacheKey, User
39 39 from rhodecode.model.meta import Session
40 40 from rhodecode.model.repo import ReadmeFinder
41 41 from rhodecode.model.scm import ScmModel
42 42
43 43 log = logging.getLogger(__name__)
44 44
45 45
46 46 class RepoSummaryView(RepoAppView):
47 47
48 48 def load_default_context(self):
49 49 c = self._get_local_tmpl_context(include_app_defaults=True)
50 50 c.rhodecode_repo = None
51 51 if not c.repository_requirements_missing:
52 52 c.rhodecode_repo = self.rhodecode_vcs_repo
53 53 return c
54 54
55 55 def _get_readme_data(self, db_repo, renderer_type):
56 56
57 57 log.debug('Looking for README file')
58 58
59 59 cache_namespace_uid = 'cache_repo_instance.{}_{}'.format(
60 60 db_repo.repo_id, CacheKey.CACHE_TYPE_README)
61 61 invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
62 62 repo_id=self.db_repo.repo_id)
63 63 region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid)
64 64
65 65 @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
66 66 def generate_repo_readme(repo_id, _repo_name, _renderer_type):
67 67 readme_data = None
68 68 readme_node = None
69 69 readme_filename = None
70 70 commit = self._get_landing_commit_or_none(db_repo)
71 71 if commit:
72 72 log.debug("Searching for a README file.")
73 73 readme_node = ReadmeFinder(_renderer_type).search(commit)
74 74 if readme_node:
75 75 relative_urls = {
76 76 'raw': h.route_path(
77 77 'repo_file_raw', repo_name=_repo_name,
78 78 commit_id=commit.raw_id, f_path=readme_node.path),
79 79 'standard': h.route_path(
80 80 'repo_files', repo_name=_repo_name,
81 81 commit_id=commit.raw_id, f_path=readme_node.path),
82 82 }
83 83 readme_data = self._render_readme_or_none(
84 84 commit, readme_node, relative_urls)
85 85 readme_filename = readme_node.path
86 86 return readme_data, readme_filename
87 87
88 88 inv_context_manager = rc_cache.InvalidationContext(
89 89 uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
90 90 with inv_context_manager as invalidation_context:
91 # check for stored invalidation signal, and maybe purge the cache
92 # before computing it again
91 args = (db_repo.repo_id, db_repo.repo_name, renderer_type,)
92 # re-compute and store cache if we get invalidate signal
93 93 if invalidation_context.should_invalidate():
94 generate_repo_readme.invalidate(
95 db_repo.repo_id, db_repo.repo_name, renderer_type)
96
97 instance = generate_repo_readme(
98 db_repo.repo_id, db_repo.repo_name, renderer_type)
94 instance = generate_repo_readme.refresh(*args)
95 else:
96 instance = generate_repo_readme(*args)
99 97
100 98 log.debug(
101 99 'Repo readme generated and computed in %.3fs',
102 100 inv_context_manager.compute_time)
103 101 return instance
104 102
105 103 def _get_landing_commit_or_none(self, db_repo):
106 104 log.debug("Getting the landing commit.")
107 105 try:
108 106 commit = db_repo.get_landing_commit()
109 107 if not isinstance(commit, EmptyCommit):
110 108 return commit
111 109 else:
112 110 log.debug("Repository is empty, no README to render.")
113 111 except CommitError:
114 112 log.exception(
115 113 "Problem getting commit when trying to render the README.")
116 114
117 115 def _render_readme_or_none(self, commit, readme_node, relative_urls):
118 116 log.debug(
119 117 'Found README file `%s` rendering...', readme_node.path)
120 118 renderer = MarkupRenderer()
121 119 try:
122 120 html_source = renderer.render(
123 121 readme_node.content, filename=readme_node.path)
124 122 if relative_urls:
125 123 return relative_links(html_source, relative_urls)
126 124 return html_source
127 125 except Exception:
128 126 log.exception(
129 127 "Exception while trying to render the README")
130 128
131 129 def _load_commits_context(self, c):
132 130 p = safe_int(self.request.GET.get('page'), 1)
133 131 size = safe_int(self.request.GET.get('size'), 10)
134 132
135 133 def url_generator(**kw):
136 134 query_params = {
137 135 'size': size
138 136 }
139 137 query_params.update(kw)
140 138 return h.route_path(
141 139 'repo_summary_commits',
142 140 repo_name=c.rhodecode_db_repo.repo_name, _query=query_params)
143 141
144 142 pre_load = ['author', 'branch', 'date', 'message']
145 143 try:
146 144 collection = self.rhodecode_vcs_repo.get_commits(pre_load=pre_load)
147 145 except EmptyRepositoryError:
148 146 collection = self.rhodecode_vcs_repo
149 147
150 148 c.repo_commits = h.RepoPage(
151 149 collection, page=p, items_per_page=size, url=url_generator)
152 150 page_ids = [x.raw_id for x in c.repo_commits]
153 151 c.comments = self.db_repo.get_comments(page_ids)
154 152 c.statuses = self.db_repo.statuses(page_ids)
155 153
156 154 @LoginRequired()
157 155 @HasRepoPermissionAnyDecorator(
158 156 'repository.read', 'repository.write', 'repository.admin')
159 157 @view_config(
160 158 route_name='repo_summary_commits', request_method='GET',
161 159 renderer='rhodecode:templates/summary/summary_commits.mako')
162 160 def summary_commits(self):
163 161 c = self.load_default_context()
164 162 self._load_commits_context(c)
165 163 return self._get_template_context(c)
166 164
167 165 @LoginRequired()
168 166 @HasRepoPermissionAnyDecorator(
169 167 'repository.read', 'repository.write', 'repository.admin')
170 168 @view_config(
171 169 route_name='repo_summary', request_method='GET',
172 170 renderer='rhodecode:templates/summary/summary.mako')
173 171 @view_config(
174 172 route_name='repo_summary_slash', request_method='GET',
175 173 renderer='rhodecode:templates/summary/summary.mako')
176 174 @view_config(
177 175 route_name='repo_summary_explicit', request_method='GET',
178 176 renderer='rhodecode:templates/summary/summary.mako')
179 177 def summary(self):
180 178 c = self.load_default_context()
181 179
182 180 # Prepare the clone URL
183 181 username = ''
184 182 if self._rhodecode_user.username != User.DEFAULT_USER:
185 183 username = safe_str(self._rhodecode_user.username)
186 184
187 185 _def_clone_uri = _def_clone_uri_id = c.clone_uri_tmpl
188 186 _def_clone_uri_ssh = c.clone_uri_ssh_tmpl
189 187
190 188 if '{repo}' in _def_clone_uri:
191 189 _def_clone_uri_id = _def_clone_uri.replace(
192 190 '{repo}', '_{repoid}')
193 191 elif '{repoid}' in _def_clone_uri:
194 192 _def_clone_uri_id = _def_clone_uri.replace(
195 193 '_{repoid}', '{repo}')
196 194
197 195 c.clone_repo_url = self.db_repo.clone_url(
198 196 user=username, uri_tmpl=_def_clone_uri)
199 197 c.clone_repo_url_id = self.db_repo.clone_url(
200 198 user=username, uri_tmpl=_def_clone_uri_id)
201 199 c.clone_repo_url_ssh = self.db_repo.clone_url(
202 200 uri_tmpl=_def_clone_uri_ssh, ssh=True)
203 201
204 202 # If enabled, get statistics data
205 203
206 204 c.show_stats = bool(self.db_repo.enable_statistics)
207 205
208 206 stats = Session().query(Statistics) \
209 207 .filter(Statistics.repository == self.db_repo) \
210 208 .scalar()
211 209
212 210 c.stats_percentage = 0
213 211
214 212 if stats and stats.languages:
215 213 c.no_data = False is self.db_repo.enable_statistics
216 214 lang_stats_d = json.loads(stats.languages)
217 215
218 216 # Sort first by decreasing count and second by the file extension,
219 217 # so we have a consistent output.
220 218 lang_stats_items = sorted(lang_stats_d.iteritems(),
221 219 key=lambda k: (-k[1], k[0]))[:10]
222 220 lang_stats = [(x, {"count": y,
223 221 "desc": LANGUAGES_EXTENSIONS_MAP.get(x)})
224 222 for x, y in lang_stats_items]
225 223
226 224 c.trending_languages = json.dumps(lang_stats)
227 225 else:
228 226 c.no_data = True
229 227 c.trending_languages = json.dumps({})
230 228
231 229 scm_model = ScmModel()
232 230 c.enable_downloads = self.db_repo.enable_downloads
233 231 c.repository_followers = scm_model.get_followers(self.db_repo)
234 232 c.repository_forks = scm_model.get_forks(self.db_repo)
235 233 c.repository_is_user_following = scm_model.is_following_repo(
236 234 self.db_repo_name, self._rhodecode_user.user_id)
237 235
238 236 # first interaction with the VCS instance after here...
239 237 if c.repository_requirements_missing:
240 238 self.request.override_renderer = \
241 239 'rhodecode:templates/summary/missing_requirements.mako'
242 240 return self._get_template_context(c)
243 241
244 242 c.readme_data, c.readme_file = \
245 243 self._get_readme_data(self.db_repo, c.visual.default_renderer)
246 244
247 245 # loads the summary commits template context
248 246 self._load_commits_context(c)
249 247
250 248 return self._get_template_context(c)
251 249
252 250 def get_request_commit_id(self):
253 251 return self.request.matchdict['commit_id']
254 252
255 253 @LoginRequired()
256 254 @HasRepoPermissionAnyDecorator(
257 255 'repository.read', 'repository.write', 'repository.admin')
258 256 @view_config(
259 257 route_name='repo_stats', request_method='GET',
260 258 renderer='json_ext')
261 259 def repo_stats(self):
262 260 commit_id = self.get_request_commit_id()
263 261 show_stats = bool(self.db_repo.enable_statistics)
264 262 repo_id = self.db_repo.repo_id
265 263
266 264 cache_seconds = safe_int(
267 265 rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
268 266 cache_on = cache_seconds > 0
269 267 log.debug(
270 268 'Computing REPO TREE for repo_id %s commit_id `%s` '
271 269 'with caching: %s[TTL: %ss]' % (
272 270 repo_id, commit_id, cache_on, cache_seconds or 0))
273 271
274 272 cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
275 273 region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
276 274
277 275 @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
278 276 condition=cache_on)
279 277 def compute_stats(repo_id, commit_id, show_stats):
280 278 code_stats = {}
281 279 size = 0
282 280 try:
283 281 scm_instance = self.db_repo.scm_instance()
284 282 commit = scm_instance.get_commit(commit_id)
285 283
286 284 for node in commit.get_filenodes_generator():
287 285 size += node.size
288 286 if not show_stats:
289 287 continue
290 288 ext = string.lower(node.extension)
291 289 ext_info = LANGUAGES_EXTENSIONS_MAP.get(ext)
292 290 if ext_info:
293 291 if ext in code_stats:
294 292 code_stats[ext]['count'] += 1
295 293 else:
296 294 code_stats[ext] = {"count": 1, "desc": ext_info}
297 295 except (EmptyRepositoryError, CommitDoesNotExistError):
298 296 pass
299 297 return {'size': h.format_byte_size_binary(size),
300 298 'code_stats': code_stats}
301 299
302 300 stats = compute_stats(self.db_repo.repo_id, commit_id, show_stats)
303 301 return stats
304 302
305 303 @LoginRequired()
306 304 @HasRepoPermissionAnyDecorator(
307 305 'repository.read', 'repository.write', 'repository.admin')
308 306 @view_config(
309 307 route_name='repo_refs_data', request_method='GET',
310 308 renderer='json_ext')
311 309 def repo_refs_data(self):
312 310 _ = self.request.translate
313 311 self.load_default_context()
314 312
315 313 repo = self.rhodecode_vcs_repo
316 314 refs_to_create = [
317 315 (_("Branch"), repo.branches, 'branch'),
318 316 (_("Tag"), repo.tags, 'tag'),
319 317 (_("Bookmark"), repo.bookmarks, 'book'),
320 318 ]
321 319 res = self._create_reference_data(
322 320 repo, self.db_repo_name, refs_to_create)
323 321 data = {
324 322 'more': False,
325 323 'results': res
326 324 }
327 325 return data
328 326
329 327 @LoginRequired()
330 328 @HasRepoPermissionAnyDecorator(
331 329 'repository.read', 'repository.write', 'repository.admin')
332 330 @view_config(
333 331 route_name='repo_refs_changelog_data', request_method='GET',
334 332 renderer='json_ext')
335 333 def repo_refs_changelog_data(self):
336 334 _ = self.request.translate
337 335 self.load_default_context()
338 336
339 337 repo = self.rhodecode_vcs_repo
340 338
341 339 refs_to_create = [
342 340 (_("Branches"), repo.branches, 'branch'),
343 341 (_("Closed branches"), repo.branches_closed, 'branch_closed'),
344 342 # TODO: enable when vcs can handle bookmarks filters
345 343 # (_("Bookmarks"), repo.bookmarks, "book"),
346 344 ]
347 345 res = self._create_reference_data(
348 346 repo, self.db_repo_name, refs_to_create)
349 347 data = {
350 348 'more': False,
351 349 'results': res
352 350 }
353 351 return data
354 352
355 353 def _create_reference_data(self, repo, full_repo_name, refs_to_create):
356 354 format_ref_id = utils.get_format_ref_id(repo)
357 355
358 356 result = []
359 357 for title, refs, ref_type in refs_to_create:
360 358 if refs:
361 359 result.append({
362 360 'text': title,
363 361 'children': self._create_reference_items(
364 362 repo, full_repo_name, refs, ref_type,
365 363 format_ref_id),
366 364 })
367 365 return result
368 366
369 367 def _create_reference_items(self, repo, full_repo_name, refs, ref_type,
370 368 format_ref_id):
371 369 result = []
372 370 is_svn = h.is_svn(repo)
373 371 for ref_name, raw_id in refs.iteritems():
374 372 files_url = self._create_files_url(
375 373 repo, full_repo_name, ref_name, raw_id, is_svn)
376 374 result.append({
377 375 'text': ref_name,
378 376 'id': format_ref_id(ref_name, raw_id),
379 377 'raw_id': raw_id,
380 378 'type': ref_type,
381 379 'files_url': files_url,
382 380 })
383 381 return result
384 382
385 383 def _create_files_url(self, repo, full_repo_name, ref_name, raw_id, is_svn):
386 384 use_commit_id = '/' in ref_name or is_svn
387 385 return h.route_path(
388 386 'repo_files',
389 387 repo_name=full_repo_name,
390 388 f_path=ref_name if is_svn else '',
391 389 commit_id=raw_id if use_commit_id else ref_name,
392 390 _query=dict(at=ref_name))
@@ -1,319 +1,320 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2015-2018 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20 import os
21 21 import time
22 22 import logging
23 23 import functools
24 24 import threading
25 25
26 26 from dogpile.cache import CacheRegion
27 27 from dogpile.cache.util import compat
28 28
29 29 import rhodecode
30 30 from rhodecode.lib.utils import safe_str, sha1
31 31 from rhodecode.lib.utils2 import safe_unicode, str2bool
32 32 from rhodecode.model.db import Session, CacheKey, IntegrityError
33 33
34 34 from . import region_meta
35 35
36 36 log = logging.getLogger(__name__)
37 37
38 38
39 39 class RhodeCodeCacheRegion(CacheRegion):
40 40
41 41 def conditional_cache_on_arguments(
42 42 self, namespace=None,
43 43 expiration_time=None,
44 44 should_cache_fn=None,
45 45 to_str=compat.string_type,
46 46 function_key_generator=None,
47 47 condition=True):
48 48 """
49 49 Custom conditional decorator, that will not touch any dogpile internals if
50 50 condition isn't meet. This works a bit different than should_cache_fn
51 51 And it's faster in cases we don't ever want to compute cached values
52 52 """
53 53 expiration_time_is_callable = compat.callable(expiration_time)
54 54
55 55 if function_key_generator is None:
56 56 function_key_generator = self.function_key_generator
57 57
58 58 def decorator(fn):
59 59 if to_str is compat.string_type:
60 60 # backwards compatible
61 61 key_generator = function_key_generator(namespace, fn)
62 62 else:
63 63 key_generator = function_key_generator(namespace, fn, to_str=to_str)
64 64
65 65 @functools.wraps(fn)
66 66 def decorate(*arg, **kw):
67 67 key = key_generator(*arg, **kw)
68 68
69 69 @functools.wraps(fn)
70 70 def creator():
71 71 return fn(*arg, **kw)
72 72
73 73 if not condition:
74 74 return creator()
75 75
76 76 timeout = expiration_time() if expiration_time_is_callable \
77 77 else expiration_time
78 78
79 79 return self.get_or_create(key, creator, timeout, should_cache_fn)
80 80
81 81 def invalidate(*arg, **kw):
82 82 key = key_generator(*arg, **kw)
83 83 self.delete(key)
84 84
85 85 def set_(value, *arg, **kw):
86 86 key = key_generator(*arg, **kw)
87 87 self.set(key, value)
88 88
89 89 def get(*arg, **kw):
90 90 key = key_generator(*arg, **kw)
91 91 return self.get(key)
92 92
93 93 def refresh(*arg, **kw):
94 94 key = key_generator(*arg, **kw)
95 95 value = fn(*arg, **kw)
96 96 self.set(key, value)
97 97 return value
98 98
99 99 decorate.set = set_
100 100 decorate.invalidate = invalidate
101 101 decorate.refresh = refresh
102 102 decorate.get = get
103 103 decorate.original = fn
104 104 decorate.key_generator = key_generator
105 105
106 106 return decorate
107 107
108 108 return decorator
109 109
110 110
111 111 def make_region(*arg, **kw):
112 112 return RhodeCodeCacheRegion(*arg, **kw)
113 113
114 114
115 115 def get_default_cache_settings(settings, prefixes=None):
116 116 prefixes = prefixes or []
117 117 cache_settings = {}
118 118 for key in settings.keys():
119 119 for prefix in prefixes:
120 120 if key.startswith(prefix):
121 121 name = key.split(prefix)[1].strip()
122 122 val = settings[key]
123 123 if isinstance(val, basestring):
124 124 val = val.strip()
125 125 cache_settings[name] = val
126 126 return cache_settings
127 127
128 128
129 129 def compute_key_from_params(*args):
130 130 """
131 131 Helper to compute key from given params to be used in cache manager
132 132 """
133 133 return sha1("_".join(map(safe_str, args)))
134 134
135 135
136 136 def key_generator(namespace, fn):
137 137 fname = fn.__name__
138 138
139 139 def generate_key(*args):
140 140 namespace_pref = namespace or 'default'
141 141 arg_key = compute_key_from_params(*args)
142 142 final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
143 143
144 144 return final_key
145 145
146 146 return generate_key
147 147
148 148
149 149 def get_or_create_region(region_name, region_namespace=None):
150 150 from rhodecode.lib.rc_cache.backends import FileNamespaceBackend
151 151 region_obj = region_meta.dogpile_cache_regions.get(region_name)
152 152 if not region_obj:
153 153 raise EnvironmentError(
154 154 'Region `{}` not in configured: {}.'.format(
155 155 region_name, region_meta.dogpile_cache_regions.keys()))
156 156
157 157 region_uid_name = '{}:{}'.format(region_name, region_namespace)
158 158 if isinstance(region_obj.actual_backend, FileNamespaceBackend):
159 159 region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
160 160 if region_exist:
161 161 log.debug('Using already configured region: %s', region_namespace)
162 162 return region_exist
163 163 cache_dir = region_meta.dogpile_config_defaults['cache_dir']
164 164 expiration_time = region_obj.expiration_time
165 165
166 166 if not os.path.isdir(cache_dir):
167 167 os.makedirs(cache_dir)
168 168 new_region = make_region(
169 169 name=region_uid_name, function_key_generator=key_generator
170 170 )
171 171 namespace_filename = os.path.join(
172 172 cache_dir, "{}.cache.dbm".format(region_namespace))
173 173 # special type that allows 1db per namespace
174 174 new_region.configure(
175 175 backend='dogpile.cache.rc.file_namespace',
176 176 expiration_time=expiration_time,
177 177 arguments={"filename": namespace_filename}
178 178 )
179 179
180 180 # create and save in region caches
181 181 log.debug('configuring new region: %s',region_uid_name)
182 182 region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
183 183
184 184 return region_obj
185 185
186 186
187 187 def clear_cache_namespace(cache_region, cache_namespace_uid):
188 188 region = get_or_create_region(cache_region, cache_namespace_uid)
189 189 cache_keys = region.backend.list_keys(prefix=cache_namespace_uid)
190 190 region.delete_multi(cache_keys)
191 191 return len(cache_keys)
192 192
193 193
194 194 class ActiveRegionCache(object):
195 195 def __init__(self, context):
196 196 self.context = context
197 197
198 198 def should_invalidate(self):
199 199 return False
200 200
201 201
202 202 class FreshRegionCache(object):
203 203 def __init__(self, context):
204 204 self.context = context
205 205
206 206 def should_invalidate(self):
207 207 return True
208 208
209 209
210 210 class InvalidationContext(object):
211 211 """
212 212 usage::
213 213
214 214 from rhodecode.lib import rc_cache
215 215
216 216 cache_namespace_uid = CacheKey.SOME_NAMESPACE.format(1)
217 217 region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
218 218
219 219 @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=True)
220 220 def heavy_compute(cache_name, param1, param2):
221 221 print('COMPUTE {}, {}, {}'.format(cache_name, param1, param2))
222 222
223 223 # invalidation namespace is shared namespace key for all process caches
224 224 # we use it to send a global signal
225 225 invalidation_namespace = 'repo_cache:1'
226 226
227 227 inv_context_manager = rc_cache.InvalidationContext(
228 228 uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
229 229 with inv_context_manager as invalidation_context:
230 # check for stored invalidation signal, and maybe purge the cache
231 # before computing it again
230 args = ('one', 'two')
231 # re-compute and store cache if we get invalidate signal
232 232 if invalidation_context.should_invalidate():
233 heavy_compute.invalidate('some_name', 'param1', 'param2')
233 result = heavy_compute.refresh(*args)
234 else:
235 result = heavy_compute(*args)
234 236
235 result = heavy_compute('some_name', 'param1', 'param2')
236 237 compute_time = inv_context_manager.compute_time
237 print(compute_time)
238 log.debug('result computed in %.3fs' ,compute_time)
238 239
239 240 # To send global invalidation signal, simply run
240 241 CacheKey.set_invalidate(invalidation_namespace)
241 242
242 243 """
243 244
244 245 def __repr__(self):
245 246 return '<InvalidationContext:{}[{}]>'.format(
246 247 safe_str(self.cache_key), safe_str(self.uid))
247 248
248 249 def __init__(self, uid, invalidation_namespace='',
249 250 raise_exception=False, thread_scoped=None):
250 251 self.uid = uid
251 252 self.invalidation_namespace = invalidation_namespace
252 253 self.raise_exception = raise_exception
253 254 self.proc_id = safe_unicode(rhodecode.CONFIG.get('instance_id') or 'DEFAULT')
254 255 self.thread_id = 'global'
255 256
256 257 if thread_scoped is None:
257 258 # if we set "default" we can override this via .ini settings
258 259 thread_scoped = str2bool(rhodecode.CONFIG.get('cache_thread_scoped'))
259 260
260 261 # Append the thread id to the cache key if this invalidation context
261 262 # should be scoped to the current thread.
262 263 if thread_scoped is True:
263 264 self.thread_id = threading.current_thread().ident
264 265
265 266 self.cache_key = compute_key_from_params(uid)
266 267 self.cache_key = 'proc:{}_thread:{}_{}'.format(
267 268 self.proc_id, self.thread_id, self.cache_key)
268 269 self.compute_time = 0
269 270
270 271 def get_or_create_cache_obj(self, uid, invalidation_namespace=''):
271 272 cache_obj = CacheKey.get_active_cache(self.cache_key)
272 273 log.debug('Fetched cache obj %s using %s cache key.', cache_obj, self.cache_key)
273 274 invalidation_namespace = invalidation_namespace or self.invalidation_namespace
274 275 if not cache_obj:
275 276 cache_obj = CacheKey(self.cache_key, cache_args=invalidation_namespace)
276 277 return cache_obj
277 278
278 279 def __enter__(self):
279 280 """
280 281 Test if current object is valid, and return CacheRegion function
281 282 that does invalidation and calculation
282 283 """
283 284 # register or get a new key based on uid
284 285 self.cache_obj = self.get_or_create_cache_obj(uid=self.uid)
285 286 self._start_time = time.time()
286 287 if self.cache_obj.cache_active:
287 288 # means our cache obj is existing and marked as it's
288 289 # cache is not outdated, we return ActiveRegionCache
289 290 self.skip_cache_active_change = True
290 291
291 292 return ActiveRegionCache(context=self)
292 293
293 294 # the key is either not existing or set to False, we return
294 295 # the real invalidator which re-computes value. We additionally set
295 296 # the flag to actually update the Database objects
296 297 self.skip_cache_active_change = False
297 298 return FreshRegionCache(context=self)
298 299
299 300 def __exit__(self, exc_type, exc_val, exc_tb):
300 301 # save compute time
301 302 self.compute_time = time.time() - self._start_time
302 303
303 304 if self.skip_cache_active_change:
304 305 return
305 306
306 307 try:
307 308 self.cache_obj.cache_active = True
308 309 Session().add(self.cache_obj)
309 310 Session().commit()
310 311 except IntegrityError:
311 312 # if we catch integrity error, it means we inserted this object
312 313 # assumption is that's really an edge race-condition case and
313 314 # it's safe is to skip it
314 315 Session().rollback()
315 316 except Exception:
316 317 log.exception('Failed to commit on cache key update')
317 318 Session().rollback()
318 319 if self.raise_exception:
319 320 raise
@@ -1,4510 +1,4511 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2010-2018 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21 """
22 22 Database Models for RhodeCode Enterprise
23 23 """
24 24
25 25 import re
26 26 import os
27 27 import time
28 28 import hashlib
29 29 import logging
30 30 import datetime
31 31 import warnings
32 32 import ipaddress
33 33 import functools
34 34 import traceback
35 35 import collections
36 36
37 37 from sqlalchemy import (
38 38 or_, and_, not_, func, TypeDecorator, event,
39 39 Index, Sequence, UniqueConstraint, ForeignKey, CheckConstraint, Column,
40 40 Boolean, String, Unicode, UnicodeText, DateTime, Integer, LargeBinary,
41 41 Text, Float, PickleType)
42 42 from sqlalchemy.sql.expression import true, false
43 43 from sqlalchemy.sql.functions import coalesce, count # noqa
44 44 from sqlalchemy.orm import (
45 45 relationship, joinedload, class_mapper, validates, aliased)
46 46 from sqlalchemy.ext.declarative import declared_attr
47 47 from sqlalchemy.ext.hybrid import hybrid_property
48 48 from sqlalchemy.exc import IntegrityError # noqa
49 49 from sqlalchemy.dialects.mysql import LONGTEXT
50 50 from zope.cachedescriptors.property import Lazy as LazyProperty
51 51
52 52 from pyramid.threadlocal import get_current_request
53 53
54 54 from rhodecode.translation import _
55 55 from rhodecode.lib.vcs import get_vcs_instance
56 56 from rhodecode.lib.vcs.backends.base import EmptyCommit, Reference
57 57 from rhodecode.lib.utils2 import (
58 58 str2bool, safe_str, get_commit_safe, safe_unicode, sha1_safe,
59 59 time_to_datetime, aslist, Optional, safe_int, get_clone_url, AttributeDict,
60 60 glob2re, StrictAttributeDict, cleaned_uri)
61 61 from rhodecode.lib.jsonalchemy import MutationObj, MutationList, JsonType, \
62 62 JsonRaw
63 63 from rhodecode.lib.ext_json import json
64 64 from rhodecode.lib.caching_query import FromCache
65 65 from rhodecode.lib.encrypt import AESCipher
66 66
67 67 from rhodecode.model.meta import Base, Session
68 68
69 69 URL_SEP = '/'
70 70 log = logging.getLogger(__name__)
71 71
72 72 # =============================================================================
73 73 # BASE CLASSES
74 74 # =============================================================================
75 75
76 76 # this is propagated from .ini file rhodecode.encrypted_values.secret or
77 77 # beaker.session.secret if first is not set.
78 78 # and initialized at environment.py
79 79 ENCRYPTION_KEY = None
80 80
81 81 # used to sort permissions by types, '#' used here is not allowed to be in
82 82 # usernames, and it's very early in sorted string.printable table.
83 83 PERMISSION_TYPE_SORT = {
84 84 'admin': '####',
85 85 'write': '###',
86 86 'read': '##',
87 87 'none': '#',
88 88 }
89 89
90 90
91 91 def display_user_sort(obj):
92 92 """
93 93 Sort function used to sort permissions in .permissions() function of
94 94 Repository, RepoGroup, UserGroup. Also it put the default user in front
95 95 of all other resources
96 96 """
97 97
98 98 if obj.username == User.DEFAULT_USER:
99 99 return '#####'
100 100 prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '')
101 101 return prefix + obj.username
102 102
103 103
104 104 def display_user_group_sort(obj):
105 105 """
106 106 Sort function used to sort permissions in .permissions() function of
107 107 Repository, RepoGroup, UserGroup. Also it put the default user in front
108 108 of all other resources
109 109 """
110 110
111 111 prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '')
112 112 return prefix + obj.users_group_name
113 113
114 114
115 115 def _hash_key(k):
116 116 return sha1_safe(k)
117 117
118 118
119 119 def in_filter_generator(qry, items, limit=500):
120 120 """
121 121 Splits IN() into multiple with OR
122 122 e.g.::
123 123 cnt = Repository.query().filter(
124 124 or_(
125 125 *in_filter_generator(Repository.repo_id, range(100000))
126 126 )).count()
127 127 """
128 128 if not items:
129 129 # empty list will cause empty query which might cause security issues
130 130 # this can lead to hidden unpleasant results
131 131 items = [-1]
132 132
133 133 parts = []
134 134 for chunk in xrange(0, len(items), limit):
135 135 parts.append(
136 136 qry.in_(items[chunk: chunk + limit])
137 137 )
138 138
139 139 return parts
140 140
141 141
142 142 base_table_args = {
143 143 'extend_existing': True,
144 144 'mysql_engine': 'InnoDB',
145 145 'mysql_charset': 'utf8',
146 146 'sqlite_autoincrement': True
147 147 }
148 148
149 149
150 150 class EncryptedTextValue(TypeDecorator):
151 151 """
152 152 Special column for encrypted long text data, use like::
153 153
154 154 value = Column("encrypted_value", EncryptedValue(), nullable=False)
155 155
156 156 This column is intelligent so if value is in unencrypted form it return
157 157 unencrypted form, but on save it always encrypts
158 158 """
159 159 impl = Text
160 160
161 161 def process_bind_param(self, value, dialect):
162 162 if not value:
163 163 return value
164 164 if value.startswith('enc$aes$') or value.startswith('enc$aes_hmac$'):
165 165 # protect against double encrypting if someone manually starts
166 166 # doing
167 167 raise ValueError('value needs to be in unencrypted format, ie. '
168 168 'not starting with enc$aes')
169 169 return 'enc$aes_hmac$%s' % AESCipher(
170 170 ENCRYPTION_KEY, hmac=True).encrypt(value)
171 171
172 172 def process_result_value(self, value, dialect):
173 173 import rhodecode
174 174
175 175 if not value:
176 176 return value
177 177
178 178 parts = value.split('$', 3)
179 179 if not len(parts) == 3:
180 180 # probably not encrypted values
181 181 return value
182 182 else:
183 183 if parts[0] != 'enc':
184 184 # parts ok but without our header ?
185 185 return value
186 186 enc_strict_mode = str2bool(rhodecode.CONFIG.get(
187 187 'rhodecode.encrypted_values.strict') or True)
188 188 # at that stage we know it's our encryption
189 189 if parts[1] == 'aes':
190 190 decrypted_data = AESCipher(ENCRYPTION_KEY).decrypt(parts[2])
191 191 elif parts[1] == 'aes_hmac':
192 192 decrypted_data = AESCipher(
193 193 ENCRYPTION_KEY, hmac=True,
194 194 strict_verification=enc_strict_mode).decrypt(parts[2])
195 195 else:
196 196 raise ValueError(
197 197 'Encryption type part is wrong, must be `aes` '
198 198 'or `aes_hmac`, got `%s` instead' % (parts[1]))
199 199 return decrypted_data
200 200
201 201
202 202 class BaseModel(object):
203 203 """
204 204 Base Model for all classes
205 205 """
206 206
207 207 @classmethod
208 208 def _get_keys(cls):
209 209 """return column names for this model """
210 210 return class_mapper(cls).c.keys()
211 211
212 212 def get_dict(self):
213 213 """
214 214 return dict with keys and values corresponding
215 215 to this model data """
216 216
217 217 d = {}
218 218 for k in self._get_keys():
219 219 d[k] = getattr(self, k)
220 220
221 221 # also use __json__() if present to get additional fields
222 222 _json_attr = getattr(self, '__json__', None)
223 223 if _json_attr:
224 224 # update with attributes from __json__
225 225 if callable(_json_attr):
226 226 _json_attr = _json_attr()
227 227 for k, val in _json_attr.iteritems():
228 228 d[k] = val
229 229 return d
230 230
231 231 def get_appstruct(self):
232 232 """return list with keys and values tuples corresponding
233 233 to this model data """
234 234
235 235 lst = []
236 236 for k in self._get_keys():
237 237 lst.append((k, getattr(self, k),))
238 238 return lst
239 239
240 240 def populate_obj(self, populate_dict):
241 241 """populate model with data from given populate_dict"""
242 242
243 243 for k in self._get_keys():
244 244 if k in populate_dict:
245 245 setattr(self, k, populate_dict[k])
246 246
247 247 @classmethod
248 248 def query(cls):
249 249 return Session().query(cls)
250 250
251 251 @classmethod
252 252 def get(cls, id_):
253 253 if id_:
254 254 return cls.query().get(id_)
255 255
256 256 @classmethod
257 257 def get_or_404(cls, id_):
258 258 from pyramid.httpexceptions import HTTPNotFound
259 259
260 260 try:
261 261 id_ = int(id_)
262 262 except (TypeError, ValueError):
263 263 raise HTTPNotFound()
264 264
265 265 res = cls.query().get(id_)
266 266 if not res:
267 267 raise HTTPNotFound()
268 268 return res
269 269
270 270 @classmethod
271 271 def getAll(cls):
272 272 # deprecated and left for backward compatibility
273 273 return cls.get_all()
274 274
275 275 @classmethod
276 276 def get_all(cls):
277 277 return cls.query().all()
278 278
279 279 @classmethod
280 280 def delete(cls, id_):
281 281 obj = cls.query().get(id_)
282 282 Session().delete(obj)
283 283
284 284 @classmethod
285 285 def identity_cache(cls, session, attr_name, value):
286 286 exist_in_session = []
287 287 for (item_cls, pkey), instance in session.identity_map.items():
288 288 if cls == item_cls and getattr(instance, attr_name) == value:
289 289 exist_in_session.append(instance)
290 290 if exist_in_session:
291 291 if len(exist_in_session) == 1:
292 292 return exist_in_session[0]
293 293 log.exception(
294 294 'multiple objects with attr %s and '
295 295 'value %s found with same name: %r',
296 296 attr_name, value, exist_in_session)
297 297
298 298 def __repr__(self):
299 299 if hasattr(self, '__unicode__'):
300 300 # python repr needs to return str
301 301 try:
302 302 return safe_str(self.__unicode__())
303 303 except UnicodeDecodeError:
304 304 pass
305 305 return '<DB:%s>' % (self.__class__.__name__)
306 306
307 307
308 308 class RhodeCodeSetting(Base, BaseModel):
309 309 __tablename__ = 'rhodecode_settings'
310 310 __table_args__ = (
311 311 UniqueConstraint('app_settings_name'),
312 312 base_table_args
313 313 )
314 314
315 315 SETTINGS_TYPES = {
316 316 'str': safe_str,
317 317 'int': safe_int,
318 318 'unicode': safe_unicode,
319 319 'bool': str2bool,
320 320 'list': functools.partial(aslist, sep=',')
321 321 }
322 322 DEFAULT_UPDATE_URL = 'https://rhodecode.com/api/v1/info/versions'
323 323 GLOBAL_CONF_KEY = 'app_settings'
324 324
325 325 app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
326 326 app_settings_name = Column("app_settings_name", String(255), nullable=True, unique=None, default=None)
327 327 _app_settings_value = Column("app_settings_value", String(4096), nullable=True, unique=None, default=None)
328 328 _app_settings_type = Column("app_settings_type", String(255), nullable=True, unique=None, default=None)
329 329
330 330 def __init__(self, key='', val='', type='unicode'):
331 331 self.app_settings_name = key
332 332 self.app_settings_type = type
333 333 self.app_settings_value = val
334 334
335 335 @validates('_app_settings_value')
336 336 def validate_settings_value(self, key, val):
337 337 assert type(val) == unicode
338 338 return val
339 339
340 340 @hybrid_property
341 341 def app_settings_value(self):
342 342 v = self._app_settings_value
343 343 _type = self.app_settings_type
344 344 if _type:
345 345 _type = self.app_settings_type.split('.')[0]
346 346 # decode the encrypted value
347 347 if 'encrypted' in self.app_settings_type:
348 348 cipher = EncryptedTextValue()
349 349 v = safe_unicode(cipher.process_result_value(v, None))
350 350
351 351 converter = self.SETTINGS_TYPES.get(_type) or \
352 352 self.SETTINGS_TYPES['unicode']
353 353 return converter(v)
354 354
355 355 @app_settings_value.setter
356 356 def app_settings_value(self, val):
357 357 """
358 358 Setter that will always make sure we use unicode in app_settings_value
359 359
360 360 :param val:
361 361 """
362 362 val = safe_unicode(val)
363 363 # encode the encrypted value
364 364 if 'encrypted' in self.app_settings_type:
365 365 cipher = EncryptedTextValue()
366 366 val = safe_unicode(cipher.process_bind_param(val, None))
367 367 self._app_settings_value = val
368 368
369 369 @hybrid_property
370 370 def app_settings_type(self):
371 371 return self._app_settings_type
372 372
373 373 @app_settings_type.setter
374 374 def app_settings_type(self, val):
375 375 if val.split('.')[0] not in self.SETTINGS_TYPES:
376 376 raise Exception('type must be one of %s got %s'
377 377 % (self.SETTINGS_TYPES.keys(), val))
378 378 self._app_settings_type = val
379 379
380 380 def __unicode__(self):
381 381 return u"<%s('%s:%s[%s]')>" % (
382 382 self.__class__.__name__,
383 383 self.app_settings_name, self.app_settings_value,
384 384 self.app_settings_type
385 385 )
386 386
387 387
388 388 class RhodeCodeUi(Base, BaseModel):
389 389 __tablename__ = 'rhodecode_ui'
390 390 __table_args__ = (
391 391 UniqueConstraint('ui_key'),
392 392 base_table_args
393 393 )
394 394
395 395 HOOK_REPO_SIZE = 'changegroup.repo_size'
396 396 # HG
397 397 HOOK_PRE_PULL = 'preoutgoing.pre_pull'
398 398 HOOK_PULL = 'outgoing.pull_logger'
399 399 HOOK_PRE_PUSH = 'prechangegroup.pre_push'
400 400 HOOK_PRETX_PUSH = 'pretxnchangegroup.pre_push'
401 401 HOOK_PUSH = 'changegroup.push_logger'
402 402 HOOK_PUSH_KEY = 'pushkey.key_push'
403 403
404 404 # TODO: johbo: Unify way how hooks are configured for git and hg,
405 405 # git part is currently hardcoded.
406 406
407 407 # SVN PATTERNS
408 408 SVN_BRANCH_ID = 'vcs_svn_branch'
409 409 SVN_TAG_ID = 'vcs_svn_tag'
410 410
411 411 ui_id = Column(
412 412 "ui_id", Integer(), nullable=False, unique=True, default=None,
413 413 primary_key=True)
414 414 ui_section = Column(
415 415 "ui_section", String(255), nullable=True, unique=None, default=None)
416 416 ui_key = Column(
417 417 "ui_key", String(255), nullable=True, unique=None, default=None)
418 418 ui_value = Column(
419 419 "ui_value", String(255), nullable=True, unique=None, default=None)
420 420 ui_active = Column(
421 421 "ui_active", Boolean(), nullable=True, unique=None, default=True)
422 422
423 423 def __repr__(self):
424 424 return '<%s[%s]%s=>%s]>' % (self.__class__.__name__, self.ui_section,
425 425 self.ui_key, self.ui_value)
426 426
427 427
428 428 class RepoRhodeCodeSetting(Base, BaseModel):
429 429 __tablename__ = 'repo_rhodecode_settings'
430 430 __table_args__ = (
431 431 UniqueConstraint(
432 432 'app_settings_name', 'repository_id',
433 433 name='uq_repo_rhodecode_setting_name_repo_id'),
434 434 base_table_args
435 435 )
436 436
437 437 repository_id = Column(
438 438 "repository_id", Integer(), ForeignKey('repositories.repo_id'),
439 439 nullable=False)
440 440 app_settings_id = Column(
441 441 "app_settings_id", Integer(), nullable=False, unique=True,
442 442 default=None, primary_key=True)
443 443 app_settings_name = Column(
444 444 "app_settings_name", String(255), nullable=True, unique=None,
445 445 default=None)
446 446 _app_settings_value = Column(
447 447 "app_settings_value", String(4096), nullable=True, unique=None,
448 448 default=None)
449 449 _app_settings_type = Column(
450 450 "app_settings_type", String(255), nullable=True, unique=None,
451 451 default=None)
452 452
453 453 repository = relationship('Repository')
454 454
455 455 def __init__(self, repository_id, key='', val='', type='unicode'):
456 456 self.repository_id = repository_id
457 457 self.app_settings_name = key
458 458 self.app_settings_type = type
459 459 self.app_settings_value = val
460 460
461 461 @validates('_app_settings_value')
462 462 def validate_settings_value(self, key, val):
463 463 assert type(val) == unicode
464 464 return val
465 465
466 466 @hybrid_property
467 467 def app_settings_value(self):
468 468 v = self._app_settings_value
469 469 type_ = self.app_settings_type
470 470 SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES
471 471 converter = SETTINGS_TYPES.get(type_) or SETTINGS_TYPES['unicode']
472 472 return converter(v)
473 473
474 474 @app_settings_value.setter
475 475 def app_settings_value(self, val):
476 476 """
477 477 Setter that will always make sure we use unicode in app_settings_value
478 478
479 479 :param val:
480 480 """
481 481 self._app_settings_value = safe_unicode(val)
482 482
483 483 @hybrid_property
484 484 def app_settings_type(self):
485 485 return self._app_settings_type
486 486
487 487 @app_settings_type.setter
488 488 def app_settings_type(self, val):
489 489 SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES
490 490 if val not in SETTINGS_TYPES:
491 491 raise Exception('type must be one of %s got %s'
492 492 % (SETTINGS_TYPES.keys(), val))
493 493 self._app_settings_type = val
494 494
495 495 def __unicode__(self):
496 496 return u"<%s('%s:%s:%s[%s]')>" % (
497 497 self.__class__.__name__, self.repository.repo_name,
498 498 self.app_settings_name, self.app_settings_value,
499 499 self.app_settings_type
500 500 )
501 501
502 502
503 503 class RepoRhodeCodeUi(Base, BaseModel):
504 504 __tablename__ = 'repo_rhodecode_ui'
505 505 __table_args__ = (
506 506 UniqueConstraint(
507 507 'repository_id', 'ui_section', 'ui_key',
508 508 name='uq_repo_rhodecode_ui_repository_id_section_key'),
509 509 base_table_args
510 510 )
511 511
512 512 repository_id = Column(
513 513 "repository_id", Integer(), ForeignKey('repositories.repo_id'),
514 514 nullable=False)
515 515 ui_id = Column(
516 516 "ui_id", Integer(), nullable=False, unique=True, default=None,
517 517 primary_key=True)
518 518 ui_section = Column(
519 519 "ui_section", String(255), nullable=True, unique=None, default=None)
520 520 ui_key = Column(
521 521 "ui_key", String(255), nullable=True, unique=None, default=None)
522 522 ui_value = Column(
523 523 "ui_value", String(255), nullable=True, unique=None, default=None)
524 524 ui_active = Column(
525 525 "ui_active", Boolean(), nullable=True, unique=None, default=True)
526 526
527 527 repository = relationship('Repository')
528 528
529 529 def __repr__(self):
530 530 return '<%s[%s:%s]%s=>%s]>' % (
531 531 self.__class__.__name__, self.repository.repo_name,
532 532 self.ui_section, self.ui_key, self.ui_value)
533 533
534 534
535 535 class User(Base, BaseModel):
536 536 __tablename__ = 'users'
537 537 __table_args__ = (
538 538 UniqueConstraint('username'), UniqueConstraint('email'),
539 539 Index('u_username_idx', 'username'),
540 540 Index('u_email_idx', 'email'),
541 541 base_table_args
542 542 )
543 543
544 544 DEFAULT_USER = 'default'
545 545 DEFAULT_USER_EMAIL = 'anonymous@rhodecode.org'
546 546 DEFAULT_GRAVATAR_URL = 'https://secure.gravatar.com/avatar/{md5email}?d=identicon&s={size}'
547 547
548 548 user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
549 549 username = Column("username", String(255), nullable=True, unique=None, default=None)
550 550 password = Column("password", String(255), nullable=True, unique=None, default=None)
551 551 active = Column("active", Boolean(), nullable=True, unique=None, default=True)
552 552 admin = Column("admin", Boolean(), nullable=True, unique=None, default=False)
553 553 name = Column("firstname", String(255), nullable=True, unique=None, default=None)
554 554 lastname = Column("lastname", String(255), nullable=True, unique=None, default=None)
555 555 _email = Column("email", String(255), nullable=True, unique=None, default=None)
556 556 last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None)
557 557 last_activity = Column('last_activity', DateTime(timezone=False), nullable=True, unique=None, default=None)
558 558
559 559 extern_type = Column("extern_type", String(255), nullable=True, unique=None, default=None)
560 560 extern_name = Column("extern_name", String(255), nullable=True, unique=None, default=None)
561 561 _api_key = Column("api_key", String(255), nullable=True, unique=None, default=None)
562 562 inherit_default_permissions = Column("inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True)
563 563 created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
564 564 _user_data = Column("user_data", LargeBinary(), nullable=True) # JSON data
565 565
566 566 user_log = relationship('UserLog')
567 567 user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
568 568
569 569 repositories = relationship('Repository')
570 570 repository_groups = relationship('RepoGroup')
571 571 user_groups = relationship('UserGroup')
572 572
573 573 user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
574 574 followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all')
575 575
576 576 repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all')
577 577 repo_group_to_perm = relationship('UserRepoGroupToPerm', primaryjoin='UserRepoGroupToPerm.user_id==User.user_id', cascade='all')
578 578 user_group_to_perm = relationship('UserUserGroupToPerm', primaryjoin='UserUserGroupToPerm.user_id==User.user_id', cascade='all')
579 579
580 580 group_member = relationship('UserGroupMember', cascade='all')
581 581
582 582 notifications = relationship('UserNotification', cascade='all')
583 583 # notifications assigned to this user
584 584 user_created_notifications = relationship('Notification', cascade='all')
585 585 # comments created by this user
586 586 user_comments = relationship('ChangesetComment', cascade='all')
587 587 # user profile extra info
588 588 user_emails = relationship('UserEmailMap', cascade='all')
589 589 user_ip_map = relationship('UserIpMap', cascade='all')
590 590 user_auth_tokens = relationship('UserApiKeys', cascade='all')
591 591 user_ssh_keys = relationship('UserSshKeys', cascade='all')
592 592
593 593 # gists
594 594 user_gists = relationship('Gist', cascade='all')
595 595 # user pull requests
596 596 user_pull_requests = relationship('PullRequest', cascade='all')
597 597 # external identities
598 598 extenal_identities = relationship(
599 599 'ExternalIdentity',
600 600 primaryjoin="User.user_id==ExternalIdentity.local_user_id",
601 601 cascade='all')
602 602 # review rules
603 603 user_review_rules = relationship('RepoReviewRuleUser', cascade='all')
604 604
605 605 def __unicode__(self):
606 606 return u"<%s('id:%s:%s')>" % (self.__class__.__name__,
607 607 self.user_id, self.username)
608 608
609 609 @hybrid_property
610 610 def email(self):
611 611 return self._email
612 612
613 613 @email.setter
614 614 def email(self, val):
615 615 self._email = val.lower() if val else None
616 616
617 617 @hybrid_property
618 618 def first_name(self):
619 619 from rhodecode.lib import helpers as h
620 620 if self.name:
621 621 return h.escape(self.name)
622 622 return self.name
623 623
624 624 @hybrid_property
625 625 def last_name(self):
626 626 from rhodecode.lib import helpers as h
627 627 if self.lastname:
628 628 return h.escape(self.lastname)
629 629 return self.lastname
630 630
631 631 @hybrid_property
632 632 def api_key(self):
633 633 """
634 634 Fetch if exist an auth-token with role ALL connected to this user
635 635 """
636 636 user_auth_token = UserApiKeys.query()\
637 637 .filter(UserApiKeys.user_id == self.user_id)\
638 638 .filter(or_(UserApiKeys.expires == -1,
639 639 UserApiKeys.expires >= time.time()))\
640 640 .filter(UserApiKeys.role == UserApiKeys.ROLE_ALL).first()
641 641 if user_auth_token:
642 642 user_auth_token = user_auth_token.api_key
643 643
644 644 return user_auth_token
645 645
646 646 @api_key.setter
647 647 def api_key(self, val):
648 648 # don't allow to set API key this is deprecated for now
649 649 self._api_key = None
650 650
651 651 @property
652 652 def reviewer_pull_requests(self):
653 653 return PullRequestReviewers.query() \
654 654 .options(joinedload(PullRequestReviewers.pull_request)) \
655 655 .filter(PullRequestReviewers.user_id == self.user_id) \
656 656 .all()
657 657
658 658 @property
659 659 def firstname(self):
660 660 # alias for future
661 661 return self.name
662 662
663 663 @property
664 664 def emails(self):
665 665 other = UserEmailMap.query()\
666 666 .filter(UserEmailMap.user == self) \
667 667 .order_by(UserEmailMap.email_id.asc()) \
668 668 .all()
669 669 return [self.email] + [x.email for x in other]
670 670
671 671 @property
672 672 def auth_tokens(self):
673 673 auth_tokens = self.get_auth_tokens()
674 674 return [x.api_key for x in auth_tokens]
675 675
676 676 def get_auth_tokens(self):
677 677 return UserApiKeys.query()\
678 678 .filter(UserApiKeys.user == self)\
679 679 .order_by(UserApiKeys.user_api_key_id.asc())\
680 680 .all()
681 681
682 682 @LazyProperty
683 683 def feed_token(self):
684 684 return self.get_feed_token()
685 685
686 686 def get_feed_token(self, cache=True):
687 687 feed_tokens = UserApiKeys.query()\
688 688 .filter(UserApiKeys.user == self)\
689 689 .filter(UserApiKeys.role == UserApiKeys.ROLE_FEED)
690 690 if cache:
691 691 feed_tokens = feed_tokens.options(
692 692 FromCache("sql_cache_short", "get_user_feed_token_%s" % self.user_id))
693 693
694 694 feed_tokens = feed_tokens.all()
695 695 if feed_tokens:
696 696 return feed_tokens[0].api_key
697 697 return 'NO_FEED_TOKEN_AVAILABLE'
698 698
699 699 @classmethod
700 700 def get(cls, user_id, cache=False):
701 701 if not user_id:
702 702 return
703 703
704 704 user = cls.query()
705 705 if cache:
706 706 user = user.options(
707 707 FromCache("sql_cache_short", "get_users_%s" % user_id))
708 708 return user.get(user_id)
709 709
710 710 @classmethod
711 711 def extra_valid_auth_tokens(cls, user, role=None):
712 712 tokens = UserApiKeys.query().filter(UserApiKeys.user == user)\
713 713 .filter(or_(UserApiKeys.expires == -1,
714 714 UserApiKeys.expires >= time.time()))
715 715 if role:
716 716 tokens = tokens.filter(or_(UserApiKeys.role == role,
717 717 UserApiKeys.role == UserApiKeys.ROLE_ALL))
718 718 return tokens.all()
719 719
720 720 def authenticate_by_token(self, auth_token, roles=None, scope_repo_id=None):
721 721 from rhodecode.lib import auth
722 722
723 723 log.debug('Trying to authenticate user: %s via auth-token, '
724 724 'and roles: %s', self, roles)
725 725
726 726 if not auth_token:
727 727 return False
728 728
729 729 crypto_backend = auth.crypto_backend()
730 730
731 731 roles = (roles or []) + [UserApiKeys.ROLE_ALL]
732 732 tokens_q = UserApiKeys.query()\
733 733 .filter(UserApiKeys.user_id == self.user_id)\
734 734 .filter(or_(UserApiKeys.expires == -1,
735 735 UserApiKeys.expires >= time.time()))
736 736
737 737 tokens_q = tokens_q.filter(UserApiKeys.role.in_(roles))
738 738
739 739 plain_tokens = []
740 740 hash_tokens = []
741 741
742 742 for token in tokens_q.all():
743 743 # verify scope first
744 744 if token.repo_id:
745 745 # token has a scope, we need to verify it
746 746 if scope_repo_id != token.repo_id:
747 747 log.debug(
748 748 'Scope mismatch: token has a set repo scope: %s, '
749 749 'and calling scope is:%s, skipping further checks',
750 750 token.repo, scope_repo_id)
751 751 # token has a scope, and it doesn't match, skip token
752 752 continue
753 753
754 754 if token.api_key.startswith(crypto_backend.ENC_PREF):
755 755 hash_tokens.append(token.api_key)
756 756 else:
757 757 plain_tokens.append(token.api_key)
758 758
759 759 is_plain_match = auth_token in plain_tokens
760 760 if is_plain_match:
761 761 return True
762 762
763 763 for hashed in hash_tokens:
764 764 # TODO(marcink): this is expensive to calculate, but most secure
765 765 match = crypto_backend.hash_check(auth_token, hashed)
766 766 if match:
767 767 return True
768 768
769 769 return False
770 770
771 771 @property
772 772 def ip_addresses(self):
773 773 ret = UserIpMap.query().filter(UserIpMap.user == self).all()
774 774 return [x.ip_addr for x in ret]
775 775
776 776 @property
777 777 def username_and_name(self):
778 778 return '%s (%s %s)' % (self.username, self.first_name, self.last_name)
779 779
780 780 @property
781 781 def username_or_name_or_email(self):
782 782 full_name = self.full_name if self.full_name is not ' ' else None
783 783 return self.username or full_name or self.email
784 784
785 785 @property
786 786 def full_name(self):
787 787 return '%s %s' % (self.first_name, self.last_name)
788 788
789 789 @property
790 790 def full_name_or_username(self):
791 791 return ('%s %s' % (self.first_name, self.last_name)
792 792 if (self.first_name and self.last_name) else self.username)
793 793
794 794 @property
795 795 def full_contact(self):
796 796 return '%s %s <%s>' % (self.first_name, self.last_name, self.email)
797 797
798 798 @property
799 799 def short_contact(self):
800 800 return '%s %s' % (self.first_name, self.last_name)
801 801
802 802 @property
803 803 def is_admin(self):
804 804 return self.admin
805 805
806 806 def AuthUser(self, **kwargs):
807 807 """
808 808 Returns instance of AuthUser for this user
809 809 """
810 810 from rhodecode.lib.auth import AuthUser
811 811 return AuthUser(user_id=self.user_id, username=self.username, **kwargs)
812 812
813 813 @hybrid_property
814 814 def user_data(self):
815 815 if not self._user_data:
816 816 return {}
817 817
818 818 try:
819 819 return json.loads(self._user_data)
820 820 except TypeError:
821 821 return {}
822 822
823 823 @user_data.setter
824 824 def user_data(self, val):
825 825 if not isinstance(val, dict):
826 826 raise Exception('user_data must be dict, got %s' % type(val))
827 827 try:
828 828 self._user_data = json.dumps(val)
829 829 except Exception:
830 830 log.error(traceback.format_exc())
831 831
832 832 @classmethod
833 833 def get_by_username(cls, username, case_insensitive=False,
834 834 cache=False, identity_cache=False):
835 835 session = Session()
836 836
837 837 if case_insensitive:
838 838 q = cls.query().filter(
839 839 func.lower(cls.username) == func.lower(username))
840 840 else:
841 841 q = cls.query().filter(cls.username == username)
842 842
843 843 if cache:
844 844 if identity_cache:
845 845 val = cls.identity_cache(session, 'username', username)
846 846 if val:
847 847 return val
848 848 else:
849 849 cache_key = "get_user_by_name_%s" % _hash_key(username)
850 850 q = q.options(
851 851 FromCache("sql_cache_short", cache_key))
852 852
853 853 return q.scalar()
854 854
855 855 @classmethod
856 856 def get_by_auth_token(cls, auth_token, cache=False):
857 857 q = UserApiKeys.query()\
858 858 .filter(UserApiKeys.api_key == auth_token)\
859 859 .filter(or_(UserApiKeys.expires == -1,
860 860 UserApiKeys.expires >= time.time()))
861 861 if cache:
862 862 q = q.options(
863 863 FromCache("sql_cache_short", "get_auth_token_%s" % auth_token))
864 864
865 865 match = q.first()
866 866 if match:
867 867 return match.user
868 868
869 869 @classmethod
870 870 def get_by_email(cls, email, case_insensitive=False, cache=False):
871 871
872 872 if case_insensitive:
873 873 q = cls.query().filter(func.lower(cls.email) == func.lower(email))
874 874
875 875 else:
876 876 q = cls.query().filter(cls.email == email)
877 877
878 878 email_key = _hash_key(email)
879 879 if cache:
880 880 q = q.options(
881 881 FromCache("sql_cache_short", "get_email_key_%s" % email_key))
882 882
883 883 ret = q.scalar()
884 884 if ret is None:
885 885 q = UserEmailMap.query()
886 886 # try fetching in alternate email map
887 887 if case_insensitive:
888 888 q = q.filter(func.lower(UserEmailMap.email) == func.lower(email))
889 889 else:
890 890 q = q.filter(UserEmailMap.email == email)
891 891 q = q.options(joinedload(UserEmailMap.user))
892 892 if cache:
893 893 q = q.options(
894 894 FromCache("sql_cache_short", "get_email_map_key_%s" % email_key))
895 895 ret = getattr(q.scalar(), 'user', None)
896 896
897 897 return ret
898 898
899 899 @classmethod
900 900 def get_from_cs_author(cls, author):
901 901 """
902 902 Tries to get User objects out of commit author string
903 903
904 904 :param author:
905 905 """
906 906 from rhodecode.lib.helpers import email, author_name
907 907 # Valid email in the attribute passed, see if they're in the system
908 908 _email = email(author)
909 909 if _email:
910 910 user = cls.get_by_email(_email, case_insensitive=True)
911 911 if user:
912 912 return user
913 913 # Maybe we can match by username?
914 914 _author = author_name(author)
915 915 user = cls.get_by_username(_author, case_insensitive=True)
916 916 if user:
917 917 return user
918 918
919 919 def update_userdata(self, **kwargs):
920 920 usr = self
921 921 old = usr.user_data
922 922 old.update(**kwargs)
923 923 usr.user_data = old
924 924 Session().add(usr)
925 925 log.debug('updated userdata with ', kwargs)
926 926
927 927 def update_lastlogin(self):
928 928 """Update user lastlogin"""
929 929 self.last_login = datetime.datetime.now()
930 930 Session().add(self)
931 931 log.debug('updated user %s lastlogin', self.username)
932 932
933 933 def update_password(self, new_password):
934 934 from rhodecode.lib.auth import get_crypt_password
935 935
936 936 self.password = get_crypt_password(new_password)
937 937 Session().add(self)
938 938
939 939 @classmethod
940 940 def get_first_super_admin(cls):
941 941 user = User.query().filter(User.admin == true()).first()
942 942 if user is None:
943 943 raise Exception('FATAL: Missing administrative account!')
944 944 return user
945 945
946 946 @classmethod
947 947 def get_all_super_admins(cls):
948 948 """
949 949 Returns all admin accounts sorted by username
950 950 """
951 951 return User.query().filter(User.admin == true())\
952 952 .order_by(User.username.asc()).all()
953 953
954 954 @classmethod
955 955 def get_default_user(cls, cache=False, refresh=False):
956 956 user = User.get_by_username(User.DEFAULT_USER, cache=cache)
957 957 if user is None:
958 958 raise Exception('FATAL: Missing default account!')
959 959 if refresh:
960 960 # The default user might be based on outdated state which
961 961 # has been loaded from the cache.
962 962 # A call to refresh() ensures that the
963 963 # latest state from the database is used.
964 964 Session().refresh(user)
965 965 return user
966 966
967 967 def _get_default_perms(self, user, suffix=''):
968 968 from rhodecode.model.permission import PermissionModel
969 969 return PermissionModel().get_default_perms(user.user_perms, suffix)
970 970
971 971 def get_default_perms(self, suffix=''):
972 972 return self._get_default_perms(self, suffix)
973 973
974 974 def get_api_data(self, include_secrets=False, details='full'):
975 975 """
976 976 Common function for generating user related data for API
977 977
978 978 :param include_secrets: By default secrets in the API data will be replaced
979 979 by a placeholder value to prevent exposing this data by accident. In case
980 980 this data shall be exposed, set this flag to ``True``.
981 981
982 982 :param details: details can be 'basic|full' basic gives only a subset of
983 983 the available user information that includes user_id, name and emails.
984 984 """
985 985 user = self
986 986 user_data = self.user_data
987 987 data = {
988 988 'user_id': user.user_id,
989 989 'username': user.username,
990 990 'firstname': user.name,
991 991 'lastname': user.lastname,
992 992 'email': user.email,
993 993 'emails': user.emails,
994 994 }
995 995 if details == 'basic':
996 996 return data
997 997
998 998 auth_token_length = 40
999 999 auth_token_replacement = '*' * auth_token_length
1000 1000
1001 1001 extras = {
1002 1002 'auth_tokens': [auth_token_replacement],
1003 1003 'active': user.active,
1004 1004 'admin': user.admin,
1005 1005 'extern_type': user.extern_type,
1006 1006 'extern_name': user.extern_name,
1007 1007 'last_login': user.last_login,
1008 1008 'last_activity': user.last_activity,
1009 1009 'ip_addresses': user.ip_addresses,
1010 1010 'language': user_data.get('language')
1011 1011 }
1012 1012 data.update(extras)
1013 1013
1014 1014 if include_secrets:
1015 1015 data['auth_tokens'] = user.auth_tokens
1016 1016 return data
1017 1017
1018 1018 def __json__(self):
1019 1019 data = {
1020 1020 'full_name': self.full_name,
1021 1021 'full_name_or_username': self.full_name_or_username,
1022 1022 'short_contact': self.short_contact,
1023 1023 'full_contact': self.full_contact,
1024 1024 }
1025 1025 data.update(self.get_api_data())
1026 1026 return data
1027 1027
1028 1028
1029 1029 class UserApiKeys(Base, BaseModel):
1030 1030 __tablename__ = 'user_api_keys'
1031 1031 __table_args__ = (
1032 1032 Index('uak_api_key_idx', 'api_key', unique=True),
1033 1033 Index('uak_api_key_expires_idx', 'api_key', 'expires'),
1034 1034 base_table_args
1035 1035 )
1036 1036 __mapper_args__ = {}
1037 1037
1038 1038 # ApiKey role
1039 1039 ROLE_ALL = 'token_role_all'
1040 1040 ROLE_HTTP = 'token_role_http'
1041 1041 ROLE_VCS = 'token_role_vcs'
1042 1042 ROLE_API = 'token_role_api'
1043 1043 ROLE_FEED = 'token_role_feed'
1044 1044 ROLE_PASSWORD_RESET = 'token_password_reset'
1045 1045
1046 1046 ROLES = [ROLE_ALL, ROLE_HTTP, ROLE_VCS, ROLE_API, ROLE_FEED]
1047 1047
1048 1048 user_api_key_id = Column("user_api_key_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
1049 1049 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
1050 1050 api_key = Column("api_key", String(255), nullable=False, unique=True)
1051 1051 description = Column('description', UnicodeText().with_variant(UnicodeText(1024), 'mysql'))
1052 1052 expires = Column('expires', Float(53), nullable=False)
1053 1053 role = Column('role', String(255), nullable=True)
1054 1054 created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
1055 1055
1056 1056 # scope columns
1057 1057 repo_id = Column(
1058 1058 'repo_id', Integer(), ForeignKey('repositories.repo_id'),
1059 1059 nullable=True, unique=None, default=None)
1060 1060 repo = relationship('Repository', lazy='joined')
1061 1061
1062 1062 repo_group_id = Column(
1063 1063 'repo_group_id', Integer(), ForeignKey('groups.group_id'),
1064 1064 nullable=True, unique=None, default=None)
1065 1065 repo_group = relationship('RepoGroup', lazy='joined')
1066 1066
1067 1067 user = relationship('User', lazy='joined')
1068 1068
1069 1069 def __unicode__(self):
1070 1070 return u"<%s('%s')>" % (self.__class__.__name__, self.role)
1071 1071
1072 1072 def __json__(self):
1073 1073 data = {
1074 1074 'auth_token': self.api_key,
1075 1075 'role': self.role,
1076 1076 'scope': self.scope_humanized,
1077 1077 'expired': self.expired
1078 1078 }
1079 1079 return data
1080 1080
1081 1081 def get_api_data(self, include_secrets=False):
1082 1082 data = self.__json__()
1083 1083 if include_secrets:
1084 1084 return data
1085 1085 else:
1086 1086 data['auth_token'] = self.token_obfuscated
1087 1087 return data
1088 1088
1089 1089 @hybrid_property
1090 1090 def description_safe(self):
1091 1091 from rhodecode.lib import helpers as h
1092 1092 return h.escape(self.description)
1093 1093
1094 1094 @property
1095 1095 def expired(self):
1096 1096 if self.expires == -1:
1097 1097 return False
1098 1098 return time.time() > self.expires
1099 1099
1100 1100 @classmethod
1101 1101 def _get_role_name(cls, role):
1102 1102 return {
1103 1103 cls.ROLE_ALL: _('all'),
1104 1104 cls.ROLE_HTTP: _('http/web interface'),
1105 1105 cls.ROLE_VCS: _('vcs (git/hg/svn protocol)'),
1106 1106 cls.ROLE_API: _('api calls'),
1107 1107 cls.ROLE_FEED: _('feed access'),
1108 1108 }.get(role, role)
1109 1109
1110 1110 @property
1111 1111 def role_humanized(self):
1112 1112 return self._get_role_name(self.role)
1113 1113
1114 1114 def _get_scope(self):
1115 1115 if self.repo:
1116 1116 return repr(self.repo)
1117 1117 if self.repo_group:
1118 1118 return repr(self.repo_group) + ' (recursive)'
1119 1119 return 'global'
1120 1120
1121 1121 @property
1122 1122 def scope_humanized(self):
1123 1123 return self._get_scope()
1124 1124
1125 1125 @property
1126 1126 def token_obfuscated(self):
1127 1127 if self.api_key:
1128 1128 return self.api_key[:4] + "****"
1129 1129
1130 1130
1131 1131 class UserEmailMap(Base, BaseModel):
1132 1132 __tablename__ = 'user_email_map'
1133 1133 __table_args__ = (
1134 1134 Index('uem_email_idx', 'email'),
1135 1135 UniqueConstraint('email'),
1136 1136 base_table_args
1137 1137 )
1138 1138 __mapper_args__ = {}
1139 1139
1140 1140 email_id = Column("email_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
1141 1141 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
1142 1142 _email = Column("email", String(255), nullable=True, unique=False, default=None)
1143 1143 user = relationship('User', lazy='joined')
1144 1144
1145 1145 @validates('_email')
1146 1146 def validate_email(self, key, email):
1147 1147 # check if this email is not main one
1148 1148 main_email = Session().query(User).filter(User.email == email).scalar()
1149 1149 if main_email is not None:
1150 1150 raise AttributeError('email %s is present is user table' % email)
1151 1151 return email
1152 1152
1153 1153 @hybrid_property
1154 1154 def email(self):
1155 1155 return self._email
1156 1156
1157 1157 @email.setter
1158 1158 def email(self, val):
1159 1159 self._email = val.lower() if val else None
1160 1160
1161 1161
1162 1162 class UserIpMap(Base, BaseModel):
1163 1163 __tablename__ = 'user_ip_map'
1164 1164 __table_args__ = (
1165 1165 UniqueConstraint('user_id', 'ip_addr'),
1166 1166 base_table_args
1167 1167 )
1168 1168 __mapper_args__ = {}
1169 1169
1170 1170 ip_id = Column("ip_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
1171 1171 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
1172 1172 ip_addr = Column("ip_addr", String(255), nullable=True, unique=False, default=None)
1173 1173 active = Column("active", Boolean(), nullable=True, unique=None, default=True)
1174 1174 description = Column("description", String(10000), nullable=True, unique=None, default=None)
1175 1175 user = relationship('User', lazy='joined')
1176 1176
1177 1177 @hybrid_property
1178 1178 def description_safe(self):
1179 1179 from rhodecode.lib import helpers as h
1180 1180 return h.escape(self.description)
1181 1181
1182 1182 @classmethod
1183 1183 def _get_ip_range(cls, ip_addr):
1184 1184 net = ipaddress.ip_network(safe_unicode(ip_addr), strict=False)
1185 1185 return [str(net.network_address), str(net.broadcast_address)]
1186 1186
1187 1187 def __json__(self):
1188 1188 return {
1189 1189 'ip_addr': self.ip_addr,
1190 1190 'ip_range': self._get_ip_range(self.ip_addr),
1191 1191 }
1192 1192
1193 1193 def __unicode__(self):
1194 1194 return u"<%s('user_id:%s=>%s')>" % (self.__class__.__name__,
1195 1195 self.user_id, self.ip_addr)
1196 1196
1197 1197
1198 1198 class UserSshKeys(Base, BaseModel):
1199 1199 __tablename__ = 'user_ssh_keys'
1200 1200 __table_args__ = (
1201 1201 Index('usk_ssh_key_fingerprint_idx', 'ssh_key_fingerprint'),
1202 1202
1203 1203 UniqueConstraint('ssh_key_fingerprint'),
1204 1204
1205 1205 base_table_args
1206 1206 )
1207 1207 __mapper_args__ = {}
1208 1208
1209 1209 ssh_key_id = Column('ssh_key_id', Integer(), nullable=False, unique=True, default=None, primary_key=True)
1210 1210 ssh_key_data = Column('ssh_key_data', String(10240), nullable=False, unique=None, default=None)
1211 1211 ssh_key_fingerprint = Column('ssh_key_fingerprint', String(255), nullable=False, unique=None, default=None)
1212 1212
1213 1213 description = Column('description', UnicodeText().with_variant(UnicodeText(1024), 'mysql'))
1214 1214
1215 1215 created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
1216 1216 accessed_on = Column('accessed_on', DateTime(timezone=False), nullable=True, default=None)
1217 1217 user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
1218 1218
1219 1219 user = relationship('User', lazy='joined')
1220 1220
1221 1221 def __json__(self):
1222 1222 data = {
1223 1223 'ssh_fingerprint': self.ssh_key_fingerprint,
1224 1224 'description': self.description,
1225 1225 'created_on': self.created_on
1226 1226 }
1227 1227 return data
1228 1228
1229 1229 def get_api_data(self):
1230 1230 data = self.__json__()
1231 1231 return data
1232 1232
1233 1233
1234 1234 class UserLog(Base, BaseModel):
1235 1235 __tablename__ = 'user_logs'
1236 1236 __table_args__ = (
1237 1237 base_table_args,
1238 1238 )
1239 1239
1240 1240 VERSION_1 = 'v1'
1241 1241 VERSION_2 = 'v2'
1242 1242 VERSIONS = [VERSION_1, VERSION_2]
1243 1243
1244 1244 user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
1245 1245 user_id = Column("user_id", Integer(), ForeignKey('users.user_id',ondelete='SET NULL'), nullable=True, unique=None, default=None)
1246 1246 username = Column("username", String(255), nullable=True, unique=None, default=None)
1247 1247 repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id', ondelete='SET NULL'), nullable=True, unique=None, default=None)
1248 1248 repository_name = Column("repository_name", String(255), nullable=True, unique=None, default=None)
1249 1249 user_ip = Column("user_ip", String(255), nullable=True, unique=None, default=None)
1250 1250 action = Column("action", Text().with_variant(Text(1200000), 'mysql'), nullable=True, unique=None, default=None)
1251 1251 action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None)
1252 1252
1253 1253 version = Column("version", String(255), nullable=True, default=VERSION_1)
1254 1254 user_data = Column('user_data_json', MutationObj.as_mutable(JsonType(dialect_map=dict(mysql=LONGTEXT()))))
1255 1255 action_data = Column('action_data_json', MutationObj.as_mutable(JsonType(dialect_map=dict(mysql=LONGTEXT()))))
1256 1256
1257 1257 def __unicode__(self):
1258 1258 return u"<%s('id:%s:%s')>" % (
1259 1259 self.__class__.__name__, self.repository_name, self.action)
1260 1260
1261 1261 def __json__(self):
1262 1262 return {
1263 1263 'user_id': self.user_id,
1264 1264 'username': self.username,
1265 1265 'repository_id': self.repository_id,
1266 1266 'repository_name': self.repository_name,
1267 1267 'user_ip': self.user_ip,
1268 1268 'action_date': self.action_date,
1269 1269 'action': self.action,
1270 1270 }
1271 1271
1272 1272 @hybrid_property
1273 1273 def entry_id(self):
1274 1274 return self.user_log_id
1275 1275
1276 1276 @property
1277 1277 def action_as_day(self):
1278 1278 return datetime.date(*self.action_date.timetuple()[:3])
1279 1279
1280 1280 user = relationship('User')
1281 1281 repository = relationship('Repository', cascade='')
1282 1282
1283 1283
1284 1284 class UserGroup(Base, BaseModel):
1285 1285 __tablename__ = 'users_groups'
1286 1286 __table_args__ = (
1287 1287 base_table_args,
1288 1288 )
1289 1289
1290 1290 users_group_id = Column("users_group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
1291 1291 users_group_name = Column("users_group_name", String(255), nullable=False, unique=True, default=None)
1292 1292 user_group_description = Column("user_group_description", String(10000), nullable=True, unique=None, default=None)
1293 1293 users_group_active = Column("users_group_active", Boolean(), nullable=True, unique=None, default=None)
1294 1294 inherit_default_permissions = Column("users_group_inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True)
1295 1295 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)
1296 1296 created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
1297 1297 _group_data = Column("group_data", LargeBinary(), nullable=True) # JSON data
1298 1298
1299 1299 members = relationship('UserGroupMember', cascade="all, delete, delete-orphan", lazy="joined")
1300 1300 users_group_to_perm = relationship('UserGroupToPerm', cascade='all')
1301 1301 users_group_repo_to_perm = relationship('UserGroupRepoToPerm', cascade='all')
1302 1302 users_group_repo_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
1303 1303 user_user_group_to_perm = relationship('UserUserGroupToPerm', cascade='all')
1304 1304 user_group_user_group_to_perm = relationship('UserGroupUserGroupToPerm ', primaryjoin="UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id", cascade='all')
1305 1305
1306 1306 user_group_review_rules = relationship('RepoReviewRuleUserGroup', cascade='all')
1307 1307 user = relationship('User', primaryjoin="User.user_id==UserGroup.user_id")
1308 1308
1309 1309 @classmethod
1310 1310 def _load_group_data(cls, column):
1311 1311 if not column:
1312 1312 return {}
1313 1313
1314 1314 try:
1315 1315 return json.loads(column) or {}
1316 1316 except TypeError:
1317 1317 return {}
1318 1318
1319 1319 @hybrid_property
1320 1320 def description_safe(self):
1321 1321 from rhodecode.lib import helpers as h
1322 1322 return h.escape(self.user_group_description)
1323 1323
1324 1324 @hybrid_property
1325 1325 def group_data(self):
1326 1326 return self._load_group_data(self._group_data)
1327 1327
1328 1328 @group_data.expression
1329 1329 def group_data(self, **kwargs):
1330 1330 return self._group_data
1331 1331
1332 1332 @group_data.setter
1333 1333 def group_data(self, val):
1334 1334 try:
1335 1335 self._group_data = json.dumps(val)
1336 1336 except Exception:
1337 1337 log.error(traceback.format_exc())
1338 1338
1339 1339 @classmethod
1340 1340 def _load_sync(cls, group_data):
1341 1341 if group_data:
1342 1342 return group_data.get('extern_type')
1343 1343
1344 1344 @property
1345 1345 def sync(self):
1346 1346 return self._load_sync(self.group_data)
1347 1347
1348 1348 def __unicode__(self):
1349 1349 return u"<%s('id:%s:%s')>" % (self.__class__.__name__,
1350 1350 self.users_group_id,
1351 1351 self.users_group_name)
1352 1352
1353 1353 @classmethod
1354 1354 def get_by_group_name(cls, group_name, cache=False,
1355 1355 case_insensitive=False):
1356 1356 if case_insensitive:
1357 1357 q = cls.query().filter(func.lower(cls.users_group_name) ==
1358 1358 func.lower(group_name))
1359 1359
1360 1360 else:
1361 1361 q = cls.query().filter(cls.users_group_name == group_name)
1362 1362 if cache:
1363 1363 q = q.options(
1364 1364 FromCache("sql_cache_short", "get_group_%s" % _hash_key(group_name)))
1365 1365 return q.scalar()
1366 1366
1367 1367 @classmethod
1368 1368 def get(cls, user_group_id, cache=False):
1369 1369 if not user_group_id:
1370 1370 return
1371 1371
1372 1372 user_group = cls.query()
1373 1373 if cache:
1374 1374 user_group = user_group.options(
1375 1375 FromCache("sql_cache_short", "get_users_group_%s" % user_group_id))
1376 1376 return user_group.get(user_group_id)
1377 1377
1378 1378 def permissions(self, with_admins=True, with_owner=True):
1379 1379 q = UserUserGroupToPerm.query().filter(UserUserGroupToPerm.user_group == self)
1380 1380 q = q.options(joinedload(UserUserGroupToPerm.user_group),
1381 1381 joinedload(UserUserGroupToPerm.user),
1382 1382 joinedload(UserUserGroupToPerm.permission),)
1383 1383
1384 1384 # get owners and admins and permissions. We do a trick of re-writing
1385 1385 # objects from sqlalchemy to named-tuples due to sqlalchemy session
1386 1386 # has a global reference and changing one object propagates to all
1387 1387 # others. This means if admin is also an owner admin_row that change
1388 1388 # would propagate to both objects
1389 1389 perm_rows = []
1390 1390 for _usr in q.all():
1391 1391 usr = AttributeDict(_usr.user.get_dict())
1392 1392 usr.permission = _usr.permission.permission_name
1393 1393 perm_rows.append(usr)
1394 1394
1395 1395 # filter the perm rows by 'default' first and then sort them by
1396 1396 # admin,write,read,none permissions sorted again alphabetically in
1397 1397 # each group
1398 1398 perm_rows = sorted(perm_rows, key=display_user_sort)
1399 1399
1400 1400 _admin_perm = 'usergroup.admin'
1401 1401 owner_row = []
1402 1402 if with_owner:
1403 1403 usr = AttributeDict(self.user.get_dict())
1404 1404 usr.owner_row = True
1405 1405 usr.permission = _admin_perm
1406 1406 owner_row.append(usr)
1407 1407
1408 1408 super_admin_rows = []
1409 1409 if with_admins:
1410 1410 for usr in User.get_all_super_admins():
1411 1411 # if this admin is also owner, don't double the record
1412 1412 if usr.user_id == owner_row[0].user_id:
1413 1413 owner_row[0].admin_row = True
1414 1414 else:
1415 1415 usr = AttributeDict(usr.get_dict())
1416 1416 usr.admin_row = True
1417 1417 usr.permission = _admin_perm
1418 1418 super_admin_rows.append(usr)
1419 1419
1420 1420 return super_admin_rows + owner_row + perm_rows
1421 1421
1422 1422 def permission_user_groups(self):
1423 1423 q = UserGroupUserGroupToPerm.query().filter(UserGroupUserGroupToPerm.target_user_group == self)
1424 1424 q = q.options(joinedload(UserGroupUserGroupToPerm.user_group),
1425 1425 joinedload(UserGroupUserGroupToPerm.target_user_group),
1426 1426 joinedload(UserGroupUserGroupToPerm.permission),)
1427 1427
1428 1428 perm_rows = []
1429 1429 for _user_group in q.all():
1430 1430 usr = AttributeDict(_user_group.user_group.get_dict())
1431 1431 usr.permission = _user_group.permission.permission_name
1432 1432 perm_rows.append(usr)
1433 1433
1434 1434 perm_rows = sorted(perm_rows, key=display_user_group_sort)
1435 1435 return perm_rows
1436 1436
1437 1437 def _get_default_perms(self, user_group, suffix=''):
1438 1438 from rhodecode.model.permission import PermissionModel
1439 1439 return PermissionModel().get_default_perms(user_group.users_group_to_perm, suffix)
1440 1440
1441 1441 def get_default_perms(self, suffix=''):
1442 1442 return self._get_default_perms(self, suffix)
1443 1443
1444 1444 def get_api_data(self, with_group_members=True, include_secrets=False):
1445 1445 """
1446 1446 :param include_secrets: See :meth:`User.get_api_data`, this parameter is
1447 1447 basically forwarded.
1448 1448
1449 1449 """
1450 1450 user_group = self
1451 1451 data = {
1452 1452 'users_group_id': user_group.users_group_id,
1453 1453 'group_name': user_group.users_group_name,
1454 1454 'group_description': user_group.user_group_description,
1455 1455 'active': user_group.users_group_active,
1456 1456 'owner': user_group.user.username,
1457 1457 'sync': user_group.sync,
1458 1458 'owner_email': user_group.user.email,
1459 1459 }
1460 1460
1461 1461 if with_group_members:
1462 1462 users = []
1463 1463 for user in user_group.members:
1464 1464 user = user.user
1465 1465 users.append(user.get_api_data(include_secrets=include_secrets))
1466 1466 data['users'] = users
1467 1467
1468 1468 return data
1469 1469
1470 1470
1471 1471 class UserGroupMember(Base, BaseModel):
1472 1472 __tablename__ = 'users_groups_members'
1473 1473 __table_args__ = (
1474 1474 base_table_args,
1475 1475 )
1476 1476
1477 1477 users_group_member_id = Column("users_group_member_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
1478 1478 users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
1479 1479 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
1480 1480
1481 1481 user = relationship('User', lazy='joined')
1482 1482 users_group = relationship('UserGroup')
1483 1483
1484 1484 def __init__(self, gr_id='', u_id=''):
1485 1485 self.users_group_id = gr_id
1486 1486 self.user_id = u_id
1487 1487
1488 1488
1489 1489 class RepositoryField(Base, BaseModel):
1490 1490 __tablename__ = 'repositories_fields'
1491 1491 __table_args__ = (
1492 1492 UniqueConstraint('repository_id', 'field_key'), # no-multi field
1493 1493 base_table_args,
1494 1494 )
1495 1495
1496 1496 PREFIX = 'ex_' # prefix used in form to not conflict with already existing fields
1497 1497
1498 1498 repo_field_id = Column("repo_field_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
1499 1499 repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
1500 1500 field_key = Column("field_key", String(250))
1501 1501 field_label = Column("field_label", String(1024), nullable=False)
1502 1502 field_value = Column("field_value", String(10000), nullable=False)
1503 1503 field_desc = Column("field_desc", String(1024), nullable=False)
1504 1504 field_type = Column("field_type", String(255), nullable=False, unique=None)
1505 1505 created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
1506 1506
1507 1507 repository = relationship('Repository')
1508 1508
1509 1509 @property
1510 1510 def field_key_prefixed(self):
1511 1511 return 'ex_%s' % self.field_key
1512 1512
1513 1513 @classmethod
1514 1514 def un_prefix_key(cls, key):
1515 1515 if key.startswith(cls.PREFIX):
1516 1516 return key[len(cls.PREFIX):]
1517 1517 return key
1518 1518
1519 1519 @classmethod
1520 1520 def get_by_key_name(cls, key, repo):
1521 1521 row = cls.query()\
1522 1522 .filter(cls.repository == repo)\
1523 1523 .filter(cls.field_key == key).scalar()
1524 1524 return row
1525 1525
1526 1526
1527 1527 class Repository(Base, BaseModel):
1528 1528 __tablename__ = 'repositories'
1529 1529 __table_args__ = (
1530 1530 Index('r_repo_name_idx', 'repo_name', mysql_length=255),
1531 1531 base_table_args,
1532 1532 )
1533 1533 DEFAULT_CLONE_URI = '{scheme}://{user}@{netloc}/{repo}'
1534 1534 DEFAULT_CLONE_URI_ID = '{scheme}://{user}@{netloc}/_{repoid}'
1535 1535 DEFAULT_CLONE_URI_SSH = 'ssh://{sys_user}@{hostname}/{repo}'
1536 1536
1537 1537 STATE_CREATED = 'repo_state_created'
1538 1538 STATE_PENDING = 'repo_state_pending'
1539 1539 STATE_ERROR = 'repo_state_error'
1540 1540
1541 1541 LOCK_AUTOMATIC = 'lock_auto'
1542 1542 LOCK_API = 'lock_api'
1543 1543 LOCK_WEB = 'lock_web'
1544 1544 LOCK_PULL = 'lock_pull'
1545 1545
1546 1546 NAME_SEP = URL_SEP
1547 1547
1548 1548 repo_id = Column(
1549 1549 "repo_id", Integer(), nullable=False, unique=True, default=None,
1550 1550 primary_key=True)
1551 1551 _repo_name = Column(
1552 1552 "repo_name", Text(), nullable=False, default=None)
1553 1553 _repo_name_hash = Column(
1554 1554 "repo_name_hash", String(255), nullable=False, unique=True)
1555 1555 repo_state = Column("repo_state", String(255), nullable=True)
1556 1556
1557 1557 clone_uri = Column(
1558 1558 "clone_uri", EncryptedTextValue(), nullable=True, unique=False,
1559 1559 default=None)
1560 1560 push_uri = Column(
1561 1561 "push_uri", EncryptedTextValue(), nullable=True, unique=False,
1562 1562 default=None)
1563 1563 repo_type = Column(
1564 1564 "repo_type", String(255), nullable=False, unique=False, default=None)
1565 1565 user_id = Column(
1566 1566 "user_id", Integer(), ForeignKey('users.user_id'), nullable=False,
1567 1567 unique=False, default=None)
1568 1568 private = Column(
1569 1569 "private", Boolean(), nullable=True, unique=None, default=None)
1570 1570 enable_statistics = Column(
1571 1571 "statistics", Boolean(), nullable=True, unique=None, default=True)
1572 1572 enable_downloads = Column(
1573 1573 "downloads", Boolean(), nullable=True, unique=None, default=True)
1574 1574 description = Column(
1575 1575 "description", String(10000), nullable=True, unique=None, default=None)
1576 1576 created_on = Column(
1577 1577 'created_on', DateTime(timezone=False), nullable=True, unique=None,
1578 1578 default=datetime.datetime.now)
1579 1579 updated_on = Column(
1580 1580 'updated_on', DateTime(timezone=False), nullable=True, unique=None,
1581 1581 default=datetime.datetime.now)
1582 1582 _landing_revision = Column(
1583 1583 "landing_revision", String(255), nullable=False, unique=False,
1584 1584 default=None)
1585 1585 enable_locking = Column(
1586 1586 "enable_locking", Boolean(), nullable=False, unique=None,
1587 1587 default=False)
1588 1588 _locked = Column(
1589 1589 "locked", String(255), nullable=True, unique=False, default=None)
1590 1590 _changeset_cache = Column(
1591 1591 "changeset_cache", LargeBinary(), nullable=True) # JSON data
1592 1592
1593 1593 fork_id = Column(
1594 1594 "fork_id", Integer(), ForeignKey('repositories.repo_id'),
1595 1595 nullable=True, unique=False, default=None)
1596 1596 group_id = Column(
1597 1597 "group_id", Integer(), ForeignKey('groups.group_id'), nullable=True,
1598 1598 unique=False, default=None)
1599 1599
1600 1600 user = relationship('User', lazy='joined')
1601 1601 fork = relationship('Repository', remote_side=repo_id, lazy='joined')
1602 1602 group = relationship('RepoGroup', lazy='joined')
1603 1603 repo_to_perm = relationship(
1604 1604 'UserRepoToPerm', cascade='all',
1605 1605 order_by='UserRepoToPerm.repo_to_perm_id')
1606 1606 users_group_to_perm = relationship('UserGroupRepoToPerm', cascade='all')
1607 1607 stats = relationship('Statistics', cascade='all', uselist=False)
1608 1608
1609 1609 followers = relationship(
1610 1610 'UserFollowing',
1611 1611 primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id',
1612 1612 cascade='all')
1613 1613 extra_fields = relationship(
1614 1614 'RepositoryField', cascade="all, delete, delete-orphan")
1615 1615 logs = relationship('UserLog')
1616 1616 comments = relationship(
1617 1617 'ChangesetComment', cascade="all, delete, delete-orphan")
1618 1618 pull_requests_source = relationship(
1619 1619 'PullRequest',
1620 1620 primaryjoin='PullRequest.source_repo_id==Repository.repo_id',
1621 1621 cascade="all, delete, delete-orphan")
1622 1622 pull_requests_target = relationship(
1623 1623 'PullRequest',
1624 1624 primaryjoin='PullRequest.target_repo_id==Repository.repo_id',
1625 1625 cascade="all, delete, delete-orphan")
1626 1626 ui = relationship('RepoRhodeCodeUi', cascade="all")
1627 1627 settings = relationship('RepoRhodeCodeSetting', cascade="all")
1628 1628 integrations = relationship('Integration',
1629 1629 cascade="all, delete, delete-orphan")
1630 1630
1631 1631 scoped_tokens = relationship('UserApiKeys', cascade="all")
1632 1632
1633 1633 def __unicode__(self):
1634 1634 return u"<%s('%s:%s')>" % (self.__class__.__name__, self.repo_id,
1635 1635 safe_unicode(self.repo_name))
1636 1636
1637 1637 @hybrid_property
1638 1638 def description_safe(self):
1639 1639 from rhodecode.lib import helpers as h
1640 1640 return h.escape(self.description)
1641 1641
1642 1642 @hybrid_property
1643 1643 def landing_rev(self):
1644 1644 # always should return [rev_type, rev]
1645 1645 if self._landing_revision:
1646 1646 _rev_info = self._landing_revision.split(':')
1647 1647 if len(_rev_info) < 2:
1648 1648 _rev_info.insert(0, 'rev')
1649 1649 return [_rev_info[0], _rev_info[1]]
1650 1650 return [None, None]
1651 1651
1652 1652 @landing_rev.setter
1653 1653 def landing_rev(self, val):
1654 1654 if ':' not in val:
1655 1655 raise ValueError('value must be delimited with `:` and consist '
1656 1656 'of <rev_type>:<rev>, got %s instead' % val)
1657 1657 self._landing_revision = val
1658 1658
1659 1659 @hybrid_property
1660 1660 def locked(self):
1661 1661 if self._locked:
1662 1662 user_id, timelocked, reason = self._locked.split(':')
1663 1663 lock_values = int(user_id), timelocked, reason
1664 1664 else:
1665 1665 lock_values = [None, None, None]
1666 1666 return lock_values
1667 1667
1668 1668 @locked.setter
1669 1669 def locked(self, val):
1670 1670 if val and isinstance(val, (list, tuple)):
1671 1671 self._locked = ':'.join(map(str, val))
1672 1672 else:
1673 1673 self._locked = None
1674 1674
1675 1675 @hybrid_property
1676 1676 def changeset_cache(self):
1677 1677 from rhodecode.lib.vcs.backends.base import EmptyCommit
1678 1678 dummy = EmptyCommit().__json__()
1679 1679 if not self._changeset_cache:
1680 1680 return dummy
1681 1681 try:
1682 1682 return json.loads(self._changeset_cache)
1683 1683 except TypeError:
1684 1684 return dummy
1685 1685 except Exception:
1686 1686 log.error(traceback.format_exc())
1687 1687 return dummy
1688 1688
1689 1689 @changeset_cache.setter
1690 1690 def changeset_cache(self, val):
1691 1691 try:
1692 1692 self._changeset_cache = json.dumps(val)
1693 1693 except Exception:
1694 1694 log.error(traceback.format_exc())
1695 1695
1696 1696 @hybrid_property
1697 1697 def repo_name(self):
1698 1698 return self._repo_name
1699 1699
1700 1700 @repo_name.setter
1701 1701 def repo_name(self, value):
1702 1702 self._repo_name = value
1703 1703 self._repo_name_hash = hashlib.sha1(safe_str(value)).hexdigest()
1704 1704
1705 1705 @classmethod
1706 1706 def normalize_repo_name(cls, repo_name):
1707 1707 """
1708 1708 Normalizes os specific repo_name to the format internally stored inside
1709 1709 database using URL_SEP
1710 1710
1711 1711 :param cls:
1712 1712 :param repo_name:
1713 1713 """
1714 1714 return cls.NAME_SEP.join(repo_name.split(os.sep))
1715 1715
1716 1716 @classmethod
1717 1717 def get_by_repo_name(cls, repo_name, cache=False, identity_cache=False):
1718 1718 session = Session()
1719 1719 q = session.query(cls).filter(cls.repo_name == repo_name)
1720 1720
1721 1721 if cache:
1722 1722 if identity_cache:
1723 1723 val = cls.identity_cache(session, 'repo_name', repo_name)
1724 1724 if val:
1725 1725 return val
1726 1726 else:
1727 1727 cache_key = "get_repo_by_name_%s" % _hash_key(repo_name)
1728 1728 q = q.options(
1729 1729 FromCache("sql_cache_short", cache_key))
1730 1730
1731 1731 return q.scalar()
1732 1732
1733 1733 @classmethod
1734 1734 def get_by_id_or_repo_name(cls, repoid):
1735 1735 if isinstance(repoid, (int, long)):
1736 1736 try:
1737 1737 repo = cls.get(repoid)
1738 1738 except ValueError:
1739 1739 repo = None
1740 1740 else:
1741 1741 repo = cls.get_by_repo_name(repoid)
1742 1742 return repo
1743 1743
1744 1744 @classmethod
1745 1745 def get_by_full_path(cls, repo_full_path):
1746 1746 repo_name = repo_full_path.split(cls.base_path(), 1)[-1]
1747 1747 repo_name = cls.normalize_repo_name(repo_name)
1748 1748 return cls.get_by_repo_name(repo_name.strip(URL_SEP))
1749 1749
1750 1750 @classmethod
1751 1751 def get_repo_forks(cls, repo_id):
1752 1752 return cls.query().filter(Repository.fork_id == repo_id)
1753 1753
1754 1754 @classmethod
1755 1755 def base_path(cls):
1756 1756 """
1757 1757 Returns base path when all repos are stored
1758 1758
1759 1759 :param cls:
1760 1760 """
1761 1761 q = Session().query(RhodeCodeUi)\
1762 1762 .filter(RhodeCodeUi.ui_key == cls.NAME_SEP)
1763 1763 q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
1764 1764 return q.one().ui_value
1765 1765
1766 1766 @classmethod
1767 1767 def get_all_repos(cls, user_id=Optional(None), group_id=Optional(None),
1768 1768 case_insensitive=True):
1769 1769 q = Repository.query()
1770 1770
1771 1771 if not isinstance(user_id, Optional):
1772 1772 q = q.filter(Repository.user_id == user_id)
1773 1773
1774 1774 if not isinstance(group_id, Optional):
1775 1775 q = q.filter(Repository.group_id == group_id)
1776 1776
1777 1777 if case_insensitive:
1778 1778 q = q.order_by(func.lower(Repository.repo_name))
1779 1779 else:
1780 1780 q = q.order_by(Repository.repo_name)
1781 1781 return q.all()
1782 1782
1783 1783 @property
1784 1784 def forks(self):
1785 1785 """
1786 1786 Return forks of this repo
1787 1787 """
1788 1788 return Repository.get_repo_forks(self.repo_id)
1789 1789
1790 1790 @property
1791 1791 def parent(self):
1792 1792 """
1793 1793 Returns fork parent
1794 1794 """
1795 1795 return self.fork
1796 1796
1797 1797 @property
1798 1798 def just_name(self):
1799 1799 return self.repo_name.split(self.NAME_SEP)[-1]
1800 1800
1801 1801 @property
1802 1802 def groups_with_parents(self):
1803 1803 groups = []
1804 1804 if self.group is None:
1805 1805 return groups
1806 1806
1807 1807 cur_gr = self.group
1808 1808 groups.insert(0, cur_gr)
1809 1809 while 1:
1810 1810 gr = getattr(cur_gr, 'parent_group', None)
1811 1811 cur_gr = cur_gr.parent_group
1812 1812 if gr is None:
1813 1813 break
1814 1814 groups.insert(0, gr)
1815 1815
1816 1816 return groups
1817 1817
1818 1818 @property
1819 1819 def groups_and_repo(self):
1820 1820 return self.groups_with_parents, self
1821 1821
1822 1822 @LazyProperty
1823 1823 def repo_path(self):
1824 1824 """
1825 1825 Returns base full path for that repository means where it actually
1826 1826 exists on a filesystem
1827 1827 """
1828 1828 q = Session().query(RhodeCodeUi).filter(
1829 1829 RhodeCodeUi.ui_key == self.NAME_SEP)
1830 1830 q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
1831 1831 return q.one().ui_value
1832 1832
1833 1833 @property
1834 1834 def repo_full_path(self):
1835 1835 p = [self.repo_path]
1836 1836 # we need to split the name by / since this is how we store the
1837 1837 # names in the database, but that eventually needs to be converted
1838 1838 # into a valid system path
1839 1839 p += self.repo_name.split(self.NAME_SEP)
1840 1840 return os.path.join(*map(safe_unicode, p))
1841 1841
1842 1842 @property
1843 1843 def cache_keys(self):
1844 1844 """
1845 1845 Returns associated cache keys for that repo
1846 1846 """
1847 1847 invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
1848 1848 repo_id=self.repo_id)
1849 1849 return CacheKey.query()\
1850 1850 .filter(CacheKey.cache_args == invalidation_namespace)\
1851 1851 .order_by(CacheKey.cache_key)\
1852 1852 .all()
1853 1853
1854 1854 @property
1855 1855 def cached_diffs_relative_dir(self):
1856 1856 """
1857 1857 Return a relative to the repository store path of cached diffs
1858 1858 used for safe display for users, who shouldn't know the absolute store
1859 1859 path
1860 1860 """
1861 1861 return os.path.join(
1862 1862 os.path.dirname(self.repo_name),
1863 1863 self.cached_diffs_dir.split(os.path.sep)[-1])
1864 1864
1865 1865 @property
1866 1866 def cached_diffs_dir(self):
1867 1867 path = self.repo_full_path
1868 1868 return os.path.join(
1869 1869 os.path.dirname(path),
1870 1870 '.__shadow_diff_cache_repo_{}'.format(self.repo_id))
1871 1871
1872 1872 def cached_diffs(self):
1873 1873 diff_cache_dir = self.cached_diffs_dir
1874 1874 if os.path.isdir(diff_cache_dir):
1875 1875 return os.listdir(diff_cache_dir)
1876 1876 return []
1877 1877
1878 1878 def shadow_repos(self):
1879 1879 shadow_repos_pattern = '.__shadow_repo_{}'.format(self.repo_id)
1880 1880 return [
1881 1881 x for x in os.listdir(os.path.dirname(self.repo_full_path))
1882 1882 if x.startswith(shadow_repos_pattern)]
1883 1883
1884 1884 def get_new_name(self, repo_name):
1885 1885 """
1886 1886 returns new full repository name based on assigned group and new new
1887 1887
1888 1888 :param group_name:
1889 1889 """
1890 1890 path_prefix = self.group.full_path_splitted if self.group else []
1891 1891 return self.NAME_SEP.join(path_prefix + [repo_name])
1892 1892
1893 1893 @property
1894 1894 def _config(self):
1895 1895 """
1896 1896 Returns db based config object.
1897 1897 """
1898 1898 from rhodecode.lib.utils import make_db_config
1899 1899 return make_db_config(clear_session=False, repo=self)
1900 1900
1901 1901 def permissions(self, with_admins=True, with_owner=True):
1902 1902 q = UserRepoToPerm.query().filter(UserRepoToPerm.repository == self)
1903 1903 q = q.options(joinedload(UserRepoToPerm.repository),
1904 1904 joinedload(UserRepoToPerm.user),
1905 1905 joinedload(UserRepoToPerm.permission),)
1906 1906
1907 1907 # get owners and admins and permissions. We do a trick of re-writing
1908 1908 # objects from sqlalchemy to named-tuples due to sqlalchemy session
1909 1909 # has a global reference and changing one object propagates to all
1910 1910 # others. This means if admin is also an owner admin_row that change
1911 1911 # would propagate to both objects
1912 1912 perm_rows = []
1913 1913 for _usr in q.all():
1914 1914 usr = AttributeDict(_usr.user.get_dict())
1915 1915 usr.permission = _usr.permission.permission_name
1916 1916 perm_rows.append(usr)
1917 1917
1918 1918 # filter the perm rows by 'default' first and then sort them by
1919 1919 # admin,write,read,none permissions sorted again alphabetically in
1920 1920 # each group
1921 1921 perm_rows = sorted(perm_rows, key=display_user_sort)
1922 1922
1923 1923 _admin_perm = 'repository.admin'
1924 1924 owner_row = []
1925 1925 if with_owner:
1926 1926 usr = AttributeDict(self.user.get_dict())
1927 1927 usr.owner_row = True
1928 1928 usr.permission = _admin_perm
1929 1929 owner_row.append(usr)
1930 1930
1931 1931 super_admin_rows = []
1932 1932 if with_admins:
1933 1933 for usr in User.get_all_super_admins():
1934 1934 # if this admin is also owner, don't double the record
1935 1935 if usr.user_id == owner_row[0].user_id:
1936 1936 owner_row[0].admin_row = True
1937 1937 else:
1938 1938 usr = AttributeDict(usr.get_dict())
1939 1939 usr.admin_row = True
1940 1940 usr.permission = _admin_perm
1941 1941 super_admin_rows.append(usr)
1942 1942
1943 1943 return super_admin_rows + owner_row + perm_rows
1944 1944
1945 1945 def permission_user_groups(self):
1946 1946 q = UserGroupRepoToPerm.query().filter(
1947 1947 UserGroupRepoToPerm.repository == self)
1948 1948 q = q.options(joinedload(UserGroupRepoToPerm.repository),
1949 1949 joinedload(UserGroupRepoToPerm.users_group),
1950 1950 joinedload(UserGroupRepoToPerm.permission),)
1951 1951
1952 1952 perm_rows = []
1953 1953 for _user_group in q.all():
1954 1954 usr = AttributeDict(_user_group.users_group.get_dict())
1955 1955 usr.permission = _user_group.permission.permission_name
1956 1956 perm_rows.append(usr)
1957 1957
1958 1958 perm_rows = sorted(perm_rows, key=display_user_group_sort)
1959 1959 return perm_rows
1960 1960
1961 1961 def get_api_data(self, include_secrets=False):
1962 1962 """
1963 1963 Common function for generating repo api data
1964 1964
1965 1965 :param include_secrets: See :meth:`User.get_api_data`.
1966 1966
1967 1967 """
1968 1968 # TODO: mikhail: Here there is an anti-pattern, we probably need to
1969 1969 # move this methods on models level.
1970 1970 from rhodecode.model.settings import SettingsModel
1971 1971 from rhodecode.model.repo import RepoModel
1972 1972
1973 1973 repo = self
1974 1974 _user_id, _time, _reason = self.locked
1975 1975
1976 1976 data = {
1977 1977 'repo_id': repo.repo_id,
1978 1978 'repo_name': repo.repo_name,
1979 1979 'repo_type': repo.repo_type,
1980 1980 'clone_uri': repo.clone_uri or '',
1981 1981 'push_uri': repo.push_uri or '',
1982 1982 'url': RepoModel().get_url(self),
1983 1983 'private': repo.private,
1984 1984 'created_on': repo.created_on,
1985 1985 'description': repo.description_safe,
1986 1986 'landing_rev': repo.landing_rev,
1987 1987 'owner': repo.user.username,
1988 1988 'fork_of': repo.fork.repo_name if repo.fork else None,
1989 1989 'fork_of_id': repo.fork.repo_id if repo.fork else None,
1990 1990 'enable_statistics': repo.enable_statistics,
1991 1991 'enable_locking': repo.enable_locking,
1992 1992 'enable_downloads': repo.enable_downloads,
1993 1993 'last_changeset': repo.changeset_cache,
1994 1994 'locked_by': User.get(_user_id).get_api_data(
1995 1995 include_secrets=include_secrets) if _user_id else None,
1996 1996 'locked_date': time_to_datetime(_time) if _time else None,
1997 1997 'lock_reason': _reason if _reason else None,
1998 1998 }
1999 1999
2000 2000 # TODO: mikhail: should be per-repo settings here
2001 2001 rc_config = SettingsModel().get_all_settings()
2002 2002 repository_fields = str2bool(
2003 2003 rc_config.get('rhodecode_repository_fields'))
2004 2004 if repository_fields:
2005 2005 for f in self.extra_fields:
2006 2006 data[f.field_key_prefixed] = f.field_value
2007 2007
2008 2008 return data
2009 2009
2010 2010 @classmethod
2011 2011 def lock(cls, repo, user_id, lock_time=None, lock_reason=None):
2012 2012 if not lock_time:
2013 2013 lock_time = time.time()
2014 2014 if not lock_reason:
2015 2015 lock_reason = cls.LOCK_AUTOMATIC
2016 2016 repo.locked = [user_id, lock_time, lock_reason]
2017 2017 Session().add(repo)
2018 2018 Session().commit()
2019 2019
2020 2020 @classmethod
2021 2021 def unlock(cls, repo):
2022 2022 repo.locked = None
2023 2023 Session().add(repo)
2024 2024 Session().commit()
2025 2025
2026 2026 @classmethod
2027 2027 def getlock(cls, repo):
2028 2028 return repo.locked
2029 2029
2030 2030 def is_user_lock(self, user_id):
2031 2031 if self.lock[0]:
2032 2032 lock_user_id = safe_int(self.lock[0])
2033 2033 user_id = safe_int(user_id)
2034 2034 # both are ints, and they are equal
2035 2035 return all([lock_user_id, user_id]) and lock_user_id == user_id
2036 2036
2037 2037 return False
2038 2038
2039 2039 def get_locking_state(self, action, user_id, only_when_enabled=True):
2040 2040 """
2041 2041 Checks locking on this repository, if locking is enabled and lock is
2042 2042 present returns a tuple of make_lock, locked, locked_by.
2043 2043 make_lock can have 3 states None (do nothing) True, make lock
2044 2044 False release lock, This value is later propagated to hooks, which
2045 2045 do the locking. Think about this as signals passed to hooks what to do.
2046 2046
2047 2047 """
2048 2048 # TODO: johbo: This is part of the business logic and should be moved
2049 2049 # into the RepositoryModel.
2050 2050
2051 2051 if action not in ('push', 'pull'):
2052 2052 raise ValueError("Invalid action value: %s" % repr(action))
2053 2053
2054 2054 # defines if locked error should be thrown to user
2055 2055 currently_locked = False
2056 2056 # defines if new lock should be made, tri-state
2057 2057 make_lock = None
2058 2058 repo = self
2059 2059 user = User.get(user_id)
2060 2060
2061 2061 lock_info = repo.locked
2062 2062
2063 2063 if repo and (repo.enable_locking or not only_when_enabled):
2064 2064 if action == 'push':
2065 2065 # check if it's already locked !, if it is compare users
2066 2066 locked_by_user_id = lock_info[0]
2067 2067 if user.user_id == locked_by_user_id:
2068 2068 log.debug(
2069 2069 'Got `push` action from user %s, now unlocking', user)
2070 2070 # unlock if we have push from user who locked
2071 2071 make_lock = False
2072 2072 else:
2073 2073 # we're not the same user who locked, ban with
2074 2074 # code defined in settings (default is 423 HTTP Locked) !
2075 2075 log.debug('Repo %s is currently locked by %s', repo, user)
2076 2076 currently_locked = True
2077 2077 elif action == 'pull':
2078 2078 # [0] user [1] date
2079 2079 if lock_info[0] and lock_info[1]:
2080 2080 log.debug('Repo %s is currently locked by %s', repo, user)
2081 2081 currently_locked = True
2082 2082 else:
2083 2083 log.debug('Setting lock on repo %s by %s', repo, user)
2084 2084 make_lock = True
2085 2085
2086 2086 else:
2087 2087 log.debug('Repository %s do not have locking enabled', repo)
2088 2088
2089 2089 log.debug('FINAL locking values make_lock:%s,locked:%s,locked_by:%s',
2090 2090 make_lock, currently_locked, lock_info)
2091 2091
2092 2092 from rhodecode.lib.auth import HasRepoPermissionAny
2093 2093 perm_check = HasRepoPermissionAny('repository.write', 'repository.admin')
2094 2094 if make_lock and not perm_check(repo_name=repo.repo_name, user=user):
2095 2095 # if we don't have at least write permission we cannot make a lock
2096 2096 log.debug('lock state reset back to FALSE due to lack '
2097 2097 'of at least read permission')
2098 2098 make_lock = False
2099 2099
2100 2100 return make_lock, currently_locked, lock_info
2101 2101
2102 2102 @property
2103 2103 def last_db_change(self):
2104 2104 return self.updated_on
2105 2105
2106 2106 @property
2107 2107 def clone_uri_hidden(self):
2108 2108 clone_uri = self.clone_uri
2109 2109 if clone_uri:
2110 2110 import urlobject
2111 2111 url_obj = urlobject.URLObject(cleaned_uri(clone_uri))
2112 2112 if url_obj.password:
2113 2113 clone_uri = url_obj.with_password('*****')
2114 2114 return clone_uri
2115 2115
2116 2116 @property
2117 2117 def push_uri_hidden(self):
2118 2118 push_uri = self.push_uri
2119 2119 if push_uri:
2120 2120 import urlobject
2121 2121 url_obj = urlobject.URLObject(cleaned_uri(push_uri))
2122 2122 if url_obj.password:
2123 2123 push_uri = url_obj.with_password('*****')
2124 2124 return push_uri
2125 2125
2126 2126 def clone_url(self, **override):
2127 2127 from rhodecode.model.settings import SettingsModel
2128 2128
2129 2129 uri_tmpl = None
2130 2130 if 'with_id' in override:
2131 2131 uri_tmpl = self.DEFAULT_CLONE_URI_ID
2132 2132 del override['with_id']
2133 2133
2134 2134 if 'uri_tmpl' in override:
2135 2135 uri_tmpl = override['uri_tmpl']
2136 2136 del override['uri_tmpl']
2137 2137
2138 2138 ssh = False
2139 2139 if 'ssh' in override:
2140 2140 ssh = True
2141 2141 del override['ssh']
2142 2142
2143 2143 # we didn't override our tmpl from **overrides
2144 2144 if not uri_tmpl:
2145 2145 rc_config = SettingsModel().get_all_settings(cache=True)
2146 2146 if ssh:
2147 2147 uri_tmpl = rc_config.get(
2148 2148 'rhodecode_clone_uri_ssh_tmpl') or self.DEFAULT_CLONE_URI_SSH
2149 2149 else:
2150 2150 uri_tmpl = rc_config.get(
2151 2151 'rhodecode_clone_uri_tmpl') or self.DEFAULT_CLONE_URI
2152 2152
2153 2153 request = get_current_request()
2154 2154 return get_clone_url(request=request,
2155 2155 uri_tmpl=uri_tmpl,
2156 2156 repo_name=self.repo_name,
2157 2157 repo_id=self.repo_id, **override)
2158 2158
2159 2159 def set_state(self, state):
2160 2160 self.repo_state = state
2161 2161 Session().add(self)
2162 2162 #==========================================================================
2163 2163 # SCM PROPERTIES
2164 2164 #==========================================================================
2165 2165
2166 2166 def get_commit(self, commit_id=None, commit_idx=None, pre_load=None):
2167 2167 return get_commit_safe(
2168 2168 self.scm_instance(), commit_id, commit_idx, pre_load=pre_load)
2169 2169
2170 2170 def get_changeset(self, rev=None, pre_load=None):
2171 2171 warnings.warn("Use get_commit", DeprecationWarning)
2172 2172 commit_id = None
2173 2173 commit_idx = None
2174 2174 if isinstance(rev, basestring):
2175 2175 commit_id = rev
2176 2176 else:
2177 2177 commit_idx = rev
2178 2178 return self.get_commit(commit_id=commit_id, commit_idx=commit_idx,
2179 2179 pre_load=pre_load)
2180 2180
2181 2181 def get_landing_commit(self):
2182 2182 """
2183 2183 Returns landing commit, or if that doesn't exist returns the tip
2184 2184 """
2185 2185 _rev_type, _rev = self.landing_rev
2186 2186 commit = self.get_commit(_rev)
2187 2187 if isinstance(commit, EmptyCommit):
2188 2188 return self.get_commit()
2189 2189 return commit
2190 2190
2191 2191 def update_commit_cache(self, cs_cache=None, config=None):
2192 2192 """
2193 2193 Update cache of last changeset for repository, keys should be::
2194 2194
2195 2195 short_id
2196 2196 raw_id
2197 2197 revision
2198 2198 parents
2199 2199 message
2200 2200 date
2201 2201 author
2202 2202
2203 2203 :param cs_cache:
2204 2204 """
2205 2205 from rhodecode.lib.vcs.backends.base import BaseChangeset
2206 2206 if cs_cache is None:
2207 2207 # use no-cache version here
2208 2208 scm_repo = self.scm_instance(cache=False, config=config)
2209 2209 if scm_repo:
2210 2210 cs_cache = scm_repo.get_commit(
2211 2211 pre_load=["author", "date", "message", "parents"])
2212 2212 else:
2213 2213 cs_cache = EmptyCommit()
2214 2214
2215 2215 if isinstance(cs_cache, BaseChangeset):
2216 2216 cs_cache = cs_cache.__json__()
2217 2217
2218 2218 def is_outdated(new_cs_cache):
2219 2219 if (new_cs_cache['raw_id'] != self.changeset_cache['raw_id'] or
2220 2220 new_cs_cache['revision'] != self.changeset_cache['revision']):
2221 2221 return True
2222 2222 return False
2223 2223
2224 2224 # check if we have maybe already latest cached revision
2225 2225 if is_outdated(cs_cache) or not self.changeset_cache:
2226 2226 _default = datetime.datetime.utcnow()
2227 2227 last_change = cs_cache.get('date') or _default
2228 2228 if self.updated_on and self.updated_on > last_change:
2229 2229 # we check if last update is newer than the new value
2230 2230 # if yes, we use the current timestamp instead. Imagine you get
2231 2231 # old commit pushed 1y ago, we'd set last update 1y to ago.
2232 2232 last_change = _default
2233 2233 log.debug('updated repo %s with new cs cache %s',
2234 2234 self.repo_name, cs_cache)
2235 2235 self.updated_on = last_change
2236 2236 self.changeset_cache = cs_cache
2237 2237 Session().add(self)
2238 2238 Session().commit()
2239 2239 else:
2240 2240 log.debug('Skipping update_commit_cache for repo:`%s` '
2241 2241 'commit already with latest changes', self.repo_name)
2242 2242
2243 2243 @property
2244 2244 def tip(self):
2245 2245 return self.get_commit('tip')
2246 2246
2247 2247 @property
2248 2248 def author(self):
2249 2249 return self.tip.author
2250 2250
2251 2251 @property
2252 2252 def last_change(self):
2253 2253 return self.scm_instance().last_change
2254 2254
2255 2255 def get_comments(self, revisions=None):
2256 2256 """
2257 2257 Returns comments for this repository grouped by revisions
2258 2258
2259 2259 :param revisions: filter query by revisions only
2260 2260 """
2261 2261 cmts = ChangesetComment.query()\
2262 2262 .filter(ChangesetComment.repo == self)
2263 2263 if revisions:
2264 2264 cmts = cmts.filter(ChangesetComment.revision.in_(revisions))
2265 2265 grouped = collections.defaultdict(list)
2266 2266 for cmt in cmts.all():
2267 2267 grouped[cmt.revision].append(cmt)
2268 2268 return grouped
2269 2269
2270 2270 def statuses(self, revisions=None):
2271 2271 """
2272 2272 Returns statuses for this repository
2273 2273
2274 2274 :param revisions: list of revisions to get statuses for
2275 2275 """
2276 2276 statuses = ChangesetStatus.query()\
2277 2277 .filter(ChangesetStatus.repo == self)\
2278 2278 .filter(ChangesetStatus.version == 0)
2279 2279
2280 2280 if revisions:
2281 2281 # Try doing the filtering in chunks to avoid hitting limits
2282 2282 size = 500
2283 2283 status_results = []
2284 2284 for chunk in xrange(0, len(revisions), size):
2285 2285 status_results += statuses.filter(
2286 2286 ChangesetStatus.revision.in_(
2287 2287 revisions[chunk: chunk+size])
2288 2288 ).all()
2289 2289 else:
2290 2290 status_results = statuses.all()
2291 2291
2292 2292 grouped = {}
2293 2293
2294 2294 # maybe we have open new pullrequest without a status?
2295 2295 stat = ChangesetStatus.STATUS_UNDER_REVIEW
2296 2296 status_lbl = ChangesetStatus.get_status_lbl(stat)
2297 2297 for pr in PullRequest.query().filter(PullRequest.source_repo == self).all():
2298 2298 for rev in pr.revisions:
2299 2299 pr_id = pr.pull_request_id
2300 2300 pr_repo = pr.target_repo.repo_name
2301 2301 grouped[rev] = [stat, status_lbl, pr_id, pr_repo]
2302 2302
2303 2303 for stat in status_results:
2304 2304 pr_id = pr_repo = None
2305 2305 if stat.pull_request:
2306 2306 pr_id = stat.pull_request.pull_request_id
2307 2307 pr_repo = stat.pull_request.target_repo.repo_name
2308 2308 grouped[stat.revision] = [str(stat.status), stat.status_lbl,
2309 2309 pr_id, pr_repo]
2310 2310 return grouped
2311 2311
2312 2312 # ==========================================================================
2313 2313 # SCM CACHE INSTANCE
2314 2314 # ==========================================================================
2315 2315
2316 2316 def scm_instance(self, **kwargs):
2317 2317 import rhodecode
2318 2318
2319 2319 # Passing a config will not hit the cache currently only used
2320 2320 # for repo2dbmapper
2321 2321 config = kwargs.pop('config', None)
2322 2322 cache = kwargs.pop('cache', None)
2323 2323 full_cache = str2bool(rhodecode.CONFIG.get('vcs_full_cache'))
2324 2324 # if cache is NOT defined use default global, else we have a full
2325 2325 # control over cache behaviour
2326 2326 if cache is None and full_cache and not config:
2327 2327 return self._get_instance_cached()
2328 2328 return self._get_instance(cache=bool(cache), config=config)
2329 2329
2330 2330 def _get_instance_cached(self):
2331 2331 from rhodecode.lib import rc_cache
2332 2332
2333 2333 cache_namespace_uid = 'cache_repo_instance.{}'.format(self.repo_id)
2334 2334 invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
2335 2335 repo_id=self.repo_id)
2336 2336 region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid)
2337 2337
2338 2338 @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
2339 2339 def get_instance_cached(repo_id):
2340 2340 return self._get_instance()
2341 2341
2342 2342 inv_context_manager = rc_cache.InvalidationContext(
2343 2343 uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
2344 2344 with inv_context_manager as invalidation_context:
2345 # check for stored invalidation signal, and maybe purge the cache
2346 # before computing it again
2345 args = (self.repo_id,)
2346 # re-compute and store cache if we get invalidate signal
2347 2347 if invalidation_context.should_invalidate():
2348 get_instance_cached.invalidate(self.repo_id)
2349
2350 instance = get_instance_cached(self.repo_id)
2348 instance = get_instance_cached.refresh(*args)
2349 else:
2350 instance = get_instance_cached(*args)
2351
2351 2352 log.debug(
2352 2353 'Repo instance fetched in %.3fs', inv_context_manager.compute_time)
2353 2354 return instance
2354 2355
2355 2356 def _get_instance(self, cache=True, config=None):
2356 2357 config = config or self._config
2357 2358 custom_wire = {
2358 2359 'cache': cache # controls the vcs.remote cache
2359 2360 }
2360 2361 repo = get_vcs_instance(
2361 2362 repo_path=safe_str(self.repo_full_path),
2362 2363 config=config,
2363 2364 with_wire=custom_wire,
2364 2365 create=False,
2365 2366 _vcs_alias=self.repo_type)
2366 2367
2367 2368 return repo
2368 2369
2369 2370 def __json__(self):
2370 2371 return {'landing_rev': self.landing_rev}
2371 2372
2372 2373 def get_dict(self):
2373 2374
2374 2375 # Since we transformed `repo_name` to a hybrid property, we need to
2375 2376 # keep compatibility with the code which uses `repo_name` field.
2376 2377
2377 2378 result = super(Repository, self).get_dict()
2378 2379 result['repo_name'] = result.pop('_repo_name', None)
2379 2380 return result
2380 2381
2381 2382
2382 2383 class RepoGroup(Base, BaseModel):
2383 2384 __tablename__ = 'groups'
2384 2385 __table_args__ = (
2385 2386 UniqueConstraint('group_name', 'group_parent_id'),
2386 2387 CheckConstraint('group_id != group_parent_id'),
2387 2388 base_table_args,
2388 2389 )
2389 2390 __mapper_args__ = {'order_by': 'group_name'}
2390 2391
2391 2392 CHOICES_SEPARATOR = '/' # used to generate select2 choices for nested groups
2392 2393
2393 2394 group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
2394 2395 group_name = Column("group_name", String(255), nullable=False, unique=True, default=None)
2395 2396 group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)
2396 2397 group_description = Column("group_description", String(10000), nullable=True, unique=None, default=None)
2397 2398 enable_locking = Column("enable_locking", Boolean(), nullable=False, unique=None, default=False)
2398 2399 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)
2399 2400 created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
2400 2401 updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
2401 2402 personal = Column('personal', Boolean(), nullable=True, unique=None, default=None)
2402 2403
2403 2404 repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')
2404 2405 users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
2405 2406 parent_group = relationship('RepoGroup', remote_side=group_id)
2406 2407 user = relationship('User')
2407 2408 integrations = relationship('Integration',
2408 2409 cascade="all, delete, delete-orphan")
2409 2410
2410 2411 def __init__(self, group_name='', parent_group=None):
2411 2412 self.group_name = group_name
2412 2413 self.parent_group = parent_group
2413 2414
2414 2415 def __unicode__(self):
2415 2416 return u"<%s('id:%s:%s')>" % (
2416 2417 self.__class__.__name__, self.group_id, self.group_name)
2417 2418
2418 2419 @hybrid_property
2419 2420 def description_safe(self):
2420 2421 from rhodecode.lib import helpers as h
2421 2422 return h.escape(self.group_description)
2422 2423
2423 2424 @classmethod
2424 2425 def _generate_choice(cls, repo_group):
2425 2426 from webhelpers.html import literal as _literal
2426 2427 _name = lambda k: _literal(cls.CHOICES_SEPARATOR.join(k))
2427 2428 return repo_group.group_id, _name(repo_group.full_path_splitted)
2428 2429
2429 2430 @classmethod
2430 2431 def groups_choices(cls, groups=None, show_empty_group=True):
2431 2432 if not groups:
2432 2433 groups = cls.query().all()
2433 2434
2434 2435 repo_groups = []
2435 2436 if show_empty_group:
2436 2437 repo_groups = [(-1, u'-- %s --' % _('No parent'))]
2437 2438
2438 2439 repo_groups.extend([cls._generate_choice(x) for x in groups])
2439 2440
2440 2441 repo_groups = sorted(
2441 2442 repo_groups, key=lambda t: t[1].split(cls.CHOICES_SEPARATOR)[0])
2442 2443 return repo_groups
2443 2444
2444 2445 @classmethod
2445 2446 def url_sep(cls):
2446 2447 return URL_SEP
2447 2448
2448 2449 @classmethod
2449 2450 def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
2450 2451 if case_insensitive:
2451 2452 gr = cls.query().filter(func.lower(cls.group_name)
2452 2453 == func.lower(group_name))
2453 2454 else:
2454 2455 gr = cls.query().filter(cls.group_name == group_name)
2455 2456 if cache:
2456 2457 name_key = _hash_key(group_name)
2457 2458 gr = gr.options(
2458 2459 FromCache("sql_cache_short", "get_group_%s" % name_key))
2459 2460 return gr.scalar()
2460 2461
2461 2462 @classmethod
2462 2463 def get_user_personal_repo_group(cls, user_id):
2463 2464 user = User.get(user_id)
2464 2465 if user.username == User.DEFAULT_USER:
2465 2466 return None
2466 2467
2467 2468 return cls.query()\
2468 2469 .filter(cls.personal == true()) \
2469 2470 .filter(cls.user == user).scalar()
2470 2471
2471 2472 @classmethod
2472 2473 def get_all_repo_groups(cls, user_id=Optional(None), group_id=Optional(None),
2473 2474 case_insensitive=True):
2474 2475 q = RepoGroup.query()
2475 2476
2476 2477 if not isinstance(user_id, Optional):
2477 2478 q = q.filter(RepoGroup.user_id == user_id)
2478 2479
2479 2480 if not isinstance(group_id, Optional):
2480 2481 q = q.filter(RepoGroup.group_parent_id == group_id)
2481 2482
2482 2483 if case_insensitive:
2483 2484 q = q.order_by(func.lower(RepoGroup.group_name))
2484 2485 else:
2485 2486 q = q.order_by(RepoGroup.group_name)
2486 2487 return q.all()
2487 2488
2488 2489 @property
2489 2490 def parents(self):
2490 2491 parents_recursion_limit = 10
2491 2492 groups = []
2492 2493 if self.parent_group is None:
2493 2494 return groups
2494 2495 cur_gr = self.parent_group
2495 2496 groups.insert(0, cur_gr)
2496 2497 cnt = 0
2497 2498 while 1:
2498 2499 cnt += 1
2499 2500 gr = getattr(cur_gr, 'parent_group', None)
2500 2501 cur_gr = cur_gr.parent_group
2501 2502 if gr is None:
2502 2503 break
2503 2504 if cnt == parents_recursion_limit:
2504 2505 # this will prevent accidental infinit loops
2505 2506 log.error(('more than %s parents found for group %s, stopping '
2506 2507 'recursive parent fetching' % (parents_recursion_limit, self)))
2507 2508 break
2508 2509
2509 2510 groups.insert(0, gr)
2510 2511 return groups
2511 2512
2512 2513 @property
2513 2514 def last_db_change(self):
2514 2515 return self.updated_on
2515 2516
2516 2517 @property
2517 2518 def children(self):
2518 2519 return RepoGroup.query().filter(RepoGroup.parent_group == self)
2519 2520
2520 2521 @property
2521 2522 def name(self):
2522 2523 return self.group_name.split(RepoGroup.url_sep())[-1]
2523 2524
2524 2525 @property
2525 2526 def full_path(self):
2526 2527 return self.group_name
2527 2528
2528 2529 @property
2529 2530 def full_path_splitted(self):
2530 2531 return self.group_name.split(RepoGroup.url_sep())
2531 2532
2532 2533 @property
2533 2534 def repositories(self):
2534 2535 return Repository.query()\
2535 2536 .filter(Repository.group == self)\
2536 2537 .order_by(Repository.repo_name)
2537 2538
2538 2539 @property
2539 2540 def repositories_recursive_count(self):
2540 2541 cnt = self.repositories.count()
2541 2542
2542 2543 def children_count(group):
2543 2544 cnt = 0
2544 2545 for child in group.children:
2545 2546 cnt += child.repositories.count()
2546 2547 cnt += children_count(child)
2547 2548 return cnt
2548 2549
2549 2550 return cnt + children_count(self)
2550 2551
2551 2552 def _recursive_objects(self, include_repos=True):
2552 2553 all_ = []
2553 2554
2554 2555 def _get_members(root_gr):
2555 2556 if include_repos:
2556 2557 for r in root_gr.repositories:
2557 2558 all_.append(r)
2558 2559 childs = root_gr.children.all()
2559 2560 if childs:
2560 2561 for gr in childs:
2561 2562 all_.append(gr)
2562 2563 _get_members(gr)
2563 2564
2564 2565 _get_members(self)
2565 2566 return [self] + all_
2566 2567
2567 2568 def recursive_groups_and_repos(self):
2568 2569 """
2569 2570 Recursive return all groups, with repositories in those groups
2570 2571 """
2571 2572 return self._recursive_objects()
2572 2573
2573 2574 def recursive_groups(self):
2574 2575 """
2575 2576 Returns all children groups for this group including children of children
2576 2577 """
2577 2578 return self._recursive_objects(include_repos=False)
2578 2579
2579 2580 def get_new_name(self, group_name):
2580 2581 """
2581 2582 returns new full group name based on parent and new name
2582 2583
2583 2584 :param group_name:
2584 2585 """
2585 2586 path_prefix = (self.parent_group.full_path_splitted if
2586 2587 self.parent_group else [])
2587 2588 return RepoGroup.url_sep().join(path_prefix + [group_name])
2588 2589
2589 2590 def permissions(self, with_admins=True, with_owner=True):
2590 2591 q = UserRepoGroupToPerm.query().filter(UserRepoGroupToPerm.group == self)
2591 2592 q = q.options(joinedload(UserRepoGroupToPerm.group),
2592 2593 joinedload(UserRepoGroupToPerm.user),
2593 2594 joinedload(UserRepoGroupToPerm.permission),)
2594 2595
2595 2596 # get owners and admins and permissions. We do a trick of re-writing
2596 2597 # objects from sqlalchemy to named-tuples due to sqlalchemy session
2597 2598 # has a global reference and changing one object propagates to all
2598 2599 # others. This means if admin is also an owner admin_row that change
2599 2600 # would propagate to both objects
2600 2601 perm_rows = []
2601 2602 for _usr in q.all():
2602 2603 usr = AttributeDict(_usr.user.get_dict())
2603 2604 usr.permission = _usr.permission.permission_name
2604 2605 perm_rows.append(usr)
2605 2606
2606 2607 # filter the perm rows by 'default' first and then sort them by
2607 2608 # admin,write,read,none permissions sorted again alphabetically in
2608 2609 # each group
2609 2610 perm_rows = sorted(perm_rows, key=display_user_sort)
2610 2611
2611 2612 _admin_perm = 'group.admin'
2612 2613 owner_row = []
2613 2614 if with_owner:
2614 2615 usr = AttributeDict(self.user.get_dict())
2615 2616 usr.owner_row = True
2616 2617 usr.permission = _admin_perm
2617 2618 owner_row.append(usr)
2618 2619
2619 2620 super_admin_rows = []
2620 2621 if with_admins:
2621 2622 for usr in User.get_all_super_admins():
2622 2623 # if this admin is also owner, don't double the record
2623 2624 if usr.user_id == owner_row[0].user_id:
2624 2625 owner_row[0].admin_row = True
2625 2626 else:
2626 2627 usr = AttributeDict(usr.get_dict())
2627 2628 usr.admin_row = True
2628 2629 usr.permission = _admin_perm
2629 2630 super_admin_rows.append(usr)
2630 2631
2631 2632 return super_admin_rows + owner_row + perm_rows
2632 2633
2633 2634 def permission_user_groups(self):
2634 2635 q = UserGroupRepoGroupToPerm.query().filter(UserGroupRepoGroupToPerm.group == self)
2635 2636 q = q.options(joinedload(UserGroupRepoGroupToPerm.group),
2636 2637 joinedload(UserGroupRepoGroupToPerm.users_group),
2637 2638 joinedload(UserGroupRepoGroupToPerm.permission),)
2638 2639
2639 2640 perm_rows = []
2640 2641 for _user_group in q.all():
2641 2642 usr = AttributeDict(_user_group.users_group.get_dict())
2642 2643 usr.permission = _user_group.permission.permission_name
2643 2644 perm_rows.append(usr)
2644 2645
2645 2646 perm_rows = sorted(perm_rows, key=display_user_group_sort)
2646 2647 return perm_rows
2647 2648
2648 2649 def get_api_data(self):
2649 2650 """
2650 2651 Common function for generating api data
2651 2652
2652 2653 """
2653 2654 group = self
2654 2655 data = {
2655 2656 'group_id': group.group_id,
2656 2657 'group_name': group.group_name,
2657 2658 'group_description': group.description_safe,
2658 2659 'parent_group': group.parent_group.group_name if group.parent_group else None,
2659 2660 'repositories': [x.repo_name for x in group.repositories],
2660 2661 'owner': group.user.username,
2661 2662 }
2662 2663 return data
2663 2664
2664 2665
2665 2666 class Permission(Base, BaseModel):
2666 2667 __tablename__ = 'permissions'
2667 2668 __table_args__ = (
2668 2669 Index('p_perm_name_idx', 'permission_name'),
2669 2670 base_table_args,
2670 2671 )
2671 2672
2672 2673 PERMS = [
2673 2674 ('hg.admin', _('RhodeCode Super Administrator')),
2674 2675
2675 2676 ('repository.none', _('Repository no access')),
2676 2677 ('repository.read', _('Repository read access')),
2677 2678 ('repository.write', _('Repository write access')),
2678 2679 ('repository.admin', _('Repository admin access')),
2679 2680
2680 2681 ('group.none', _('Repository group no access')),
2681 2682 ('group.read', _('Repository group read access')),
2682 2683 ('group.write', _('Repository group write access')),
2683 2684 ('group.admin', _('Repository group admin access')),
2684 2685
2685 2686 ('usergroup.none', _('User group no access')),
2686 2687 ('usergroup.read', _('User group read access')),
2687 2688 ('usergroup.write', _('User group write access')),
2688 2689 ('usergroup.admin', _('User group admin access')),
2689 2690
2690 2691 ('hg.repogroup.create.false', _('Repository Group creation disabled')),
2691 2692 ('hg.repogroup.create.true', _('Repository Group creation enabled')),
2692 2693
2693 2694 ('hg.usergroup.create.false', _('User Group creation disabled')),
2694 2695 ('hg.usergroup.create.true', _('User Group creation enabled')),
2695 2696
2696 2697 ('hg.create.none', _('Repository creation disabled')),
2697 2698 ('hg.create.repository', _('Repository creation enabled')),
2698 2699 ('hg.create.write_on_repogroup.true', _('Repository creation enabled with write permission to a repository group')),
2699 2700 ('hg.create.write_on_repogroup.false', _('Repository creation disabled with write permission to a repository group')),
2700 2701
2701 2702 ('hg.fork.none', _('Repository forking disabled')),
2702 2703 ('hg.fork.repository', _('Repository forking enabled')),
2703 2704
2704 2705 ('hg.register.none', _('Registration disabled')),
2705 2706 ('hg.register.manual_activate', _('User Registration with manual account activation')),
2706 2707 ('hg.register.auto_activate', _('User Registration with automatic account activation')),
2707 2708
2708 2709 ('hg.password_reset.enabled', _('Password reset enabled')),
2709 2710 ('hg.password_reset.hidden', _('Password reset hidden')),
2710 2711 ('hg.password_reset.disabled', _('Password reset disabled')),
2711 2712
2712 2713 ('hg.extern_activate.manual', _('Manual activation of external account')),
2713 2714 ('hg.extern_activate.auto', _('Automatic activation of external account')),
2714 2715
2715 2716 ('hg.inherit_default_perms.false', _('Inherit object permissions from default user disabled')),
2716 2717 ('hg.inherit_default_perms.true', _('Inherit object permissions from default user enabled')),
2717 2718 ]
2718 2719
2719 2720 # definition of system default permissions for DEFAULT user
2720 2721 DEFAULT_USER_PERMISSIONS = [
2721 2722 'repository.read',
2722 2723 'group.read',
2723 2724 'usergroup.read',
2724 2725 'hg.create.repository',
2725 2726 'hg.repogroup.create.false',
2726 2727 'hg.usergroup.create.false',
2727 2728 'hg.create.write_on_repogroup.true',
2728 2729 'hg.fork.repository',
2729 2730 'hg.register.manual_activate',
2730 2731 'hg.password_reset.enabled',
2731 2732 'hg.extern_activate.auto',
2732 2733 'hg.inherit_default_perms.true',
2733 2734 ]
2734 2735
2735 2736 # defines which permissions are more important higher the more important
2736 2737 # Weight defines which permissions are more important.
2737 2738 # The higher number the more important.
2738 2739 PERM_WEIGHTS = {
2739 2740 'repository.none': 0,
2740 2741 'repository.read': 1,
2741 2742 'repository.write': 3,
2742 2743 'repository.admin': 4,
2743 2744
2744 2745 'group.none': 0,
2745 2746 'group.read': 1,
2746 2747 'group.write': 3,
2747 2748 'group.admin': 4,
2748 2749
2749 2750 'usergroup.none': 0,
2750 2751 'usergroup.read': 1,
2751 2752 'usergroup.write': 3,
2752 2753 'usergroup.admin': 4,
2753 2754
2754 2755 'hg.repogroup.create.false': 0,
2755 2756 'hg.repogroup.create.true': 1,
2756 2757
2757 2758 'hg.usergroup.create.false': 0,
2758 2759 'hg.usergroup.create.true': 1,
2759 2760
2760 2761 'hg.fork.none': 0,
2761 2762 'hg.fork.repository': 1,
2762 2763 'hg.create.none': 0,
2763 2764 'hg.create.repository': 1
2764 2765 }
2765 2766
2766 2767 permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
2767 2768 permission_name = Column("permission_name", String(255), nullable=True, unique=None, default=None)
2768 2769 permission_longname = Column("permission_longname", String(255), nullable=True, unique=None, default=None)
2769 2770
2770 2771 def __unicode__(self):
2771 2772 return u"<%s('%s:%s')>" % (
2772 2773 self.__class__.__name__, self.permission_id, self.permission_name
2773 2774 )
2774 2775
2775 2776 @classmethod
2776 2777 def get_by_key(cls, key):
2777 2778 return cls.query().filter(cls.permission_name == key).scalar()
2778 2779
2779 2780 @classmethod
2780 2781 def get_default_repo_perms(cls, user_id, repo_id=None):
2781 2782 q = Session().query(UserRepoToPerm, Repository, Permission)\
2782 2783 .join((Permission, UserRepoToPerm.permission_id == Permission.permission_id))\
2783 2784 .join((Repository, UserRepoToPerm.repository_id == Repository.repo_id))\
2784 2785 .filter(UserRepoToPerm.user_id == user_id)
2785 2786 if repo_id:
2786 2787 q = q.filter(UserRepoToPerm.repository_id == repo_id)
2787 2788 return q.all()
2788 2789
2789 2790 @classmethod
2790 2791 def get_default_repo_perms_from_user_group(cls, user_id, repo_id=None):
2791 2792 q = Session().query(UserGroupRepoToPerm, Repository, Permission)\
2792 2793 .join(
2793 2794 Permission,
2794 2795 UserGroupRepoToPerm.permission_id == Permission.permission_id)\
2795 2796 .join(
2796 2797 Repository,
2797 2798 UserGroupRepoToPerm.repository_id == Repository.repo_id)\
2798 2799 .join(
2799 2800 UserGroup,
2800 2801 UserGroupRepoToPerm.users_group_id ==
2801 2802 UserGroup.users_group_id)\
2802 2803 .join(
2803 2804 UserGroupMember,
2804 2805 UserGroupRepoToPerm.users_group_id ==
2805 2806 UserGroupMember.users_group_id)\
2806 2807 .filter(
2807 2808 UserGroupMember.user_id == user_id,
2808 2809 UserGroup.users_group_active == true())
2809 2810 if repo_id:
2810 2811 q = q.filter(UserGroupRepoToPerm.repository_id == repo_id)
2811 2812 return q.all()
2812 2813
2813 2814 @classmethod
2814 2815 def get_default_group_perms(cls, user_id, repo_group_id=None):
2815 2816 q = Session().query(UserRepoGroupToPerm, RepoGroup, Permission)\
2816 2817 .join((Permission, UserRepoGroupToPerm.permission_id == Permission.permission_id))\
2817 2818 .join((RepoGroup, UserRepoGroupToPerm.group_id == RepoGroup.group_id))\
2818 2819 .filter(UserRepoGroupToPerm.user_id == user_id)
2819 2820 if repo_group_id:
2820 2821 q = q.filter(UserRepoGroupToPerm.group_id == repo_group_id)
2821 2822 return q.all()
2822 2823
2823 2824 @classmethod
2824 2825 def get_default_group_perms_from_user_group(
2825 2826 cls, user_id, repo_group_id=None):
2826 2827 q = Session().query(UserGroupRepoGroupToPerm, RepoGroup, Permission)\
2827 2828 .join(
2828 2829 Permission,
2829 2830 UserGroupRepoGroupToPerm.permission_id ==
2830 2831 Permission.permission_id)\
2831 2832 .join(
2832 2833 RepoGroup,
2833 2834 UserGroupRepoGroupToPerm.group_id == RepoGroup.group_id)\
2834 2835 .join(
2835 2836 UserGroup,
2836 2837 UserGroupRepoGroupToPerm.users_group_id ==
2837 2838 UserGroup.users_group_id)\
2838 2839 .join(
2839 2840 UserGroupMember,
2840 2841 UserGroupRepoGroupToPerm.users_group_id ==
2841 2842 UserGroupMember.users_group_id)\
2842 2843 .filter(
2843 2844 UserGroupMember.user_id == user_id,
2844 2845 UserGroup.users_group_active == true())
2845 2846 if repo_group_id:
2846 2847 q = q.filter(UserGroupRepoGroupToPerm.group_id == repo_group_id)
2847 2848 return q.all()
2848 2849
2849 2850 @classmethod
2850 2851 def get_default_user_group_perms(cls, user_id, user_group_id=None):
2851 2852 q = Session().query(UserUserGroupToPerm, UserGroup, Permission)\
2852 2853 .join((Permission, UserUserGroupToPerm.permission_id == Permission.permission_id))\
2853 2854 .join((UserGroup, UserUserGroupToPerm.user_group_id == UserGroup.users_group_id))\
2854 2855 .filter(UserUserGroupToPerm.user_id == user_id)
2855 2856 if user_group_id:
2856 2857 q = q.filter(UserUserGroupToPerm.user_group_id == user_group_id)
2857 2858 return q.all()
2858 2859
2859 2860 @classmethod
2860 2861 def get_default_user_group_perms_from_user_group(
2861 2862 cls, user_id, user_group_id=None):
2862 2863 TargetUserGroup = aliased(UserGroup, name='target_user_group')
2863 2864 q = Session().query(UserGroupUserGroupToPerm, UserGroup, Permission)\
2864 2865 .join(
2865 2866 Permission,
2866 2867 UserGroupUserGroupToPerm.permission_id ==
2867 2868 Permission.permission_id)\
2868 2869 .join(
2869 2870 TargetUserGroup,
2870 2871 UserGroupUserGroupToPerm.target_user_group_id ==
2871 2872 TargetUserGroup.users_group_id)\
2872 2873 .join(
2873 2874 UserGroup,
2874 2875 UserGroupUserGroupToPerm.user_group_id ==
2875 2876 UserGroup.users_group_id)\
2876 2877 .join(
2877 2878 UserGroupMember,
2878 2879 UserGroupUserGroupToPerm.user_group_id ==
2879 2880 UserGroupMember.users_group_id)\
2880 2881 .filter(
2881 2882 UserGroupMember.user_id == user_id,
2882 2883 UserGroup.users_group_active == true())
2883 2884 if user_group_id:
2884 2885 q = q.filter(
2885 2886 UserGroupUserGroupToPerm.user_group_id == user_group_id)
2886 2887
2887 2888 return q.all()
2888 2889
2889 2890
2890 2891 class UserRepoToPerm(Base, BaseModel):
2891 2892 __tablename__ = 'repo_to_perm'
2892 2893 __table_args__ = (
2893 2894 UniqueConstraint('user_id', 'repository_id', 'permission_id'),
2894 2895 base_table_args
2895 2896 )
2896 2897
2897 2898 repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
2898 2899 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
2899 2900 permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
2900 2901 repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
2901 2902
2902 2903 user = relationship('User')
2903 2904 repository = relationship('Repository')
2904 2905 permission = relationship('Permission')
2905 2906
2906 2907 @classmethod
2907 2908 def create(cls, user, repository, permission):
2908 2909 n = cls()
2909 2910 n.user = user
2910 2911 n.repository = repository
2911 2912 n.permission = permission
2912 2913 Session().add(n)
2913 2914 return n
2914 2915
2915 2916 def __unicode__(self):
2916 2917 return u'<%s => %s >' % (self.user, self.repository)
2917 2918
2918 2919
2919 2920 class UserUserGroupToPerm(Base, BaseModel):
2920 2921 __tablename__ = 'user_user_group_to_perm'
2921 2922 __table_args__ = (
2922 2923 UniqueConstraint('user_id', 'user_group_id', 'permission_id'),
2923 2924 base_table_args
2924 2925 )
2925 2926
2926 2927 user_user_group_to_perm_id = Column("user_user_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
2927 2928 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
2928 2929 permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
2929 2930 user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
2930 2931
2931 2932 user = relationship('User')
2932 2933 user_group = relationship('UserGroup')
2933 2934 permission = relationship('Permission')
2934 2935
2935 2936 @classmethod
2936 2937 def create(cls, user, user_group, permission):
2937 2938 n = cls()
2938 2939 n.user = user
2939 2940 n.user_group = user_group
2940 2941 n.permission = permission
2941 2942 Session().add(n)
2942 2943 return n
2943 2944
2944 2945 def __unicode__(self):
2945 2946 return u'<%s => %s >' % (self.user, self.user_group)
2946 2947
2947 2948
2948 2949 class UserToPerm(Base, BaseModel):
2949 2950 __tablename__ = 'user_to_perm'
2950 2951 __table_args__ = (
2951 2952 UniqueConstraint('user_id', 'permission_id'),
2952 2953 base_table_args
2953 2954 )
2954 2955
2955 2956 user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
2956 2957 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
2957 2958 permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
2958 2959
2959 2960 user = relationship('User')
2960 2961 permission = relationship('Permission', lazy='joined')
2961 2962
2962 2963 def __unicode__(self):
2963 2964 return u'<%s => %s >' % (self.user, self.permission)
2964 2965
2965 2966
2966 2967 class UserGroupRepoToPerm(Base, BaseModel):
2967 2968 __tablename__ = 'users_group_repo_to_perm'
2968 2969 __table_args__ = (
2969 2970 UniqueConstraint('repository_id', 'users_group_id', 'permission_id'),
2970 2971 base_table_args
2971 2972 )
2972 2973
2973 2974 users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
2974 2975 users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
2975 2976 permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
2976 2977 repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
2977 2978
2978 2979 users_group = relationship('UserGroup')
2979 2980 permission = relationship('Permission')
2980 2981 repository = relationship('Repository')
2981 2982
2982 2983 @classmethod
2983 2984 def create(cls, users_group, repository, permission):
2984 2985 n = cls()
2985 2986 n.users_group = users_group
2986 2987 n.repository = repository
2987 2988 n.permission = permission
2988 2989 Session().add(n)
2989 2990 return n
2990 2991
2991 2992 def __unicode__(self):
2992 2993 return u'<UserGroupRepoToPerm:%s => %s >' % (self.users_group, self.repository)
2993 2994
2994 2995
2995 2996 class UserGroupUserGroupToPerm(Base, BaseModel):
2996 2997 __tablename__ = 'user_group_user_group_to_perm'
2997 2998 __table_args__ = (
2998 2999 UniqueConstraint('target_user_group_id', 'user_group_id', 'permission_id'),
2999 3000 CheckConstraint('target_user_group_id != user_group_id'),
3000 3001 base_table_args
3001 3002 )
3002 3003
3003 3004 user_group_user_group_to_perm_id = Column("user_group_user_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
3004 3005 target_user_group_id = Column("target_user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
3005 3006 permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
3006 3007 user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
3007 3008
3008 3009 target_user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id')
3009 3010 user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.user_group_id==UserGroup.users_group_id')
3010 3011 permission = relationship('Permission')
3011 3012
3012 3013 @classmethod
3013 3014 def create(cls, target_user_group, user_group, permission):
3014 3015 n = cls()
3015 3016 n.target_user_group = target_user_group
3016 3017 n.user_group = user_group
3017 3018 n.permission = permission
3018 3019 Session().add(n)
3019 3020 return n
3020 3021
3021 3022 def __unicode__(self):
3022 3023 return u'<UserGroupUserGroup:%s => %s >' % (self.target_user_group, self.user_group)
3023 3024
3024 3025
3025 3026 class UserGroupToPerm(Base, BaseModel):
3026 3027 __tablename__ = 'users_group_to_perm'
3027 3028 __table_args__ = (
3028 3029 UniqueConstraint('users_group_id', 'permission_id',),
3029 3030 base_table_args
3030 3031 )
3031 3032
3032 3033 users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
3033 3034 users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
3034 3035 permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
3035 3036
3036 3037 users_group = relationship('UserGroup')
3037 3038 permission = relationship('Permission')
3038 3039
3039 3040
3040 3041 class UserRepoGroupToPerm(Base, BaseModel):
3041 3042 __tablename__ = 'user_repo_group_to_perm'
3042 3043 __table_args__ = (
3043 3044 UniqueConstraint('user_id', 'group_id', 'permission_id'),
3044 3045 base_table_args
3045 3046 )
3046 3047
3047 3048 group_to_perm_id = Column("group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
3048 3049 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
3049 3050 group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None)
3050 3051 permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
3051 3052
3052 3053 user = relationship('User')
3053 3054 group = relationship('RepoGroup')
3054 3055 permission = relationship('Permission')
3055 3056
3056 3057 @classmethod
3057 3058 def create(cls, user, repository_group, permission):
3058 3059 n = cls()
3059 3060 n.user = user
3060 3061 n.group = repository_group
3061 3062 n.permission = permission
3062 3063 Session().add(n)
3063 3064 return n
3064 3065
3065 3066
3066 3067 class UserGroupRepoGroupToPerm(Base, BaseModel):
3067 3068 __tablename__ = 'users_group_repo_group_to_perm'
3068 3069 __table_args__ = (
3069 3070 UniqueConstraint('users_group_id', 'group_id'),
3070 3071 base_table_args
3071 3072 )
3072 3073
3073 3074 users_group_repo_group_to_perm_id = Column("users_group_repo_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
3074 3075 users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
3075 3076 group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None)
3076 3077 permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
3077 3078
3078 3079 users_group = relationship('UserGroup')
3079 3080 permission = relationship('Permission')
3080 3081 group = relationship('RepoGroup')
3081 3082
3082 3083 @classmethod
3083 3084 def create(cls, user_group, repository_group, permission):
3084 3085 n = cls()
3085 3086 n.users_group = user_group
3086 3087 n.group = repository_group
3087 3088 n.permission = permission
3088 3089 Session().add(n)
3089 3090 return n
3090 3091
3091 3092 def __unicode__(self):
3092 3093 return u'<UserGroupRepoGroupToPerm:%s => %s >' % (self.users_group, self.group)
3093 3094
3094 3095
3095 3096 class Statistics(Base, BaseModel):
3096 3097 __tablename__ = 'statistics'
3097 3098 __table_args__ = (
3098 3099 base_table_args
3099 3100 )
3100 3101
3101 3102 stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
3102 3103 repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True, default=None)
3103 3104 stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)
3104 3105 commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False)#JSON data
3105 3106 commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)#JSON data
3106 3107 languages = Column("languages", LargeBinary(1000000), nullable=False)#JSON data
3107 3108
3108 3109 repository = relationship('Repository', single_parent=True)
3109 3110
3110 3111
3111 3112 class UserFollowing(Base, BaseModel):
3112 3113 __tablename__ = 'user_followings'
3113 3114 __table_args__ = (
3114 3115 UniqueConstraint('user_id', 'follows_repository_id'),
3115 3116 UniqueConstraint('user_id', 'follows_user_id'),
3116 3117 base_table_args
3117 3118 )
3118 3119
3119 3120 user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
3120 3121 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
3121 3122 follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None)
3122 3123 follows_user_id = Column("follows_user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
3123 3124 follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
3124 3125
3125 3126 user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')
3126 3127
3127 3128 follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
3128 3129 follows_repository = relationship('Repository', order_by='Repository.repo_name')
3129 3130
3130 3131 @classmethod
3131 3132 def get_repo_followers(cls, repo_id):
3132 3133 return cls.query().filter(cls.follows_repo_id == repo_id)
3133 3134
3134 3135
3135 3136 class CacheKey(Base, BaseModel):
3136 3137 __tablename__ = 'cache_invalidation'
3137 3138 __table_args__ = (
3138 3139 UniqueConstraint('cache_key'),
3139 3140 Index('key_idx', 'cache_key'),
3140 3141 base_table_args,
3141 3142 )
3142 3143
3143 3144 CACHE_TYPE_FEED = 'FEED'
3144 3145 CACHE_TYPE_README = 'README'
3145 3146 # namespaces used to register process/thread aware caches
3146 3147 REPO_INVALIDATION_NAMESPACE = 'repo_cache:{repo_id}'
3147 3148 SETTINGS_INVALIDATION_NAMESPACE = 'system_settings'
3148 3149
3149 3150 cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
3150 3151 cache_key = Column("cache_key", String(255), nullable=True, unique=None, default=None)
3151 3152 cache_args = Column("cache_args", String(255), nullable=True, unique=None, default=None)
3152 3153 cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
3153 3154
3154 3155 def __init__(self, cache_key, cache_args=''):
3155 3156 self.cache_key = cache_key
3156 3157 self.cache_args = cache_args
3157 3158 self.cache_active = False
3158 3159
3159 3160 def __unicode__(self):
3160 3161 return u"<%s('%s:%s[%s]')>" % (
3161 3162 self.__class__.__name__,
3162 3163 self.cache_id, self.cache_key, self.cache_active)
3163 3164
3164 3165 def _cache_key_partition(self):
3165 3166 prefix, repo_name, suffix = self.cache_key.partition(self.cache_args)
3166 3167 return prefix, repo_name, suffix
3167 3168
3168 3169 def get_prefix(self):
3169 3170 """
3170 3171 Try to extract prefix from existing cache key. The key could consist
3171 3172 of prefix, repo_name, suffix
3172 3173 """
3173 3174 # this returns prefix, repo_name, suffix
3174 3175 return self._cache_key_partition()[0]
3175 3176
3176 3177 def get_suffix(self):
3177 3178 """
3178 3179 get suffix that might have been used in _get_cache_key to
3179 3180 generate self.cache_key. Only used for informational purposes
3180 3181 in repo_edit.mako.
3181 3182 """
3182 3183 # prefix, repo_name, suffix
3183 3184 return self._cache_key_partition()[2]
3184 3185
3185 3186 @classmethod
3186 3187 def delete_all_cache(cls):
3187 3188 """
3188 3189 Delete all cache keys from database.
3189 3190 Should only be run when all instances are down and all entries
3190 3191 thus stale.
3191 3192 """
3192 3193 cls.query().delete()
3193 3194 Session().commit()
3194 3195
3195 3196 @classmethod
3196 3197 def set_invalidate(cls, cache_uid, delete=False):
3197 3198 """
3198 3199 Mark all caches of a repo as invalid in the database.
3199 3200 """
3200 3201
3201 3202 try:
3202 3203 qry = Session().query(cls).filter(cls.cache_args == cache_uid)
3203 3204 if delete:
3204 3205 qry.delete()
3205 3206 log.debug('cache objects deleted for cache args %s',
3206 3207 safe_str(cache_uid))
3207 3208 else:
3208 3209 qry.update({"cache_active": False})
3209 3210 log.debug('cache objects marked as invalid for cache args %s',
3210 3211 safe_str(cache_uid))
3211 3212
3212 3213 Session().commit()
3213 3214 except Exception:
3214 3215 log.exception(
3215 3216 'Cache key invalidation failed for cache args %s',
3216 3217 safe_str(cache_uid))
3217 3218 Session().rollback()
3218 3219
3219 3220 @classmethod
3220 3221 def get_active_cache(cls, cache_key):
3221 3222 inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()
3222 3223 if inv_obj:
3223 3224 return inv_obj
3224 3225 return None
3225 3226
3226 3227
3227 3228 class ChangesetComment(Base, BaseModel):
3228 3229 __tablename__ = 'changeset_comments'
3229 3230 __table_args__ = (
3230 3231 Index('cc_revision_idx', 'revision'),
3231 3232 base_table_args,
3232 3233 )
3233 3234
3234 3235 COMMENT_OUTDATED = u'comment_outdated'
3235 3236 COMMENT_TYPE_NOTE = u'note'
3236 3237 COMMENT_TYPE_TODO = u'todo'
3237 3238 COMMENT_TYPES = [COMMENT_TYPE_NOTE, COMMENT_TYPE_TODO]
3238 3239
3239 3240 comment_id = Column('comment_id', Integer(), nullable=False, primary_key=True)
3240 3241 repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False)
3241 3242 revision = Column('revision', String(40), nullable=True)
3242 3243 pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
3243 3244 pull_request_version_id = Column("pull_request_version_id", Integer(), ForeignKey('pull_request_versions.pull_request_version_id'), nullable=True)
3244 3245 line_no = Column('line_no', Unicode(10), nullable=True)
3245 3246 hl_lines = Column('hl_lines', Unicode(512), nullable=True)
3246 3247 f_path = Column('f_path', Unicode(1000), nullable=True)
3247 3248 user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
3248 3249 text = Column('text', UnicodeText().with_variant(UnicodeText(25000), 'mysql'), nullable=False)
3249 3250 created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
3250 3251 modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
3251 3252 renderer = Column('renderer', Unicode(64), nullable=True)
3252 3253 display_state = Column('display_state', Unicode(128), nullable=True)
3253 3254
3254 3255 comment_type = Column('comment_type', Unicode(128), nullable=True, default=COMMENT_TYPE_NOTE)
3255 3256 resolved_comment_id = Column('resolved_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'), nullable=True)
3256 3257 resolved_comment = relationship('ChangesetComment', remote_side=comment_id, backref='resolved_by')
3257 3258 author = relationship('User', lazy='joined')
3258 3259 repo = relationship('Repository')
3259 3260 status_change = relationship('ChangesetStatus', cascade="all, delete, delete-orphan", lazy='joined')
3260 3261 pull_request = relationship('PullRequest', lazy='joined')
3261 3262 pull_request_version = relationship('PullRequestVersion')
3262 3263
3263 3264 @classmethod
3264 3265 def get_users(cls, revision=None, pull_request_id=None):
3265 3266 """
3266 3267 Returns user associated with this ChangesetComment. ie those
3267 3268 who actually commented
3268 3269
3269 3270 :param cls:
3270 3271 :param revision:
3271 3272 """
3272 3273 q = Session().query(User)\
3273 3274 .join(ChangesetComment.author)
3274 3275 if revision:
3275 3276 q = q.filter(cls.revision == revision)
3276 3277 elif pull_request_id:
3277 3278 q = q.filter(cls.pull_request_id == pull_request_id)
3278 3279 return q.all()
3279 3280
3280 3281 @classmethod
3281 3282 def get_index_from_version(cls, pr_version, versions):
3282 3283 num_versions = [x.pull_request_version_id for x in versions]
3283 3284 try:
3284 3285 return num_versions.index(pr_version) +1
3285 3286 except (IndexError, ValueError):
3286 3287 return
3287 3288
3288 3289 @property
3289 3290 def outdated(self):
3290 3291 return self.display_state == self.COMMENT_OUTDATED
3291 3292
3292 3293 def outdated_at_version(self, version):
3293 3294 """
3294 3295 Checks if comment is outdated for given pull request version
3295 3296 """
3296 3297 return self.outdated and self.pull_request_version_id != version
3297 3298
3298 3299 def older_than_version(self, version):
3299 3300 """
3300 3301 Checks if comment is made from previous version than given
3301 3302 """
3302 3303 if version is None:
3303 3304 return self.pull_request_version_id is not None
3304 3305
3305 3306 return self.pull_request_version_id < version
3306 3307
3307 3308 @property
3308 3309 def resolved(self):
3309 3310 return self.resolved_by[0] if self.resolved_by else None
3310 3311
3311 3312 @property
3312 3313 def is_todo(self):
3313 3314 return self.comment_type == self.COMMENT_TYPE_TODO
3314 3315
3315 3316 @property
3316 3317 def is_inline(self):
3317 3318 return self.line_no and self.f_path
3318 3319
3319 3320 def get_index_version(self, versions):
3320 3321 return self.get_index_from_version(
3321 3322 self.pull_request_version_id, versions)
3322 3323
3323 3324 def __repr__(self):
3324 3325 if self.comment_id:
3325 3326 return '<DB:Comment #%s>' % self.comment_id
3326 3327 else:
3327 3328 return '<DB:Comment at %#x>' % id(self)
3328 3329
3329 3330 def get_api_data(self):
3330 3331 comment = self
3331 3332 data = {
3332 3333 'comment_id': comment.comment_id,
3333 3334 'comment_type': comment.comment_type,
3334 3335 'comment_text': comment.text,
3335 3336 'comment_status': comment.status_change,
3336 3337 'comment_f_path': comment.f_path,
3337 3338 'comment_lineno': comment.line_no,
3338 3339 'comment_author': comment.author,
3339 3340 'comment_created_on': comment.created_on
3340 3341 }
3341 3342 return data
3342 3343
3343 3344 def __json__(self):
3344 3345 data = dict()
3345 3346 data.update(self.get_api_data())
3346 3347 return data
3347 3348
3348 3349
3349 3350 class ChangesetStatus(Base, BaseModel):
3350 3351 __tablename__ = 'changeset_statuses'
3351 3352 __table_args__ = (
3352 3353 Index('cs_revision_idx', 'revision'),
3353 3354 Index('cs_version_idx', 'version'),
3354 3355 UniqueConstraint('repo_id', 'revision', 'version'),
3355 3356 base_table_args
3356 3357 )
3357 3358
3358 3359 STATUS_NOT_REVIEWED = DEFAULT = 'not_reviewed'
3359 3360 STATUS_APPROVED = 'approved'
3360 3361 STATUS_REJECTED = 'rejected'
3361 3362 STATUS_UNDER_REVIEW = 'under_review'
3362 3363
3363 3364 STATUSES = [
3364 3365 (STATUS_NOT_REVIEWED, _("Not Reviewed")), # (no icon) and default
3365 3366 (STATUS_APPROVED, _("Approved")),
3366 3367 (STATUS_REJECTED, _("Rejected")),
3367 3368 (STATUS_UNDER_REVIEW, _("Under Review")),
3368 3369 ]
3369 3370
3370 3371 changeset_status_id = Column('changeset_status_id', Integer(), nullable=False, primary_key=True)
3371 3372 repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False)
3372 3373 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None)
3373 3374 revision = Column('revision', String(40), nullable=False)
3374 3375 status = Column('status', String(128), nullable=False, default=DEFAULT)
3375 3376 changeset_comment_id = Column('changeset_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'))
3376 3377 modified_at = Column('modified_at', DateTime(), nullable=False, default=datetime.datetime.now)
3377 3378 version = Column('version', Integer(), nullable=False, default=0)
3378 3379 pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
3379 3380
3380 3381 author = relationship('User', lazy='joined')
3381 3382 repo = relationship('Repository')
3382 3383 comment = relationship('ChangesetComment', lazy='joined')
3383 3384 pull_request = relationship('PullRequest', lazy='joined')
3384 3385
3385 3386 def __unicode__(self):
3386 3387 return u"<%s('%s[v%s]:%s')>" % (
3387 3388 self.__class__.__name__,
3388 3389 self.status, self.version, self.author
3389 3390 )
3390 3391
3391 3392 @classmethod
3392 3393 def get_status_lbl(cls, value):
3393 3394 return dict(cls.STATUSES).get(value)
3394 3395
3395 3396 @property
3396 3397 def status_lbl(self):
3397 3398 return ChangesetStatus.get_status_lbl(self.status)
3398 3399
3399 3400 def get_api_data(self):
3400 3401 status = self
3401 3402 data = {
3402 3403 'status_id': status.changeset_status_id,
3403 3404 'status': status.status,
3404 3405 }
3405 3406 return data
3406 3407
3407 3408 def __json__(self):
3408 3409 data = dict()
3409 3410 data.update(self.get_api_data())
3410 3411 return data
3411 3412
3412 3413
3413 3414 class _PullRequestBase(BaseModel):
3414 3415 """
3415 3416 Common attributes of pull request and version entries.
3416 3417 """
3417 3418
3418 3419 # .status values
3419 3420 STATUS_NEW = u'new'
3420 3421 STATUS_OPEN = u'open'
3421 3422 STATUS_CLOSED = u'closed'
3422 3423
3423 3424 title = Column('title', Unicode(255), nullable=True)
3424 3425 description = Column(
3425 3426 'description', UnicodeText().with_variant(UnicodeText(10240), 'mysql'),
3426 3427 nullable=True)
3427 3428 description_renderer = Column('description_renderer', Unicode(64), nullable=True)
3428 3429
3429 3430 # new/open/closed status of pull request (not approve/reject/etc)
3430 3431 status = Column('status', Unicode(255), nullable=False, default=STATUS_NEW)
3431 3432 created_on = Column(
3432 3433 'created_on', DateTime(timezone=False), nullable=False,
3433 3434 default=datetime.datetime.now)
3434 3435 updated_on = Column(
3435 3436 'updated_on', DateTime(timezone=False), nullable=False,
3436 3437 default=datetime.datetime.now)
3437 3438
3438 3439 @declared_attr
3439 3440 def user_id(cls):
3440 3441 return Column(
3441 3442 "user_id", Integer(), ForeignKey('users.user_id'), nullable=False,
3442 3443 unique=None)
3443 3444
3444 3445 # 500 revisions max
3445 3446 _revisions = Column(
3446 3447 'revisions', UnicodeText().with_variant(UnicodeText(20500), 'mysql'))
3447 3448
3448 3449 @declared_attr
3449 3450 def source_repo_id(cls):
3450 3451 # TODO: dan: rename column to source_repo_id
3451 3452 return Column(
3452 3453 'org_repo_id', Integer(), ForeignKey('repositories.repo_id'),
3453 3454 nullable=False)
3454 3455
3455 3456 source_ref = Column('org_ref', Unicode(255), nullable=False)
3456 3457
3457 3458 @declared_attr
3458 3459 def target_repo_id(cls):
3459 3460 # TODO: dan: rename column to target_repo_id
3460 3461 return Column(
3461 3462 'other_repo_id', Integer(), ForeignKey('repositories.repo_id'),
3462 3463 nullable=False)
3463 3464
3464 3465 target_ref = Column('other_ref', Unicode(255), nullable=False)
3465 3466 _shadow_merge_ref = Column('shadow_merge_ref', Unicode(255), nullable=True)
3466 3467
3467 3468 # TODO: dan: rename column to last_merge_source_rev
3468 3469 _last_merge_source_rev = Column(
3469 3470 'last_merge_org_rev', String(40), nullable=True)
3470 3471 # TODO: dan: rename column to last_merge_target_rev
3471 3472 _last_merge_target_rev = Column(
3472 3473 'last_merge_other_rev', String(40), nullable=True)
3473 3474 _last_merge_status = Column('merge_status', Integer(), nullable=True)
3474 3475 merge_rev = Column('merge_rev', String(40), nullable=True)
3475 3476
3476 3477 reviewer_data = Column(
3477 3478 'reviewer_data_json', MutationObj.as_mutable(
3478 3479 JsonType(dialect_map=dict(mysql=UnicodeText(16384)))))
3479 3480
3480 3481 @property
3481 3482 def reviewer_data_json(self):
3482 3483 return json.dumps(self.reviewer_data)
3483 3484
3484 3485 @hybrid_property
3485 3486 def description_safe(self):
3486 3487 from rhodecode.lib import helpers as h
3487 3488 return h.escape(self.description)
3488 3489
3489 3490 @hybrid_property
3490 3491 def revisions(self):
3491 3492 return self._revisions.split(':') if self._revisions else []
3492 3493
3493 3494 @revisions.setter
3494 3495 def revisions(self, val):
3495 3496 self._revisions = ':'.join(val)
3496 3497
3497 3498 @hybrid_property
3498 3499 def last_merge_status(self):
3499 3500 return safe_int(self._last_merge_status)
3500 3501
3501 3502 @last_merge_status.setter
3502 3503 def last_merge_status(self, val):
3503 3504 self._last_merge_status = val
3504 3505
3505 3506 @declared_attr
3506 3507 def author(cls):
3507 3508 return relationship('User', lazy='joined')
3508 3509
3509 3510 @declared_attr
3510 3511 def source_repo(cls):
3511 3512 return relationship(
3512 3513 'Repository',
3513 3514 primaryjoin='%s.source_repo_id==Repository.repo_id' % cls.__name__)
3514 3515
3515 3516 @property
3516 3517 def source_ref_parts(self):
3517 3518 return self.unicode_to_reference(self.source_ref)
3518 3519
3519 3520 @declared_attr
3520 3521 def target_repo(cls):
3521 3522 return relationship(
3522 3523 'Repository',
3523 3524 primaryjoin='%s.target_repo_id==Repository.repo_id' % cls.__name__)
3524 3525
3525 3526 @property
3526 3527 def target_ref_parts(self):
3527 3528 return self.unicode_to_reference(self.target_ref)
3528 3529
3529 3530 @property
3530 3531 def shadow_merge_ref(self):
3531 3532 return self.unicode_to_reference(self._shadow_merge_ref)
3532 3533
3533 3534 @shadow_merge_ref.setter
3534 3535 def shadow_merge_ref(self, ref):
3535 3536 self._shadow_merge_ref = self.reference_to_unicode(ref)
3536 3537
3537 3538 def unicode_to_reference(self, raw):
3538 3539 """
3539 3540 Convert a unicode (or string) to a reference object.
3540 3541 If unicode evaluates to False it returns None.
3541 3542 """
3542 3543 if raw:
3543 3544 refs = raw.split(':')
3544 3545 return Reference(*refs)
3545 3546 else:
3546 3547 return None
3547 3548
3548 3549 def reference_to_unicode(self, ref):
3549 3550 """
3550 3551 Convert a reference object to unicode.
3551 3552 If reference is None it returns None.
3552 3553 """
3553 3554 if ref:
3554 3555 return u':'.join(ref)
3555 3556 else:
3556 3557 return None
3557 3558
3558 3559 def get_api_data(self, with_merge_state=True):
3559 3560 from rhodecode.model.pull_request import PullRequestModel
3560 3561
3561 3562 pull_request = self
3562 3563 if with_merge_state:
3563 3564 merge_status = PullRequestModel().merge_status(pull_request)
3564 3565 merge_state = {
3565 3566 'status': merge_status[0],
3566 3567 'message': safe_unicode(merge_status[1]),
3567 3568 }
3568 3569 else:
3569 3570 merge_state = {'status': 'not_available',
3570 3571 'message': 'not_available'}
3571 3572
3572 3573 merge_data = {
3573 3574 'clone_url': PullRequestModel().get_shadow_clone_url(pull_request),
3574 3575 'reference': (
3575 3576 pull_request.shadow_merge_ref._asdict()
3576 3577 if pull_request.shadow_merge_ref else None),
3577 3578 }
3578 3579
3579 3580 data = {
3580 3581 'pull_request_id': pull_request.pull_request_id,
3581 3582 'url': PullRequestModel().get_url(pull_request),
3582 3583 'title': pull_request.title,
3583 3584 'description': pull_request.description,
3584 3585 'status': pull_request.status,
3585 3586 'created_on': pull_request.created_on,
3586 3587 'updated_on': pull_request.updated_on,
3587 3588 'commit_ids': pull_request.revisions,
3588 3589 'review_status': pull_request.calculated_review_status(),
3589 3590 'mergeable': merge_state,
3590 3591 'source': {
3591 3592 'clone_url': pull_request.source_repo.clone_url(),
3592 3593 'repository': pull_request.source_repo.repo_name,
3593 3594 'reference': {
3594 3595 'name': pull_request.source_ref_parts.name,
3595 3596 'type': pull_request.source_ref_parts.type,
3596 3597 'commit_id': pull_request.source_ref_parts.commit_id,
3597 3598 },
3598 3599 },
3599 3600 'target': {
3600 3601 'clone_url': pull_request.target_repo.clone_url(),
3601 3602 'repository': pull_request.target_repo.repo_name,
3602 3603 'reference': {
3603 3604 'name': pull_request.target_ref_parts.name,
3604 3605 'type': pull_request.target_ref_parts.type,
3605 3606 'commit_id': pull_request.target_ref_parts.commit_id,
3606 3607 },
3607 3608 },
3608 3609 'merge': merge_data,
3609 3610 'author': pull_request.author.get_api_data(include_secrets=False,
3610 3611 details='basic'),
3611 3612 'reviewers': [
3612 3613 {
3613 3614 'user': reviewer.get_api_data(include_secrets=False,
3614 3615 details='basic'),
3615 3616 'reasons': reasons,
3616 3617 'review_status': st[0][1].status if st else 'not_reviewed',
3617 3618 }
3618 3619 for obj, reviewer, reasons, mandatory, st in
3619 3620 pull_request.reviewers_statuses()
3620 3621 ]
3621 3622 }
3622 3623
3623 3624 return data
3624 3625
3625 3626
3626 3627 class PullRequest(Base, _PullRequestBase):
3627 3628 __tablename__ = 'pull_requests'
3628 3629 __table_args__ = (
3629 3630 base_table_args,
3630 3631 )
3631 3632
3632 3633 pull_request_id = Column(
3633 3634 'pull_request_id', Integer(), nullable=False, primary_key=True)
3634 3635
3635 3636 def __repr__(self):
3636 3637 if self.pull_request_id:
3637 3638 return '<DB:PullRequest #%s>' % self.pull_request_id
3638 3639 else:
3639 3640 return '<DB:PullRequest at %#x>' % id(self)
3640 3641
3641 3642 reviewers = relationship('PullRequestReviewers',
3642 3643 cascade="all, delete, delete-orphan")
3643 3644 statuses = relationship('ChangesetStatus',
3644 3645 cascade="all, delete, delete-orphan")
3645 3646 comments = relationship('ChangesetComment',
3646 3647 cascade="all, delete, delete-orphan")
3647 3648 versions = relationship('PullRequestVersion',
3648 3649 cascade="all, delete, delete-orphan",
3649 3650 lazy='dynamic')
3650 3651
3651 3652 @classmethod
3652 3653 def get_pr_display_object(cls, pull_request_obj, org_pull_request_obj,
3653 3654 internal_methods=None):
3654 3655
3655 3656 class PullRequestDisplay(object):
3656 3657 """
3657 3658 Special object wrapper for showing PullRequest data via Versions
3658 3659 It mimics PR object as close as possible. This is read only object
3659 3660 just for display
3660 3661 """
3661 3662
3662 3663 def __init__(self, attrs, internal=None):
3663 3664 self.attrs = attrs
3664 3665 # internal have priority over the given ones via attrs
3665 3666 self.internal = internal or ['versions']
3666 3667
3667 3668 def __getattr__(self, item):
3668 3669 if item in self.internal:
3669 3670 return getattr(self, item)
3670 3671 try:
3671 3672 return self.attrs[item]
3672 3673 except KeyError:
3673 3674 raise AttributeError(
3674 3675 '%s object has no attribute %s' % (self, item))
3675 3676
3676 3677 def __repr__(self):
3677 3678 return '<DB:PullRequestDisplay #%s>' % self.attrs.get('pull_request_id')
3678 3679
3679 3680 def versions(self):
3680 3681 return pull_request_obj.versions.order_by(
3681 3682 PullRequestVersion.pull_request_version_id).all()
3682 3683
3683 3684 def is_closed(self):
3684 3685 return pull_request_obj.is_closed()
3685 3686
3686 3687 @property
3687 3688 def pull_request_version_id(self):
3688 3689 return getattr(pull_request_obj, 'pull_request_version_id', None)
3689 3690
3690 3691 attrs = StrictAttributeDict(pull_request_obj.get_api_data())
3691 3692
3692 3693 attrs.author = StrictAttributeDict(
3693 3694 pull_request_obj.author.get_api_data())
3694 3695 if pull_request_obj.target_repo:
3695 3696 attrs.target_repo = StrictAttributeDict(
3696 3697 pull_request_obj.target_repo.get_api_data())
3697 3698 attrs.target_repo.clone_url = pull_request_obj.target_repo.clone_url
3698 3699
3699 3700 if pull_request_obj.source_repo:
3700 3701 attrs.source_repo = StrictAttributeDict(
3701 3702 pull_request_obj.source_repo.get_api_data())
3702 3703 attrs.source_repo.clone_url = pull_request_obj.source_repo.clone_url
3703 3704
3704 3705 attrs.source_ref_parts = pull_request_obj.source_ref_parts
3705 3706 attrs.target_ref_parts = pull_request_obj.target_ref_parts
3706 3707 attrs.revisions = pull_request_obj.revisions
3707 3708
3708 3709 attrs.shadow_merge_ref = org_pull_request_obj.shadow_merge_ref
3709 3710 attrs.reviewer_data = org_pull_request_obj.reviewer_data
3710 3711 attrs.reviewer_data_json = org_pull_request_obj.reviewer_data_json
3711 3712
3712 3713 return PullRequestDisplay(attrs, internal=internal_methods)
3713 3714
3714 3715 def is_closed(self):
3715 3716 return self.status == self.STATUS_CLOSED
3716 3717
3717 3718 def __json__(self):
3718 3719 return {
3719 3720 'revisions': self.revisions,
3720 3721 }
3721 3722
3722 3723 def calculated_review_status(self):
3723 3724 from rhodecode.model.changeset_status import ChangesetStatusModel
3724 3725 return ChangesetStatusModel().calculated_review_status(self)
3725 3726
3726 3727 def reviewers_statuses(self):
3727 3728 from rhodecode.model.changeset_status import ChangesetStatusModel
3728 3729 return ChangesetStatusModel().reviewers_statuses(self)
3729 3730
3730 3731 @property
3731 3732 def workspace_id(self):
3732 3733 from rhodecode.model.pull_request import PullRequestModel
3733 3734 return PullRequestModel()._workspace_id(self)
3734 3735
3735 3736 def get_shadow_repo(self):
3736 3737 workspace_id = self.workspace_id
3737 3738 vcs_obj = self.target_repo.scm_instance()
3738 3739 shadow_repository_path = vcs_obj._get_shadow_repository_path(
3739 3740 self.target_repo.repo_id, workspace_id)
3740 3741 if os.path.isdir(shadow_repository_path):
3741 3742 return vcs_obj._get_shadow_instance(shadow_repository_path)
3742 3743
3743 3744
3744 3745 class PullRequestVersion(Base, _PullRequestBase):
3745 3746 __tablename__ = 'pull_request_versions'
3746 3747 __table_args__ = (
3747 3748 base_table_args,
3748 3749 )
3749 3750
3750 3751 pull_request_version_id = Column(
3751 3752 'pull_request_version_id', Integer(), nullable=False, primary_key=True)
3752 3753 pull_request_id = Column(
3753 3754 'pull_request_id', Integer(),
3754 3755 ForeignKey('pull_requests.pull_request_id'), nullable=False)
3755 3756 pull_request = relationship('PullRequest')
3756 3757
3757 3758 def __repr__(self):
3758 3759 if self.pull_request_version_id:
3759 3760 return '<DB:PullRequestVersion #%s>' % self.pull_request_version_id
3760 3761 else:
3761 3762 return '<DB:PullRequestVersion at %#x>' % id(self)
3762 3763
3763 3764 @property
3764 3765 def reviewers(self):
3765 3766 return self.pull_request.reviewers
3766 3767
3767 3768 @property
3768 3769 def versions(self):
3769 3770 return self.pull_request.versions
3770 3771
3771 3772 def is_closed(self):
3772 3773 # calculate from original
3773 3774 return self.pull_request.status == self.STATUS_CLOSED
3774 3775
3775 3776 def calculated_review_status(self):
3776 3777 return self.pull_request.calculated_review_status()
3777 3778
3778 3779 def reviewers_statuses(self):
3779 3780 return self.pull_request.reviewers_statuses()
3780 3781
3781 3782
3782 3783 class PullRequestReviewers(Base, BaseModel):
3783 3784 __tablename__ = 'pull_request_reviewers'
3784 3785 __table_args__ = (
3785 3786 base_table_args,
3786 3787 )
3787 3788
3788 3789 @hybrid_property
3789 3790 def reasons(self):
3790 3791 if not self._reasons:
3791 3792 return []
3792 3793 return self._reasons
3793 3794
3794 3795 @reasons.setter
3795 3796 def reasons(self, val):
3796 3797 val = val or []
3797 3798 if any(not isinstance(x, basestring) for x in val):
3798 3799 raise Exception('invalid reasons type, must be list of strings')
3799 3800 self._reasons = val
3800 3801
3801 3802 pull_requests_reviewers_id = Column(
3802 3803 'pull_requests_reviewers_id', Integer(), nullable=False,
3803 3804 primary_key=True)
3804 3805 pull_request_id = Column(
3805 3806 "pull_request_id", Integer(),
3806 3807 ForeignKey('pull_requests.pull_request_id'), nullable=False)
3807 3808 user_id = Column(
3808 3809 "user_id", Integer(), ForeignKey('users.user_id'), nullable=True)
3809 3810 _reasons = Column(
3810 3811 'reason', MutationList.as_mutable(
3811 3812 JsonType('list', dialect_map=dict(mysql=UnicodeText(16384)))))
3812 3813
3813 3814 mandatory = Column("mandatory", Boolean(), nullable=False, default=False)
3814 3815 user = relationship('User')
3815 3816 pull_request = relationship('PullRequest')
3816 3817
3817 3818 rule_data = Column(
3818 3819 'rule_data_json',
3819 3820 JsonType(dialect_map=dict(mysql=UnicodeText(16384))))
3820 3821
3821 3822 def rule_user_group_data(self):
3822 3823 """
3823 3824 Returns the voting user group rule data for this reviewer
3824 3825 """
3825 3826
3826 3827 if self.rule_data and 'vote_rule' in self.rule_data:
3827 3828 user_group_data = {}
3828 3829 if 'rule_user_group_entry_id' in self.rule_data:
3829 3830 # means a group with voting rules !
3830 3831 user_group_data['id'] = self.rule_data['rule_user_group_entry_id']
3831 3832 user_group_data['name'] = self.rule_data['rule_name']
3832 3833 user_group_data['vote_rule'] = self.rule_data['vote_rule']
3833 3834
3834 3835 return user_group_data
3835 3836
3836 3837 def __unicode__(self):
3837 3838 return u"<%s('id:%s')>" % (self.__class__.__name__,
3838 3839 self.pull_requests_reviewers_id)
3839 3840
3840 3841
3841 3842 class Notification(Base, BaseModel):
3842 3843 __tablename__ = 'notifications'
3843 3844 __table_args__ = (
3844 3845 Index('notification_type_idx', 'type'),
3845 3846 base_table_args,
3846 3847 )
3847 3848
3848 3849 TYPE_CHANGESET_COMMENT = u'cs_comment'
3849 3850 TYPE_MESSAGE = u'message'
3850 3851 TYPE_MENTION = u'mention'
3851 3852 TYPE_REGISTRATION = u'registration'
3852 3853 TYPE_PULL_REQUEST = u'pull_request'
3853 3854 TYPE_PULL_REQUEST_COMMENT = u'pull_request_comment'
3854 3855
3855 3856 notification_id = Column('notification_id', Integer(), nullable=False, primary_key=True)
3856 3857 subject = Column('subject', Unicode(512), nullable=True)
3857 3858 body = Column('body', UnicodeText().with_variant(UnicodeText(50000), 'mysql'), nullable=True)
3858 3859 created_by = Column("created_by", Integer(), ForeignKey('users.user_id'), nullable=True)
3859 3860 created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
3860 3861 type_ = Column('type', Unicode(255))
3861 3862
3862 3863 created_by_user = relationship('User')
3863 3864 notifications_to_users = relationship('UserNotification', lazy='joined',
3864 3865 cascade="all, delete, delete-orphan")
3865 3866
3866 3867 @property
3867 3868 def recipients(self):
3868 3869 return [x.user for x in UserNotification.query()\
3869 3870 .filter(UserNotification.notification == self)\
3870 3871 .order_by(UserNotification.user_id.asc()).all()]
3871 3872
3872 3873 @classmethod
3873 3874 def create(cls, created_by, subject, body, recipients, type_=None):
3874 3875 if type_ is None:
3875 3876 type_ = Notification.TYPE_MESSAGE
3876 3877
3877 3878 notification = cls()
3878 3879 notification.created_by_user = created_by
3879 3880 notification.subject = subject
3880 3881 notification.body = body
3881 3882 notification.type_ = type_
3882 3883 notification.created_on = datetime.datetime.now()
3883 3884
3884 3885 # For each recipient link the created notification to his account
3885 3886 for u in recipients:
3886 3887 assoc = UserNotification()
3887 3888 assoc.user_id = u.user_id
3888 3889 assoc.notification = notification
3889 3890
3890 3891 # if created_by is inside recipients mark his notification
3891 3892 # as read
3892 3893 if u.user_id == created_by.user_id:
3893 3894 assoc.read = True
3894 3895 Session().add(assoc)
3895 3896
3896 3897 Session().add(notification)
3897 3898
3898 3899 return notification
3899 3900
3900 3901
3901 3902 class UserNotification(Base, BaseModel):
3902 3903 __tablename__ = 'user_to_notification'
3903 3904 __table_args__ = (
3904 3905 UniqueConstraint('user_id', 'notification_id'),
3905 3906 base_table_args
3906 3907 )
3907 3908
3908 3909 user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), primary_key=True)
3909 3910 notification_id = Column("notification_id", Integer(), ForeignKey('notifications.notification_id'), primary_key=True)
3910 3911 read = Column('read', Boolean, default=False)
3911 3912 sent_on = Column('sent_on', DateTime(timezone=False), nullable=True, unique=None)
3912 3913
3913 3914 user = relationship('User', lazy="joined")
3914 3915 notification = relationship('Notification', lazy="joined",
3915 3916 order_by=lambda: Notification.created_on.desc(),)
3916 3917
3917 3918 def mark_as_read(self):
3918 3919 self.read = True
3919 3920 Session().add(self)
3920 3921
3921 3922
3922 3923 class Gist(Base, BaseModel):
3923 3924 __tablename__ = 'gists'
3924 3925 __table_args__ = (
3925 3926 Index('g_gist_access_id_idx', 'gist_access_id'),
3926 3927 Index('g_created_on_idx', 'created_on'),
3927 3928 base_table_args
3928 3929 )
3929 3930
3930 3931 GIST_PUBLIC = u'public'
3931 3932 GIST_PRIVATE = u'private'
3932 3933 DEFAULT_FILENAME = u'gistfile1.txt'
3933 3934
3934 3935 ACL_LEVEL_PUBLIC = u'acl_public'
3935 3936 ACL_LEVEL_PRIVATE = u'acl_private'
3936 3937
3937 3938 gist_id = Column('gist_id', Integer(), primary_key=True)
3938 3939 gist_access_id = Column('gist_access_id', Unicode(250))
3939 3940 gist_description = Column('gist_description', UnicodeText().with_variant(UnicodeText(1024), 'mysql'))
3940 3941 gist_owner = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=True)
3941 3942 gist_expires = Column('gist_expires', Float(53), nullable=False)
3942 3943 gist_type = Column('gist_type', Unicode(128), nullable=False)
3943 3944 created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
3944 3945 modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
3945 3946 acl_level = Column('acl_level', Unicode(128), nullable=True)
3946 3947
3947 3948 owner = relationship('User')
3948 3949
3949 3950 def __repr__(self):
3950 3951 return '<Gist:[%s]%s>' % (self.gist_type, self.gist_access_id)
3951 3952
3952 3953 @hybrid_property
3953 3954 def description_safe(self):
3954 3955 from rhodecode.lib import helpers as h
3955 3956 return h.escape(self.gist_description)
3956 3957
3957 3958 @classmethod
3958 3959 def get_or_404(cls, id_):
3959 3960 from pyramid.httpexceptions import HTTPNotFound
3960 3961
3961 3962 res = cls.query().filter(cls.gist_access_id == id_).scalar()
3962 3963 if not res:
3963 3964 raise HTTPNotFound()
3964 3965 return res
3965 3966
3966 3967 @classmethod
3967 3968 def get_by_access_id(cls, gist_access_id):
3968 3969 return cls.query().filter(cls.gist_access_id == gist_access_id).scalar()
3969 3970
3970 3971 def gist_url(self):
3971 3972 from rhodecode.model.gist import GistModel
3972 3973 return GistModel().get_url(self)
3973 3974
3974 3975 @classmethod
3975 3976 def base_path(cls):
3976 3977 """
3977 3978 Returns base path when all gists are stored
3978 3979
3979 3980 :param cls:
3980 3981 """
3981 3982 from rhodecode.model.gist import GIST_STORE_LOC
3982 3983 q = Session().query(RhodeCodeUi)\
3983 3984 .filter(RhodeCodeUi.ui_key == URL_SEP)
3984 3985 q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
3985 3986 return os.path.join(q.one().ui_value, GIST_STORE_LOC)
3986 3987
3987 3988 def get_api_data(self):
3988 3989 """
3989 3990 Common function for generating gist related data for API
3990 3991 """
3991 3992 gist = self
3992 3993 data = {
3993 3994 'gist_id': gist.gist_id,
3994 3995 'type': gist.gist_type,
3995 3996 'access_id': gist.gist_access_id,
3996 3997 'description': gist.gist_description,
3997 3998 'url': gist.gist_url(),
3998 3999 'expires': gist.gist_expires,
3999 4000 'created_on': gist.created_on,
4000 4001 'modified_at': gist.modified_at,
4001 4002 'content': None,
4002 4003 'acl_level': gist.acl_level,
4003 4004 }
4004 4005 return data
4005 4006
4006 4007 def __json__(self):
4007 4008 data = dict(
4008 4009 )
4009 4010 data.update(self.get_api_data())
4010 4011 return data
4011 4012 # SCM functions
4012 4013
4013 4014 def scm_instance(self, **kwargs):
4014 4015 full_repo_path = os.path.join(self.base_path(), self.gist_access_id)
4015 4016 return get_vcs_instance(
4016 4017 repo_path=safe_str(full_repo_path), create=False)
4017 4018
4018 4019
4019 4020 class ExternalIdentity(Base, BaseModel):
4020 4021 __tablename__ = 'external_identities'
4021 4022 __table_args__ = (
4022 4023 Index('local_user_id_idx', 'local_user_id'),
4023 4024 Index('external_id_idx', 'external_id'),
4024 4025 base_table_args
4025 4026 )
4026 4027
4027 4028 external_id = Column('external_id', Unicode(255), default=u'',
4028 4029 primary_key=True)
4029 4030 external_username = Column('external_username', Unicode(1024), default=u'')
4030 4031 local_user_id = Column('local_user_id', Integer(),
4031 4032 ForeignKey('users.user_id'), primary_key=True)
4032 4033 provider_name = Column('provider_name', Unicode(255), default=u'',
4033 4034 primary_key=True)
4034 4035 access_token = Column('access_token', String(1024), default=u'')
4035 4036 alt_token = Column('alt_token', String(1024), default=u'')
4036 4037 token_secret = Column('token_secret', String(1024), default=u'')
4037 4038
4038 4039 @classmethod
4039 4040 def by_external_id_and_provider(cls, external_id, provider_name,
4040 4041 local_user_id=None):
4041 4042 """
4042 4043 Returns ExternalIdentity instance based on search params
4043 4044
4044 4045 :param external_id:
4045 4046 :param provider_name:
4046 4047 :return: ExternalIdentity
4047 4048 """
4048 4049 query = cls.query()
4049 4050 query = query.filter(cls.external_id == external_id)
4050 4051 query = query.filter(cls.provider_name == provider_name)
4051 4052 if local_user_id:
4052 4053 query = query.filter(cls.local_user_id == local_user_id)
4053 4054 return query.first()
4054 4055
4055 4056 @classmethod
4056 4057 def user_by_external_id_and_provider(cls, external_id, provider_name):
4057 4058 """
4058 4059 Returns User instance based on search params
4059 4060
4060 4061 :param external_id:
4061 4062 :param provider_name:
4062 4063 :return: User
4063 4064 """
4064 4065 query = User.query()
4065 4066 query = query.filter(cls.external_id == external_id)
4066 4067 query = query.filter(cls.provider_name == provider_name)
4067 4068 query = query.filter(User.user_id == cls.local_user_id)
4068 4069 return query.first()
4069 4070
4070 4071 @classmethod
4071 4072 def by_local_user_id(cls, local_user_id):
4072 4073 """
4073 4074 Returns all tokens for user
4074 4075
4075 4076 :param local_user_id:
4076 4077 :return: ExternalIdentity
4077 4078 """
4078 4079 query = cls.query()
4079 4080 query = query.filter(cls.local_user_id == local_user_id)
4080 4081 return query
4081 4082
4082 4083
4083 4084 class Integration(Base, BaseModel):
4084 4085 __tablename__ = 'integrations'
4085 4086 __table_args__ = (
4086 4087 base_table_args
4087 4088 )
4088 4089
4089 4090 integration_id = Column('integration_id', Integer(), primary_key=True)
4090 4091 integration_type = Column('integration_type', String(255))
4091 4092 enabled = Column('enabled', Boolean(), nullable=False)
4092 4093 name = Column('name', String(255), nullable=False)
4093 4094 child_repos_only = Column('child_repos_only', Boolean(), nullable=False,
4094 4095 default=False)
4095 4096
4096 4097 settings = Column(
4097 4098 'settings_json', MutationObj.as_mutable(
4098 4099 JsonType(dialect_map=dict(mysql=UnicodeText(16384)))))
4099 4100 repo_id = Column(
4100 4101 'repo_id', Integer(), ForeignKey('repositories.repo_id'),
4101 4102 nullable=True, unique=None, default=None)
4102 4103 repo = relationship('Repository', lazy='joined')
4103 4104
4104 4105 repo_group_id = Column(
4105 4106 'repo_group_id', Integer(), ForeignKey('groups.group_id'),
4106 4107 nullable=True, unique=None, default=None)
4107 4108 repo_group = relationship('RepoGroup', lazy='joined')
4108 4109
4109 4110 @property
4110 4111 def scope(self):
4111 4112 if self.repo:
4112 4113 return repr(self.repo)
4113 4114 if self.repo_group:
4114 4115 if self.child_repos_only:
4115 4116 return repr(self.repo_group) + ' (child repos only)'
4116 4117 else:
4117 4118 return repr(self.repo_group) + ' (recursive)'
4118 4119 if self.child_repos_only:
4119 4120 return 'root_repos'
4120 4121 return 'global'
4121 4122
4122 4123 def __repr__(self):
4123 4124 return '<Integration(%r, %r)>' % (self.integration_type, self.scope)
4124 4125
4125 4126
4126 4127 class RepoReviewRuleUser(Base, BaseModel):
4127 4128 __tablename__ = 'repo_review_rules_users'
4128 4129 __table_args__ = (
4129 4130 base_table_args
4130 4131 )
4131 4132
4132 4133 repo_review_rule_user_id = Column('repo_review_rule_user_id', Integer(), primary_key=True)
4133 4134 repo_review_rule_id = Column("repo_review_rule_id", Integer(), ForeignKey('repo_review_rules.repo_review_rule_id'))
4134 4135 user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False)
4135 4136 mandatory = Column("mandatory", Boolean(), nullable=False, default=False)
4136 4137 user = relationship('User')
4137 4138
4138 4139 def rule_data(self):
4139 4140 return {
4140 4141 'mandatory': self.mandatory
4141 4142 }
4142 4143
4143 4144
4144 4145 class RepoReviewRuleUserGroup(Base, BaseModel):
4145 4146 __tablename__ = 'repo_review_rules_users_groups'
4146 4147 __table_args__ = (
4147 4148 base_table_args
4148 4149 )
4149 4150
4150 4151 VOTE_RULE_ALL = -1
4151 4152
4152 4153 repo_review_rule_users_group_id = Column('repo_review_rule_users_group_id', Integer(), primary_key=True)
4153 4154 repo_review_rule_id = Column("repo_review_rule_id", Integer(), ForeignKey('repo_review_rules.repo_review_rule_id'))
4154 4155 users_group_id = Column("users_group_id", Integer(),ForeignKey('users_groups.users_group_id'), nullable=False)
4155 4156 mandatory = Column("mandatory", Boolean(), nullable=False, default=False)
4156 4157 vote_rule = Column("vote_rule", Integer(), nullable=True, default=VOTE_RULE_ALL)
4157 4158 users_group = relationship('UserGroup')
4158 4159
4159 4160 def rule_data(self):
4160 4161 return {
4161 4162 'mandatory': self.mandatory,
4162 4163 'vote_rule': self.vote_rule
4163 4164 }
4164 4165
4165 4166 @property
4166 4167 def vote_rule_label(self):
4167 4168 if not self.vote_rule or self.vote_rule == self.VOTE_RULE_ALL:
4168 4169 return 'all must vote'
4169 4170 else:
4170 4171 return 'min. vote {}'.format(self.vote_rule)
4171 4172
4172 4173
4173 4174 class RepoReviewRule(Base, BaseModel):
4174 4175 __tablename__ = 'repo_review_rules'
4175 4176 __table_args__ = (
4176 4177 base_table_args
4177 4178 )
4178 4179
4179 4180 repo_review_rule_id = Column(
4180 4181 'repo_review_rule_id', Integer(), primary_key=True)
4181 4182 repo_id = Column(
4182 4183 "repo_id", Integer(), ForeignKey('repositories.repo_id'))
4183 4184 repo = relationship('Repository', backref='review_rules')
4184 4185
4185 4186 review_rule_name = Column('review_rule_name', String(255))
4186 4187 _branch_pattern = Column("branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob
4187 4188 _target_branch_pattern = Column("target_branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob
4188 4189 _file_pattern = Column("file_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default=u'*') # glob
4189 4190
4190 4191 use_authors_for_review = Column("use_authors_for_review", Boolean(), nullable=False, default=False)
4191 4192 forbid_author_to_review = Column("forbid_author_to_review", Boolean(), nullable=False, default=False)
4192 4193 forbid_commit_author_to_review = Column("forbid_commit_author_to_review", Boolean(), nullable=False, default=False)
4193 4194 forbid_adding_reviewers = Column("forbid_adding_reviewers", Boolean(), nullable=False, default=False)
4194 4195
4195 4196 rule_users = relationship('RepoReviewRuleUser')
4196 4197 rule_user_groups = relationship('RepoReviewRuleUserGroup')
4197 4198
4198 4199 def _validate_pattern(self, value):
4199 4200 re.compile('^' + glob2re(value) + '$')
4200 4201
4201 4202 @hybrid_property
4202 4203 def source_branch_pattern(self):
4203 4204 return self._branch_pattern or '*'
4204 4205
4205 4206 @source_branch_pattern.setter
4206 4207 def source_branch_pattern(self, value):
4207 4208 self._validate_pattern(value)
4208 4209 self._branch_pattern = value or '*'
4209 4210
4210 4211 @hybrid_property
4211 4212 def target_branch_pattern(self):
4212 4213 return self._target_branch_pattern or '*'
4213 4214
4214 4215 @target_branch_pattern.setter
4215 4216 def target_branch_pattern(self, value):
4216 4217 self._validate_pattern(value)
4217 4218 self._target_branch_pattern = value or '*'
4218 4219
4219 4220 @hybrid_property
4220 4221 def file_pattern(self):
4221 4222 return self._file_pattern or '*'
4222 4223
4223 4224 @file_pattern.setter
4224 4225 def file_pattern(self, value):
4225 4226 self._validate_pattern(value)
4226 4227 self._file_pattern = value or '*'
4227 4228
4228 4229 def matches(self, source_branch, target_branch, files_changed):
4229 4230 """
4230 4231 Check if this review rule matches a branch/files in a pull request
4231 4232
4232 4233 :param source_branch: source branch name for the commit
4233 4234 :param target_branch: target branch name for the commit
4234 4235 :param files_changed: list of file paths changed in the pull request
4235 4236 """
4236 4237
4237 4238 source_branch = source_branch or ''
4238 4239 target_branch = target_branch or ''
4239 4240 files_changed = files_changed or []
4240 4241
4241 4242 branch_matches = True
4242 4243 if source_branch or target_branch:
4243 4244 if self.source_branch_pattern == '*':
4244 4245 source_branch_match = True
4245 4246 else:
4246 4247 if self.source_branch_pattern.startswith('re:'):
4247 4248 source_pattern = self.source_branch_pattern[3:]
4248 4249 else:
4249 4250 source_pattern = '^' + glob2re(self.source_branch_pattern) + '$'
4250 4251 source_branch_regex = re.compile(source_pattern)
4251 4252 source_branch_match = bool(source_branch_regex.search(source_branch))
4252 4253 if self.target_branch_pattern == '*':
4253 4254 target_branch_match = True
4254 4255 else:
4255 4256 if self.target_branch_pattern.startswith('re:'):
4256 4257 target_pattern = self.target_branch_pattern[3:]
4257 4258 else:
4258 4259 target_pattern = '^' + glob2re(self.target_branch_pattern) + '$'
4259 4260 target_branch_regex = re.compile(target_pattern)
4260 4261 target_branch_match = bool(target_branch_regex.search(target_branch))
4261 4262
4262 4263 branch_matches = source_branch_match and target_branch_match
4263 4264
4264 4265 files_matches = True
4265 4266 if self.file_pattern != '*':
4266 4267 files_matches = False
4267 4268 if self.file_pattern.startswith('re:'):
4268 4269 file_pattern = self.file_pattern[3:]
4269 4270 else:
4270 4271 file_pattern = glob2re(self.file_pattern)
4271 4272 file_regex = re.compile(file_pattern)
4272 4273 for filename in files_changed:
4273 4274 if file_regex.search(filename):
4274 4275 files_matches = True
4275 4276 break
4276 4277
4277 4278 return branch_matches and files_matches
4278 4279
4279 4280 @property
4280 4281 def review_users(self):
4281 4282 """ Returns the users which this rule applies to """
4282 4283
4283 4284 users = collections.OrderedDict()
4284 4285
4285 4286 for rule_user in self.rule_users:
4286 4287 if rule_user.user.active:
4287 4288 if rule_user.user not in users:
4288 4289 users[rule_user.user.username] = {
4289 4290 'user': rule_user.user,
4290 4291 'source': 'user',
4291 4292 'source_data': {},
4292 4293 'data': rule_user.rule_data()
4293 4294 }
4294 4295
4295 4296 for rule_user_group in self.rule_user_groups:
4296 4297 source_data = {
4297 4298 'user_group_id': rule_user_group.users_group.users_group_id,
4298 4299 'name': rule_user_group.users_group.users_group_name,
4299 4300 'members': len(rule_user_group.users_group.members)
4300 4301 }
4301 4302 for member in rule_user_group.users_group.members:
4302 4303 if member.user.active:
4303 4304 key = member.user.username
4304 4305 if key in users:
4305 4306 # skip this member as we have him already
4306 4307 # this prevents from override the "first" matched
4307 4308 # users with duplicates in multiple groups
4308 4309 continue
4309 4310
4310 4311 users[key] = {
4311 4312 'user': member.user,
4312 4313 'source': 'user_group',
4313 4314 'source_data': source_data,
4314 4315 'data': rule_user_group.rule_data()
4315 4316 }
4316 4317
4317 4318 return users
4318 4319
4319 4320 def user_group_vote_rule(self):
4320 4321 rules = []
4321 4322 if self.rule_user_groups:
4322 4323 for user_group in self.rule_user_groups:
4323 4324 rules.append(user_group)
4324 4325 return rules
4325 4326
4326 4327 def __repr__(self):
4327 4328 return '<RepoReviewerRule(id=%r, repo=%r)>' % (
4328 4329 self.repo_review_rule_id, self.repo)
4329 4330
4330 4331
4331 4332 class ScheduleEntry(Base, BaseModel):
4332 4333 __tablename__ = 'schedule_entries'
4333 4334 __table_args__ = (
4334 4335 UniqueConstraint('schedule_name', name='s_schedule_name_idx'),
4335 4336 UniqueConstraint('task_uid', name='s_task_uid_idx'),
4336 4337 base_table_args,
4337 4338 )
4338 4339
4339 4340 schedule_types = ['crontab', 'timedelta', 'integer']
4340 4341 schedule_entry_id = Column('schedule_entry_id', Integer(), primary_key=True)
4341 4342
4342 4343 schedule_name = Column("schedule_name", String(255), nullable=False, unique=None, default=None)
4343 4344 schedule_description = Column("schedule_description", String(10000), nullable=True, unique=None, default=None)
4344 4345 schedule_enabled = Column("schedule_enabled", Boolean(), nullable=False, unique=None, default=True)
4345 4346
4346 4347 _schedule_type = Column("schedule_type", String(255), nullable=False, unique=None, default=None)
4347 4348 schedule_definition = Column('schedule_definition_json', MutationObj.as_mutable(JsonType(default=lambda: "", dialect_map=dict(mysql=LONGTEXT()))))
4348 4349
4349 4350 schedule_last_run = Column('schedule_last_run', DateTime(timezone=False), nullable=True, unique=None, default=None)
4350 4351 schedule_total_run_count = Column('schedule_total_run_count', Integer(), nullable=True, unique=None, default=0)
4351 4352
4352 4353 # task
4353 4354 task_uid = Column("task_uid", String(255), nullable=False, unique=None, default=None)
4354 4355 task_dot_notation = Column("task_dot_notation", String(4096), nullable=False, unique=None, default=None)
4355 4356 task_args = Column('task_args_json', MutationObj.as_mutable(JsonType(default=list, dialect_map=dict(mysql=LONGTEXT()))))
4356 4357 task_kwargs = Column('task_kwargs_json', MutationObj.as_mutable(JsonType(default=dict, dialect_map=dict(mysql=LONGTEXT()))))
4357 4358
4358 4359 created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
4359 4360 updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=None)
4360 4361
4361 4362 @hybrid_property
4362 4363 def schedule_type(self):
4363 4364 return self._schedule_type
4364 4365
4365 4366 @schedule_type.setter
4366 4367 def schedule_type(self, val):
4367 4368 if val not in self.schedule_types:
4368 4369 raise ValueError('Value must be on of `{}` and got `{}`'.format(
4369 4370 val, self.schedule_type))
4370 4371
4371 4372 self._schedule_type = val
4372 4373
4373 4374 @classmethod
4374 4375 def get_uid(cls, obj):
4375 4376 args = obj.task_args
4376 4377 kwargs = obj.task_kwargs
4377 4378 if isinstance(args, JsonRaw):
4378 4379 try:
4379 4380 args = json.loads(args)
4380 4381 except ValueError:
4381 4382 args = tuple()
4382 4383
4383 4384 if isinstance(kwargs, JsonRaw):
4384 4385 try:
4385 4386 kwargs = json.loads(kwargs)
4386 4387 except ValueError:
4387 4388 kwargs = dict()
4388 4389
4389 4390 dot_notation = obj.task_dot_notation
4390 4391 val = '.'.join(map(safe_str, [
4391 4392 sorted(dot_notation), args, sorted(kwargs.items())]))
4392 4393 return hashlib.sha1(val).hexdigest()
4393 4394
4394 4395 @classmethod
4395 4396 def get_by_schedule_name(cls, schedule_name):
4396 4397 return cls.query().filter(cls.schedule_name == schedule_name).scalar()
4397 4398
4398 4399 @classmethod
4399 4400 def get_by_schedule_id(cls, schedule_id):
4400 4401 return cls.query().filter(cls.schedule_entry_id == schedule_id).scalar()
4401 4402
4402 4403 @property
4403 4404 def task(self):
4404 4405 return self.task_dot_notation
4405 4406
4406 4407 @property
4407 4408 def schedule(self):
4408 4409 from rhodecode.lib.celerylib.utils import raw_2_schedule
4409 4410 schedule = raw_2_schedule(self.schedule_definition, self.schedule_type)
4410 4411 return schedule
4411 4412
4412 4413 @property
4413 4414 def args(self):
4414 4415 try:
4415 4416 return list(self.task_args or [])
4416 4417 except ValueError:
4417 4418 return list()
4418 4419
4419 4420 @property
4420 4421 def kwargs(self):
4421 4422 try:
4422 4423 return dict(self.task_kwargs or {})
4423 4424 except ValueError:
4424 4425 return dict()
4425 4426
4426 4427 def _as_raw(self, val):
4427 4428 if hasattr(val, 'de_coerce'):
4428 4429 val = val.de_coerce()
4429 4430 if val:
4430 4431 val = json.dumps(val)
4431 4432
4432 4433 return val
4433 4434
4434 4435 @property
4435 4436 def schedule_definition_raw(self):
4436 4437 return self._as_raw(self.schedule_definition)
4437 4438
4438 4439 @property
4439 4440 def args_raw(self):
4440 4441 return self._as_raw(self.task_args)
4441 4442
4442 4443 @property
4443 4444 def kwargs_raw(self):
4444 4445 return self._as_raw(self.task_kwargs)
4445 4446
4446 4447 def __repr__(self):
4447 4448 return '<DB:ScheduleEntry({}:{})>'.format(
4448 4449 self.schedule_entry_id, self.schedule_name)
4449 4450
4450 4451
4451 4452 @event.listens_for(ScheduleEntry, 'before_update')
4452 4453 def update_task_uid(mapper, connection, target):
4453 4454 target.task_uid = ScheduleEntry.get_uid(target)
4454 4455
4455 4456
4456 4457 @event.listens_for(ScheduleEntry, 'before_insert')
4457 4458 def set_task_uid(mapper, connection, target):
4458 4459 target.task_uid = ScheduleEntry.get_uid(target)
4459 4460
4460 4461
4461 4462 class DbMigrateVersion(Base, BaseModel):
4462 4463 __tablename__ = 'db_migrate_version'
4463 4464 __table_args__ = (
4464 4465 base_table_args,
4465 4466 )
4466 4467
4467 4468 repository_id = Column('repository_id', String(250), primary_key=True)
4468 4469 repository_path = Column('repository_path', Text)
4469 4470 version = Column('version', Integer)
4470 4471
4471 4472 @classmethod
4472 4473 def set_version(cls, version):
4473 4474 """
4474 4475 Helper for forcing a different version, usually for debugging purposes via ishell.
4475 4476 """
4476 4477 ver = DbMigrateVersion.query().first()
4477 4478 ver.version = version
4478 4479 Session().commit()
4479 4480
4480 4481
4481 4482 class DbSession(Base, BaseModel):
4482 4483 __tablename__ = 'db_session'
4483 4484 __table_args__ = (
4484 4485 base_table_args,
4485 4486 )
4486 4487
4487 4488 def __repr__(self):
4488 4489 return '<DB:DbSession({})>'.format(self.id)
4489 4490
4490 4491 id = Column('id', Integer())
4491 4492 namespace = Column('namespace', String(255), primary_key=True)
4492 4493 accessed = Column('accessed', DateTime, nullable=False)
4493 4494 created = Column('created', DateTime, nullable=False)
4494 4495 data = Column('data', PickleType, nullable=False)
4495 4496
4496 4497
4497 4498 class BeakerCache(Base, BaseModel):
4498 4499 __tablename__ = 'beaker_cache'
4499 4500 __table_args__ = (
4500 4501 base_table_args,
4501 4502 )
4502 4503
4503 4504 def __repr__(self):
4504 4505 return '<DB:DbSession({})>'.format(self.id)
4505 4506
4506 4507 id = Column('id', Integer())
4507 4508 namespace = Column('namespace', String(255), primary_key=True)
4508 4509 accessed = Column('accessed', DateTime, nullable=False)
4509 4510 created = Column('created', DateTime, nullable=False)
4510 4511 data = Column('data', PickleType, nullable=False)
@@ -1,683 +1,687 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # Copyright (C) 2010-2018 RhodeCode GmbH
4 4 #
5 5 # This program is free software: you can redistribute it and/or modify
6 6 # it under the terms of the GNU Affero General Public License, version 3
7 7 # (only), as published by the Free Software Foundation.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU Affero General Public License
15 15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 16 #
17 17 # This program is dual-licensed. If you wish to learn more about the
18 18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 20
21 21
22 22 """
23 23 Package for testing various lib/helper functions in rhodecode
24 24 """
25 25
26 26 import datetime
27 27 import string
28 28 import mock
29 29 import pytest
30 30
31 31 from rhodecode.tests import no_newline_id_generator
32 32 from rhodecode.tests.utils import run_test_concurrently
33 33
34 34 from rhodecode.lib import rc_cache
35 35 from rhodecode.lib.helpers import InitialsGravatar
36 36 from rhodecode.lib.utils2 import AttributeDict
37 37
38 38 from rhodecode.model.db import Repository, CacheKey
39 39
40 40
41 41 def _urls_for_proto(proto):
42 42 return [
43 43 ('%s://127.0.0.1' % proto, ['%s://' % proto, '127.0.0.1'],
44 44 '%s://127.0.0.1' % proto),
45 45 ('%s://marcink@127.0.0.1' % proto, ['%s://' % proto, '127.0.0.1'],
46 46 '%s://127.0.0.1' % proto),
47 47 ('%s://marcink:pass@127.0.0.1' % proto, ['%s://' % proto, '127.0.0.1'],
48 48 '%s://127.0.0.1' % proto),
49 49 ('%s://127.0.0.1:8080' % proto, ['%s://' % proto, '127.0.0.1', '8080'],
50 50 '%s://127.0.0.1:8080' % proto),
51 51 ('%s://domain.org' % proto, ['%s://' % proto, 'domain.org'],
52 52 '%s://domain.org' % proto),
53 53 ('%s://user:pass@domain.org:8080' % proto,
54 54 ['%s://' % proto, 'domain.org', '8080'],
55 55 '%s://domain.org:8080' % proto),
56 56 ]
57 57
58 58 TEST_URLS = _urls_for_proto('http') + _urls_for_proto('https')
59 59
60 60
61 61 @pytest.mark.parametrize("test_url, expected, expected_creds", TEST_URLS)
62 62 def test_uri_filter(test_url, expected, expected_creds):
63 63 from rhodecode.lib.utils2 import uri_filter
64 64 assert uri_filter(test_url) == expected
65 65
66 66
67 67 @pytest.mark.parametrize("test_url, expected, expected_creds", TEST_URLS)
68 68 def test_credentials_filter(test_url, expected, expected_creds):
69 69 from rhodecode.lib.utils2 import credentials_filter
70 70 assert credentials_filter(test_url) == expected_creds
71 71
72 72
73 73 @pytest.mark.parametrize("str_bool, expected", [
74 74 ('t', True),
75 75 ('true', True),
76 76 ('y', True),
77 77 ('yes', True),
78 78 ('on', True),
79 79 ('1', True),
80 80 ('Y', True),
81 81 ('yeS', True),
82 82 ('Y', True),
83 83 ('TRUE', True),
84 84 ('T', True),
85 85 ('False', False),
86 86 ('F', False),
87 87 ('FALSE', False),
88 88 ('0', False),
89 89 ('-1', False),
90 90 ('', False)
91 91 ])
92 92 def test_str2bool(str_bool, expected):
93 93 from rhodecode.lib.utils2 import str2bool
94 94 assert str2bool(str_bool) == expected
95 95
96 96
97 97 @pytest.mark.parametrize("text, expected", reduce(lambda a1,a2:a1+a2, [
98 98 [
99 99 (pref+"", []),
100 100 (pref+"Hi there @marcink", ['marcink']),
101 101 (pref+"Hi there @marcink and @bob", ['bob', 'marcink']),
102 102 (pref+"Hi there @marcink\n", ['marcink']),
103 103 (pref+"Hi there @marcink and @bob\n", ['bob', 'marcink']),
104 104 (pref+"Hi there marcin@rhodecode.com", []),
105 105 (pref+"Hi there @john.malcovic and @bob\n", ['bob', 'john.malcovic']),
106 106 (pref+"This needs to be reviewed: (@marcink,@john)", ["john", "marcink"]),
107 107 (pref+"This needs to be reviewed: (@marcink, @john)", ["john", "marcink"]),
108 108 (pref+"This needs to be reviewed: [@marcink,@john]", ["john", "marcink"]),
109 109 (pref+"This needs to be reviewed: (@marcink @john)", ["john", "marcink"]),
110 110 (pref+"@john @mary, please review", ["john", "mary"]),
111 111 (pref+"@john,@mary, please review", ["john", "mary"]),
112 112 (pref+"Hej @123, @22john,@mary, please review", ['123', '22john', 'mary']),
113 113 (pref+"@first hi there @marcink here's my email marcin@email.com "
114 114 "@lukaszb check @one_more22 it pls @ ttwelve @D[] @one@two@three ", ['first', 'lukaszb', 'marcink', 'one', 'one_more22']),
115 115 (pref+"@MARCIN @maRCiN @2one_more22 @john please see this http://org.pl", ['2one_more22', 'john', 'MARCIN', 'maRCiN']),
116 116 (pref+"@marian.user just do it @marco-polo and next extract @marco_polo", ['marco-polo', 'marco_polo', 'marian.user']),
117 117 (pref+"user.dot hej ! not-needed maril@domain.org", []),
118 118 (pref+"\n@marcin", ['marcin']),
119 119 ]
120 120 for pref in ['', '\n', 'hi !', '\t', '\n\n']]), ids=no_newline_id_generator)
121 121 def test_mention_extractor(text, expected):
122 122 from rhodecode.lib.utils2 import extract_mentioned_users
123 123 got = extract_mentioned_users(text)
124 124 assert sorted(got, key=lambda x: x.lower()) == got
125 125 assert set(expected) == set(got)
126 126
127 127 @pytest.mark.parametrize("age_args, expected, kw", [
128 128 ({}, u'just now', {}),
129 129 ({'seconds': -1}, u'1 second ago', {}),
130 130 ({'seconds': -60 * 2}, u'2 minutes ago', {}),
131 131 ({'hours': -1}, u'1 hour ago', {}),
132 132 ({'hours': -24}, u'1 day ago', {}),
133 133 ({'hours': -24 * 5}, u'5 days ago', {}),
134 134 ({'months': -1}, u'1 month ago', {}),
135 135 ({'months': -1, 'days': -2}, u'1 month and 2 days ago', {}),
136 136 ({'years': -1, 'months': -1}, u'1 year and 1 month ago', {}),
137 137 ({}, u'just now', {'short_format': True}),
138 138 ({'seconds': -1}, u'1sec ago', {'short_format': True}),
139 139 ({'seconds': -60 * 2}, u'2min ago', {'short_format': True}),
140 140 ({'hours': -1}, u'1h ago', {'short_format': True}),
141 141 ({'hours': -24}, u'1d ago', {'short_format': True}),
142 142 ({'hours': -24 * 5}, u'5d ago', {'short_format': True}),
143 143 ({'months': -1}, u'1m ago', {'short_format': True}),
144 144 ({'months': -1, 'days': -2}, u'1m, 2d ago', {'short_format': True}),
145 145 ({'years': -1, 'months': -1}, u'1y, 1m ago', {'short_format': True}),
146 146 ])
147 147 def test_age(age_args, expected, kw, baseapp):
148 148 from rhodecode.lib.utils2 import age
149 149 from dateutil import relativedelta
150 150 n = datetime.datetime(year=2012, month=5, day=17)
151 151 delt = lambda *args, **kwargs: relativedelta.relativedelta(*args, **kwargs)
152 152
153 153 def translate(elem):
154 154 return elem.interpolate()
155 155
156 156 assert translate(age(n + delt(**age_args), now=n, **kw)) == expected
157 157
158 158
159 159 @pytest.mark.parametrize("age_args, expected, kw", [
160 160 ({}, u'just now', {}),
161 161 ({'seconds': 1}, u'in 1 second', {}),
162 162 ({'seconds': 60 * 2}, u'in 2 minutes', {}),
163 163 ({'hours': 1}, u'in 1 hour', {}),
164 164 ({'hours': 24}, u'in 1 day', {}),
165 165 ({'hours': 24 * 5}, u'in 5 days', {}),
166 166 ({'months': 1}, u'in 1 month', {}),
167 167 ({'months': 1, 'days': 1}, u'in 1 month and 1 day', {}),
168 168 ({'years': 1, 'months': 1}, u'in 1 year and 1 month', {}),
169 169 ({}, u'just now', {'short_format': True}),
170 170 ({'seconds': 1}, u'in 1sec', {'short_format': True}),
171 171 ({'seconds': 60 * 2}, u'in 2min', {'short_format': True}),
172 172 ({'hours': 1}, u'in 1h', {'short_format': True}),
173 173 ({'hours': 24}, u'in 1d', {'short_format': True}),
174 174 ({'hours': 24 * 5}, u'in 5d', {'short_format': True}),
175 175 ({'months': 1}, u'in 1m', {'short_format': True}),
176 176 ({'months': 1, 'days': 1}, u'in 1m, 1d', {'short_format': True}),
177 177 ({'years': 1, 'months': 1}, u'in 1y, 1m', {'short_format': True}),
178 178 ])
179 179 def test_age_in_future(age_args, expected, kw, baseapp):
180 180 from rhodecode.lib.utils2 import age
181 181 from dateutil import relativedelta
182 182 n = datetime.datetime(year=2012, month=5, day=17)
183 183 delt = lambda *args, **kwargs: relativedelta.relativedelta(*args, **kwargs)
184 184
185 185 def translate(elem):
186 186 return elem.interpolate()
187 187
188 188 assert translate(age(n + delt(**age_args), now=n, **kw)) == expected
189 189
190 190
191 191 @pytest.mark.parametrize("sample, expected_tags", [
192 192 # entry
193 193 ((
194 194 ""
195 195 ),
196 196 [
197 197
198 198 ]),
199 199 # entry
200 200 ((
201 201 "hello world [stale]"
202 202 ),
203 203 [
204 204 ('state', '[stale]'),
205 205 ]),
206 206 # entry
207 207 ((
208 208 "hello world [v2.0.0] [v1.0.0]"
209 209 ),
210 210 [
211 211 ('generic', '[v2.0.0]'),
212 212 ('generic', '[v1.0.0]'),
213 213 ]),
214 214 # entry
215 215 ((
216 216 "he[ll]o wo[rl]d"
217 217 ),
218 218 [
219 219 ('label', '[ll]'),
220 220 ('label', '[rl]'),
221 221 ]),
222 222 # entry
223 223 ((
224 224 "hello world [stale]\n[featured]\n[stale] [dead] [dev]"
225 225 ),
226 226 [
227 227 ('state', '[stale]'),
228 228 ('state', '[featured]'),
229 229 ('state', '[stale]'),
230 230 ('state', '[dead]'),
231 231 ('state', '[dev]'),
232 232 ]),
233 233 # entry
234 234 ((
235 235 "hello world \n\n [stale] \n [url =&gt; [name](http://rc.com)]"
236 236 ),
237 237 [
238 238 ('state', '[stale]'),
239 239 ('url', '[url =&gt; [name](http://rc.com)]'),
240 240 ]),
241 241 # entry
242 242 ((
243 243 "[url =&gt; [linkNameJS](javascript:alert(document.domain))]\n"
244 244 "[url =&gt; [linkNameHTTP](http://rhodecode.com)]\n"
245 245 "[url =&gt; [linkNameHTTPS](https://rhodecode.com)]\n"
246 246 "[url =&gt; [linkNamePath](/repo_group)]\n"
247 247 ),
248 248 [
249 249 ('generic', '[linkNameJS]'),
250 250 ('url', '[url =&gt; [linkNameHTTP](http://rhodecode.com)]'),
251 251 ('url', '[url =&gt; [linkNameHTTPS](https://rhodecode.com)]'),
252 252 ('url', '[url =&gt; [linkNamePath](/repo_group)]'),
253 253 ]),
254 254 # entry
255 255 ((
256 256 "hello pta[tag] gog [[]] [[] sda ero[or]d [me =&gt;>< sa]"
257 257 "[requires] [stale] [see<>=&gt;] [see =&gt; http://url.com]"
258 258 "[requires =&gt; url] [lang =&gt; python] [just a tag] "
259 259 "<html_tag first='abc' attr=\"my.url?attr=&another=\"></html_tag>"
260 260 "[,d] [ =&gt; ULR ] [obsolete] [desc]]"
261 261 ),
262 262 [
263 263 ('label', '[desc]'),
264 264 ('label', '[obsolete]'),
265 265 ('label', '[or]'),
266 266 ('label', '[requires]'),
267 267 ('label', '[tag]'),
268 268 ('state', '[stale]'),
269 269 ('lang', '[lang =&gt; python]'),
270 270 ('ref', '[requires =&gt; url]'),
271 271 ('see', '[see =&gt; http://url.com]'),
272 272
273 273 ]),
274 274
275 275 ], ids=no_newline_id_generator)
276 276 def test_metatag_extraction(sample, expected_tags):
277 277 from rhodecode.lib.helpers import extract_metatags
278 278 tags, value = extract_metatags(sample)
279 279 assert sorted(tags) == sorted(expected_tags)
280 280
281 281
282 282 @pytest.mark.parametrize("tag_data, expected_html", [
283 283
284 284 (('state', '[stable]'), '<div class="metatag" tag="state stable">stable</div>'),
285 285 (('state', '[stale]'), '<div class="metatag" tag="state stale">stale</div>'),
286 286 (('state', '[featured]'), '<div class="metatag" tag="state featured">featured</div>'),
287 287 (('state', '[dev]'), '<div class="metatag" tag="state dev">dev</div>'),
288 288 (('state', '[dead]'), '<div class="metatag" tag="state dead">dead</div>'),
289 289
290 290 (('label', '[personal]'), '<div class="metatag" tag="label">personal</div>'),
291 291 (('generic', '[v2.0.0]'), '<div class="metatag" tag="generic">v2.0.0</div>'),
292 292
293 293 (('lang', '[lang =&gt; JavaScript]'), '<div class="metatag" tag="lang">JavaScript</div>'),
294 294 (('lang', '[lang =&gt; C++]'), '<div class="metatag" tag="lang">C++</div>'),
295 295 (('lang', '[lang =&gt; C#]'), '<div class="metatag" tag="lang">C#</div>'),
296 296 (('lang', '[lang =&gt; Delphi/Object]'), '<div class="metatag" tag="lang">Delphi/Object</div>'),
297 297 (('lang', '[lang =&gt; Objective-C]'), '<div class="metatag" tag="lang">Objective-C</div>'),
298 298 (('lang', '[lang =&gt; .NET]'), '<div class="metatag" tag="lang">.NET</div>'),
299 299
300 300 (('license', '[license =&gt; BSD 3-clause]'), '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/BSD 3-clause">BSD 3-clause</a></div>'),
301 301 (('license', '[license =&gt; GPLv3]'), '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/GPLv3">GPLv3</a></div>'),
302 302 (('license', '[license =&gt; MIT]'), '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/MIT">MIT</a></div>'),
303 303 (('license', '[license =&gt; AGPLv3]'), '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/AGPLv3">AGPLv3</a></div>'),
304 304
305 305 (('ref', '[requires =&gt; RepoName]'), '<div class="metatag" tag="ref requires">requires: <a href="/RepoName">RepoName</a></div>'),
306 306 (('ref', '[recommends =&gt; GroupName]'), '<div class="metatag" tag="ref recommends">recommends: <a href="/GroupName">GroupName</a></div>'),
307 307 (('ref', '[conflicts =&gt; SomeName]'), '<div class="metatag" tag="ref conflicts">conflicts: <a href="/SomeName">SomeName</a></div>'),
308 308 (('ref', '[base =&gt; SomeName]'), '<div class="metatag" tag="ref base">base: <a href="/SomeName">SomeName</a></div>'),
309 309
310 310 (('see', '[see =&gt; http://rhodecode.com]'), '<div class="metatag" tag="see">see: http://rhodecode.com </div>'),
311 311
312 312 (('url', '[url =&gt; [linkName](https://rhodecode.com)]'), '<div class="metatag" tag="url"> <a href="https://rhodecode.com">linkName</a> </div>'),
313 313 (('url', '[url =&gt; [example link](https://rhodecode.com)]'), '<div class="metatag" tag="url"> <a href="https://rhodecode.com">example link</a> </div>'),
314 314 (('url', '[url =&gt; [v1.0.0](https://rhodecode.com)]'), '<div class="metatag" tag="url"> <a href="https://rhodecode.com">v1.0.0</a> </div>'),
315 315
316 316 ])
317 317 def test_metatags_stylize(tag_data, expected_html):
318 318 from rhodecode.lib.helpers import style_metatag
319 319 tag_type,value = tag_data
320 320 assert style_metatag(tag_type, value) == expected_html
321 321
322 322
323 323 @pytest.mark.parametrize("tmpl_url, email, expected", [
324 324 ('http://test.com/{email}', 'test@foo.com', 'http://test.com/test@foo.com'),
325 325
326 326 ('http://test.com/{md5email}', 'test@foo.com', 'http://test.com/3cb7232fcc48743000cb86d0d5022bd9'),
327 327 ('http://test.com/{md5email}', 'testΔ…Δ‡@foo.com', 'http://test.com/978debb907a3c55cd741872ab293ef30'),
328 328
329 329 ('http://testX.com/{md5email}?s={size}', 'test@foo.com', 'http://testX.com/3cb7232fcc48743000cb86d0d5022bd9?s=24'),
330 330 ('http://testX.com/{md5email}?s={size}', 'testΔ…Δ‡@foo.com', 'http://testX.com/978debb907a3c55cd741872ab293ef30?s=24'),
331 331
332 332 ('{scheme}://{netloc}/{md5email}/{size}', 'test@foo.com', 'https://server.com/3cb7232fcc48743000cb86d0d5022bd9/24'),
333 333 ('{scheme}://{netloc}/{md5email}/{size}', 'testΔ…Δ‡@foo.com', 'https://server.com/978debb907a3c55cd741872ab293ef30/24'),
334 334
335 335 ('http://test.com/{email}', 'testΔ…Δ‡@foo.com', 'http://test.com/testΔ…Δ‡@foo.com'),
336 336 ('http://test.com/{email}?size={size}', 'test@foo.com', 'http://test.com/test@foo.com?size=24'),
337 337 ('http://test.com/{email}?size={size}', 'testΔ…Δ‡@foo.com', 'http://test.com/testΔ…Δ‡@foo.com?size=24'),
338 338 ])
339 339 def test_gravatar_url_builder(tmpl_url, email, expected, request_stub):
340 340 from rhodecode.lib.helpers import gravatar_url
341 341
342 342 def fake_tmpl_context(_url):
343 343 _c = AttributeDict()
344 344 _c.visual = AttributeDict()
345 345 _c.visual.use_gravatar = True
346 346 _c.visual.gravatar_url = _url
347 347 return _c
348 348
349 349 # mock pyramid.threadlocals
350 350 def fake_get_current_request():
351 351 request_stub.scheme = 'https'
352 352 request_stub.host = 'server.com'
353 353
354 354 request_stub._call_context = fake_tmpl_context(tmpl_url)
355 355 return request_stub
356 356
357 357 with mock.patch('rhodecode.lib.helpers.get_current_request',
358 358 fake_get_current_request):
359 359
360 360 grav = gravatar_url(email_address=email, size=24)
361 361 assert grav == expected
362 362
363 363
364 364 @pytest.mark.parametrize(
365 365 "email, first_name, last_name, expected_initials, expected_color", [
366 366
367 367 ('test@rhodecode.com', '', '', 'TR', '#8a994d'),
368 368 ('marcin.kuzminski@rhodecode.com', '', '', 'MK', '#6559b3'),
369 369 # special cases of email
370 370 ('john.van.dam@rhodecode.com', '', '', 'JD', '#526600'),
371 371 ('Guido.van.Rossum@rhodecode.com', '', '', 'GR', '#990052'),
372 372 ('Guido.van.Rossum@rhodecode.com', 'Guido', 'Van Rossum', 'GR', '#990052'),
373 373
374 374 ('rhodecode+Guido.van.Rossum@rhodecode.com', '', '', 'RR', '#46598c'),
375 375 ('pclouds@rhodecode.com', 'Nguyα»…n ThΓ‘i', 'Tgọc Duy', 'ND', '#665200'),
376 376
377 377 ('john-brown@foo.com', '', '', 'JF', '#73006b'),
378 378 ('admin@rhodecode.com', 'Marcin', 'Kuzminski', 'MK', '#104036'),
379 379 # partials
380 380 ('admin@rhodecode.com', 'Marcin', '', 'MR', '#104036'), # fn+email
381 381 ('admin@rhodecode.com', '', 'Kuzminski', 'AK', '#104036'), # em+ln
382 382 # non-ascii
383 383 ('admin@rhodecode.com', 'Marcin', 'Śuzminski', 'MS', '#104036'),
384 384 ('marcin.Ε›uzminski@rhodecode.com', '', '', 'MS', '#73000f'),
385 385
386 386 # special cases, LDAP can provide those...
387 387 ('admin@', 'Marcin', 'Śuzminski', 'MS', '#aa00ff'),
388 388 ('marcin.Ε›uzminski', '', '', 'MS', '#402020'),
389 389 ('null', '', '', 'NL', '#8c4646'),
390 390 ('some.@abc.com', 'some', '', 'SA', '#664e33')
391 391 ])
392 392 def test_initials_gravatar_pick_of_initials_and_color_algo(
393 393 email, first_name, last_name, expected_initials, expected_color):
394 394 instance = InitialsGravatar(email, first_name, last_name)
395 395 assert instance.get_initials() == expected_initials
396 396 assert instance.str2color(email) == expected_color
397 397
398 398
399 399 def test_initials_gravatar_mapping_algo():
400 400 pos = set()
401 401 instance = InitialsGravatar('', '', '')
402 402 iterations = 0
403 403
404 404 variations = []
405 405 for letter1 in string.ascii_letters:
406 406 for letter2 in string.ascii_letters[::-1][:10]:
407 407 for letter3 in string.ascii_letters[:10]:
408 408 variations.append(
409 409 '%s@rhodecode.com' % (letter1+letter2+letter3))
410 410
411 411 max_variations = 4096
412 412 for email in variations[:max_variations]:
413 413 iterations += 1
414 414 pos.add(
415 415 instance.pick_color_bank_index(email,
416 416 instance.get_color_bank()))
417 417
418 418 # we assume that we have match all 256 possible positions,
419 419 # in reasonable amount of different email addresses
420 420 assert len(pos) == 256
421 421 assert iterations == max_variations
422 422
423 423
424 424 @pytest.mark.parametrize("tmpl, repo_name, overrides, prefix, expected", [
425 425 (Repository.DEFAULT_CLONE_URI, 'group/repo1', {}, '', 'http://vps1:8000/group/repo1'),
426 426 (Repository.DEFAULT_CLONE_URI, 'group/repo1', {'user': 'marcink'}, '', 'http://marcink@vps1:8000/group/repo1'),
427 427 (Repository.DEFAULT_CLONE_URI, 'group/repo1', {}, '/rc', 'http://vps1:8000/rc/group/repo1'),
428 428 (Repository.DEFAULT_CLONE_URI, 'group/repo1', {'user': 'user'}, '/rc', 'http://user@vps1:8000/rc/group/repo1'),
429 429 (Repository.DEFAULT_CLONE_URI, 'group/repo1', {'user': 'marcink'}, '/rc', 'http://marcink@vps1:8000/rc/group/repo1'),
430 430 (Repository.DEFAULT_CLONE_URI, 'group/repo1', {'user': 'user'}, '/rc/', 'http://user@vps1:8000/rc/group/repo1'),
431 431 (Repository.DEFAULT_CLONE_URI, 'group/repo1', {'user': 'marcink'}, '/rc/', 'http://marcink@vps1:8000/rc/group/repo1'),
432 432 ('{scheme}://{user}@{netloc}/_{repoid}', 'group/repo1', {}, '', 'http://vps1:8000/_23'),
433 433 ('{scheme}://{user}@{netloc}/_{repoid}', 'group/repo1', {'user': 'marcink'}, '', 'http://marcink@vps1:8000/_23'),
434 434 ('http://{user}@{netloc}/_{repoid}', 'group/repo1', {'user': 'marcink'}, '', 'http://marcink@vps1:8000/_23'),
435 435 ('http://{netloc}/_{repoid}', 'group/repo1', {'user': 'marcink'}, '', 'http://vps1:8000/_23'),
436 436 ('https://{user}@proxy1.server.com/{repo}', 'group/repo1', {'user': 'marcink'}, '', 'https://marcink@proxy1.server.com/group/repo1'),
437 437 ('https://{user}@proxy1.server.com/{repo}', 'group/repo1', {}, '', 'https://proxy1.server.com/group/repo1'),
438 438 ('https://proxy1.server.com/{user}/{repo}', 'group/repo1', {'user': 'marcink'}, '', 'https://proxy1.server.com/marcink/group/repo1'),
439 439 ])
440 440 def test_clone_url_generator(tmpl, repo_name, overrides, prefix, expected):
441 441 from rhodecode.lib.utils2 import get_clone_url
442 442
443 443 class RequestStub(object):
444 444 def request_url(self, name):
445 445 return 'http://vps1:8000' + prefix
446 446
447 447 def route_url(self, name):
448 448 return self.request_url(name)
449 449
450 450 clone_url = get_clone_url(
451 451 request=RequestStub(),
452 452 uri_tmpl=tmpl,
453 453 repo_name=repo_name, repo_id=23, **overrides)
454 454 assert clone_url == expected
455 455
456 456
457 457 def _quick_url(text, tmpl="""<a class="revision-link" href="%s">%s</a>""", url_=None):
458 458 """
459 459 Changes `some text url[foo]` => `some text <a href="/">foo</a>
460 460
461 461 :param text:
462 462 """
463 463 import re
464 464 # quickly change expected url[] into a link
465 465 URL_PAT = re.compile(r'(?:url\[)(.+?)(?:\])')
466 466
467 467 def url_func(match_obj):
468 468 _url = match_obj.groups()[0]
469 469 return tmpl % (url_ or '/some-url', _url)
470 470 return URL_PAT.sub(url_func, text)
471 471
472 472
473 473 @pytest.mark.parametrize("sample, expected", [
474 474 ("",
475 475 ""),
476 476 ("git-svn-id: https://svn.apache.org/repos/asf/libcloud/trunk@1441655 13f79535-47bb-0310-9956-ffa450edef68",
477 477 "git-svn-id: https://svn.apache.org/repos/asf/libcloud/trunk@1441655 13f79535-47bb-0310-9956-ffa450edef68"),
478 478 ("from rev 000000000000",
479 479 "from rev url[000000000000]"),
480 480 ("from rev 000000000000123123 also rev 000000000000",
481 481 "from rev url[000000000000123123] also rev url[000000000000]"),
482 482 ("this should-000 00",
483 483 "this should-000 00"),
484 484 ("longtextffffffffff rev 123123123123",
485 485 "longtextffffffffff rev url[123123123123]"),
486 486 ("rev ffffffffffffffffffffffffffffffffffffffffffffffffff",
487 487 "rev ffffffffffffffffffffffffffffffffffffffffffffffffff"),
488 488 ("ffffffffffff some text traalaa",
489 489 "url[ffffffffffff] some text traalaa"),
490 490 ("""Multi line
491 491 123123123123
492 492 some text 123123123123
493 493 sometimes !
494 494 """,
495 495 """Multi line
496 496 url[123123123123]
497 497 some text url[123123123123]
498 498 sometimes !
499 499 """)
500 500 ], ids=no_newline_id_generator)
501 501 def test_urlify_commits(sample, expected):
502 502 def fake_url(self, *args, **kwargs):
503 503 return '/some-url'
504 504
505 505 expected = _quick_url(expected)
506 506
507 507 with mock.patch('rhodecode.lib.helpers.route_url', fake_url):
508 508 from rhodecode.lib.helpers import urlify_commits
509 509 assert urlify_commits(sample, 'repo_name') == expected
510 510
511 511
512 512 @pytest.mark.parametrize("sample, expected, url_", [
513 513 ("",
514 514 "",
515 515 ""),
516 516 ("https://svn.apache.org/repos",
517 517 "url[https://svn.apache.org/repos]",
518 518 "https://svn.apache.org/repos"),
519 519 ("http://svn.apache.org/repos",
520 520 "url[http://svn.apache.org/repos]",
521 521 "http://svn.apache.org/repos"),
522 522 ("from rev a also rev http://google.com",
523 523 "from rev a also rev url[http://google.com]",
524 524 "http://google.com"),
525 525 ("""Multi line
526 526 https://foo.bar.com
527 527 some text lalala""",
528 528 """Multi line
529 529 url[https://foo.bar.com]
530 530 some text lalala""",
531 531 "https://foo.bar.com")
532 532 ], ids=no_newline_id_generator)
533 533 def test_urlify_test(sample, expected, url_):
534 534 from rhodecode.lib.helpers import urlify_text
535 535 expected = _quick_url(expected, tmpl="""<a href="%s">%s</a>""", url_=url_)
536 536 assert urlify_text(sample) == expected
537 537
538 538
539 539 @pytest.mark.parametrize("test, expected", [
540 540 ("", None),
541 541 ("/_2", '2'),
542 542 ("_2", '2'),
543 543 ("/_2/", '2'),
544 544 ("_2/", '2'),
545 545
546 546 ("/_21", '21'),
547 547 ("_21", '21'),
548 548 ("/_21/", '21'),
549 549 ("_21/", '21'),
550 550
551 551 ("/_21/foobar", '21'),
552 552 ("_21/121", '21'),
553 553 ("/_21/_12", '21'),
554 554 ("_21/rc/foo", '21'),
555 555
556 556 ])
557 557 def test_get_repo_by_id(test, expected):
558 558 from rhodecode.model.repo import RepoModel
559 559 _test = RepoModel()._extract_id_from_repo_name(test)
560 560 assert _test == expected
561 561
562 562
563 563 def test_invalidation_context(baseapp):
564 564 repo_id = 999
565 565
566 566 cache_namespace_uid = 'cache_repo_instance.{}_{}'.format(
567 567 repo_id, CacheKey.CACHE_TYPE_README)
568 568 invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
569 569 repo_id=repo_id)
570 570 region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid)
571 571
572 572 calls = [1, 2]
573 573
574 574 @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
575 575 def _dummy_func(cache_key):
576 576 val = calls.pop(0)
577 577 return 'result:{}'.format(val)
578 578
579 579 inv_context_manager = rc_cache.InvalidationContext(
580 580 uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
581 581
582 582 # 1st call, fresh caches
583 583 with inv_context_manager as invalidation_context:
584 584 should_invalidate = invalidation_context.should_invalidate()
585 585 if should_invalidate:
586 _dummy_func.invalidate('some-key')
586 result = _dummy_func.refresh('some-key')
587 else:
587 588 result = _dummy_func('some-key')
588 589
589 590 assert isinstance(invalidation_context, rc_cache.FreshRegionCache)
590 591 assert should_invalidate is True
591 592
592 593 assert 'result:1' == result
593 594 # should be cached so calling it twice will give the same result !
594 595 result = _dummy_func('some-key')
595 596 assert 'result:1' == result
596 597
597 598 # 2nd call, we create a new context manager, this should be now key aware, and
598 599 # return an active cache region
599 600 with inv_context_manager as invalidation_context:
600 601 should_invalidate = invalidation_context.should_invalidate()
601 602 assert isinstance(invalidation_context, rc_cache.ActiveRegionCache)
602 603 assert should_invalidate is False
603 604
604 605 # Mark invalidation
605 606 CacheKey.set_invalidate(invalidation_namespace)
606 607
607 608 # 3nd call, fresh caches
608 609 with inv_context_manager as invalidation_context:
609 610 should_invalidate = invalidation_context.should_invalidate()
610 611 if should_invalidate:
611 _dummy_func.invalidate('some-key')
612 result = _dummy_func.refresh('some-key')
613 else:
612 614 result = _dummy_func('some-key')
613 615
614 616 assert isinstance(invalidation_context, rc_cache.FreshRegionCache)
615 617 assert should_invalidate is True
616 618
617 619 assert 'result:2' == result
618 620
619 621 # cached again, same result
620 622 result = _dummy_func('some-key')
621 623 assert 'result:2' == result
622 624
623 625
624 626 def test_invalidation_context_exception_in_compute(baseapp):
625 627 repo_id = 888
626 628
627 629 cache_namespace_uid = 'cache_repo_instance.{}_{}'.format(
628 630 repo_id, CacheKey.CACHE_TYPE_README)
629 631 invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
630 632 repo_id=repo_id)
631 633 region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid)
632 634
633 635 @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
634 636 def _dummy_func(cache_key):
635 637 raise Exception('Error in cache func')
636 638
637 639 with pytest.raises(Exception):
638 640 inv_context_manager = rc_cache.InvalidationContext(
639 641 uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
640 642
641 643 # 1st call, fresh caches
642 644 with inv_context_manager as invalidation_context:
643 645 should_invalidate = invalidation_context.should_invalidate()
644 646 if should_invalidate:
645 _dummy_func.invalidate('some-key-2')
647 _dummy_func.refresh('some-key-2')
648 else:
646 649 _dummy_func('some-key-2')
647 650
648 651
649 652 @pytest.mark.parametrize('execution_number', range(5))
650 653 def test_cache_invalidation_race_condition(execution_number, baseapp):
651 654 import time
652 655
653 656 repo_id = 777
654 657
655 658 cache_namespace_uid = 'cache_repo_instance.{}_{}'.format(
656 659 repo_id, CacheKey.CACHE_TYPE_README)
657 660 invalidation_namespace = CacheKey.REPO_INVALIDATION_NAMESPACE.format(
658 661 repo_id=repo_id)
659 662 region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid)
660 663
661 664 @run_test_concurrently(25)
662 665 def test_create_and_delete_cache_keys():
663 666 time.sleep(0.2)
664 667
665 668 @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
666 669 def _dummy_func(cache_key):
667 670 val = 'async'
668 671 return 'result:{}'.format(val)
669 672
670 673 inv_context_manager = rc_cache.InvalidationContext(
671 674 uid=cache_namespace_uid, invalidation_namespace=invalidation_namespace)
672 675
673 676 # 1st call, fresh caches
674 677 with inv_context_manager as invalidation_context:
675 678 should_invalidate = invalidation_context.should_invalidate()
676 679 if should_invalidate:
677 _dummy_func.invalidate('some-key-3')
680 _dummy_func.refresh('some-key-3')
681 else:
678 682 _dummy_func('some-key-3')
679 683
680 684 # Mark invalidation
681 685 CacheKey.set_invalidate(invalidation_namespace)
682 686
683 687 test_create_and_delete_cache_keys()
General Comments 0
You need to be logged in to leave comments. Login now