##// END OF EJS Templates
fix for issue #578 git hooks sometimes cannot be executed due to different python they runned under, this commit tries to fix that by altering the PATH env variable using current python that rhodecode is running
marcink -
r2869:ccbdff90 beta
parent child Browse files
Show More
@@ -1,337 +1,338 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 3 rhodecode.lib.middleware.simplegit
4 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5 5
6 6 SimpleGit middleware for handling git protocol request (push/clone etc.)
7 7 It's implemented with basic auth function
8 8
9 9 :created_on: Apr 28, 2010
10 10 :author: marcink
11 11 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
12 12 :license: GPLv3, see COPYING for more details.
13 13 """
14 14 # This program is free software: you can redistribute it and/or modify
15 15 # it under the terms of the GNU General Public License as published by
16 16 # the Free Software Foundation, either version 3 of the License, or
17 17 # (at your option) any later version.
18 18 #
19 19 # This program is distributed in the hope that it will be useful,
20 20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
21 21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 22 # GNU General Public License for more details.
23 23 #
24 24 # You should have received a copy of the GNU General Public License
25 25 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26 26
27 27 import os
28 28 import re
29 29 import logging
30 30 import traceback
31 31
32 32 from dulwich import server as dulserver
33 33 from dulwich.web import LimitedInputFilter, GunzipFilter
34 34 from rhodecode.lib.exceptions import HTTPLockedRC
35 35 from rhodecode.lib.hooks import pre_pull
36 36
37 37
38 38 class SimpleGitUploadPackHandler(dulserver.UploadPackHandler):
39 39
40 40 def handle(self):
41 41 write = lambda x: self.proto.write_sideband(1, x)
42 42
43 43 graph_walker = dulserver.ProtocolGraphWalker(self,
44 44 self.repo.object_store,
45 45 self.repo.get_peeled)
46 46 objects_iter = self.repo.fetch_objects(
47 47 graph_walker.determine_wants, graph_walker, self.progress,
48 48 get_tagged=self.get_tagged)
49 49
50 50 # Did the process short-circuit (e.g. in a stateless RPC call)? Note
51 51 # that the client still expects a 0-object pack in most cases.
52 52 if objects_iter is None:
53 53 return
54 54
55 55 self.progress("counting objects: %d, done.\n" % len(objects_iter))
56 56 dulserver.write_pack_objects(dulserver.ProtocolFile(None, write),
57 57 objects_iter)
58 58 messages = []
59 59 messages.append('thank you for using rhodecode')
60 60
61 61 for msg in messages:
62 62 self.progress(msg + "\n")
63 63 # we are done
64 64 self.proto.write("0000")
65 65
66 66
67 67 dulserver.DEFAULT_HANDLERS = {
68 68 #git-ls-remote, git-clone, git-fetch and git-pull
69 69 'git-upload-pack': SimpleGitUploadPackHandler,
70 70 #git-push
71 71 'git-receive-pack': dulserver.ReceivePackHandler,
72 72 }
73 73
74 74 # not used for now until dulwich get's fixed
75 75 #from dulwich.repo import Repo
76 76 #from dulwich.web import make_wsgi_chain
77 77
78 78 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
79 79 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
80 80 HTTPBadRequest, HTTPNotAcceptable
81 81
82 from rhodecode.lib.utils2 import safe_str
82 from rhodecode.lib.utils2 import safe_str, fix_PATH
83 83 from rhodecode.lib.base import BaseVCSController
84 84 from rhodecode.lib.auth import get_container_username
85 85 from rhodecode.lib.utils import is_valid_repo, make_ui
86 86 from rhodecode.lib.compat import json
87 87 from rhodecode.model.db import User, RhodeCodeUi
88 88
89 89 log = logging.getLogger(__name__)
90 90
91 91
92 92 GIT_PROTO_PAT = re.compile(r'^/(.+)/(info/refs|git-upload-pack|git-receive-pack)')
93 93
94 94
95 95 def is_git(environ):
96 96 path_info = environ['PATH_INFO']
97 97 isgit_path = GIT_PROTO_PAT.match(path_info)
98 98 log.debug('pathinfo: %s detected as GIT %s' % (
99 99 path_info, isgit_path != None)
100 100 )
101 101 return isgit_path
102 102
103 103
104 104 class SimpleGit(BaseVCSController):
105 105
106 106 def _handle_request(self, environ, start_response):
107 107 if not is_git(environ):
108 108 return self.application(environ, start_response)
109 109 if not self._check_ssl(environ, start_response):
110 110 return HTTPNotAcceptable('SSL REQUIRED !')(environ, start_response)
111 111
112 112 ipaddr = self._get_ip_addr(environ)
113 113 username = None
114 114 self._git_first_op = False
115 115 # skip passing error to error controller
116 116 environ['pylons.status_code_redirect'] = True
117 117
118 118 #======================================================================
119 119 # EXTRACT REPOSITORY NAME FROM ENV
120 120 #======================================================================
121 121 try:
122 122 repo_name = self.__get_repository(environ)
123 123 log.debug('Extracted repo name is %s' % repo_name)
124 124 except:
125 125 return HTTPInternalServerError()(environ, start_response)
126 126
127 127 # quick check if that dir exists...
128 128 if is_valid_repo(repo_name, self.basepath, 'git') is False:
129 129 return HTTPNotFound()(environ, start_response)
130 130
131 131 #======================================================================
132 132 # GET ACTION PULL or PUSH
133 133 #======================================================================
134 134 action = self.__get_action(environ)
135 135
136 136 #======================================================================
137 137 # CHECK ANONYMOUS PERMISSION
138 138 #======================================================================
139 139 if action in ['pull', 'push']:
140 140 anonymous_user = self.__get_user('default')
141 141 username = anonymous_user.username
142 142 anonymous_perm = self._check_permission(action, anonymous_user,
143 143 repo_name)
144 144
145 145 if anonymous_perm is not True or anonymous_user.active is False:
146 146 if anonymous_perm is not True:
147 147 log.debug('Not enough credentials to access this '
148 148 'repository as anonymous user')
149 149 if anonymous_user.active is False:
150 150 log.debug('Anonymous access is disabled, running '
151 151 'authentication')
152 152 #==============================================================
153 153 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
154 154 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
155 155 #==============================================================
156 156
157 157 # Attempting to retrieve username from the container
158 158 username = get_container_username(environ, self.config)
159 159
160 160 # If not authenticated by the container, running basic auth
161 161 if not username:
162 162 self.authenticate.realm = \
163 163 safe_str(self.config['rhodecode_realm'])
164 164 result = self.authenticate(environ)
165 165 if isinstance(result, str):
166 166 AUTH_TYPE.update(environ, 'basic')
167 167 REMOTE_USER.update(environ, result)
168 168 username = result
169 169 else:
170 170 return result.wsgi_application(environ, start_response)
171 171
172 172 #==============================================================
173 173 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
174 174 #==============================================================
175 175 try:
176 176 user = self.__get_user(username)
177 177 if user is None or not user.active:
178 178 return HTTPForbidden()(environ, start_response)
179 179 username = user.username
180 180 except:
181 181 log.error(traceback.format_exc())
182 182 return HTTPInternalServerError()(environ, start_response)
183 183
184 184 #check permissions for this repository
185 185 perm = self._check_permission(action, user, repo_name)
186 186 if perm is not True:
187 187 return HTTPForbidden()(environ, start_response)
188 188
189 189 # extras are injected into UI object and later available
190 190 # in hooks executed by rhodecode
191 191 from rhodecode import CONFIG
192 192 extras = {
193 193 'ip': ipaddr,
194 194 'username': username,
195 195 'action': action,
196 196 'repository': repo_name,
197 197 'scm': 'git',
198 198 'config': CONFIG['__file__'],
199 199 'make_lock': None,
200 200 'locked_by': [None, None]
201 201 }
202 202
203 203 #===================================================================
204 204 # GIT REQUEST HANDLING
205 205 #===================================================================
206 206 repo_path = os.path.join(safe_str(self.basepath), safe_str(repo_name))
207 207 log.debug('Repository path is %s' % repo_path)
208 208
209 209 # CHECK LOCKING only if it's not ANONYMOUS USER
210 210 if username != User.DEFAULT_USER:
211 211 log.debug('Checking locking on repository')
212 212 (make_lock,
213 213 locked,
214 214 locked_by) = self._check_locking_state(
215 215 environ=environ, action=action,
216 216 repo=repo_name, user_id=user.user_id
217 217 )
218 218 # store the make_lock for later evaluation in hooks
219 219 extras.update({'make_lock': make_lock,
220 220 'locked_by': locked_by})
221 221 # set the environ variables for this request
222 222 os.environ['RC_SCM_DATA'] = json.dumps(extras)
223 fix_PATH()
223 224 log.debug('HOOKS extras is %s' % extras)
224 225 baseui = make_ui('db')
225 226 self.__inject_extras(repo_path, baseui, extras)
226 227
227 228 try:
228 229 # invalidate cache on push
229 230 if action == 'push':
230 231 self._invalidate_cache(repo_name)
231 232 self._handle_githooks(repo_name, action, baseui, environ)
232 233
233 234 log.info('%s action on GIT repo "%s"' % (action, repo_name))
234 235 app = self.__make_app(repo_name, repo_path, extras)
235 236 return app(environ, start_response)
236 237 except HTTPLockedRC, e:
237 238 log.debug('Repositry LOCKED ret code 423!')
238 239 return e(environ, start_response)
239 240 except Exception:
240 241 log.error(traceback.format_exc())
241 242 return HTTPInternalServerError()(environ, start_response)
242 243
243 244 def __make_app(self, repo_name, repo_path, extras):
244 245 """
245 246 Make an wsgi application using dulserver
246 247
247 248 :param repo_name: name of the repository
248 249 :param repo_path: full path to the repository
249 250 """
250 251
251 252 from rhodecode.lib.middleware.pygrack import make_wsgi_app
252 253 app = make_wsgi_app(
253 254 repo_root=safe_str(self.basepath),
254 255 repo_name=repo_name,
255 256 extras=extras,
256 257 )
257 258 app = GunzipFilter(LimitedInputFilter(app))
258 259 return app
259 260
260 261 def __get_repository(self, environ):
261 262 """
262 263 Get's repository name out of PATH_INFO header
263 264
264 265 :param environ: environ where PATH_INFO is stored
265 266 """
266 267 try:
267 268 environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
268 269 repo_name = GIT_PROTO_PAT.match(environ['PATH_INFO']).group(1)
269 270 except:
270 271 log.error(traceback.format_exc())
271 272 raise
272 273
273 274 return repo_name
274 275
275 276 def __get_user(self, username):
276 277 return User.get_by_username(username)
277 278
278 279 def __get_action(self, environ):
279 280 """
280 281 Maps git request commands into a pull or push command.
281 282
282 283 :param environ:
283 284 """
284 285 service = environ['QUERY_STRING'].split('=')
285 286
286 287 if len(service) > 1:
287 288 service_cmd = service[1]
288 289 mapping = {
289 290 'git-receive-pack': 'push',
290 291 'git-upload-pack': 'pull',
291 292 }
292 293 op = mapping[service_cmd]
293 294 self._git_stored_op = op
294 295 return op
295 296 else:
296 297 # try to fallback to stored variable as we don't know if the last
297 298 # operation is pull/push
298 299 op = getattr(self, '_git_stored_op', 'pull')
299 300 return op
300 301
301 302 def _handle_githooks(self, repo_name, action, baseui, environ):
302 303 """
303 304 Handles pull action, push is handled by post-receive hook
304 305 """
305 306 from rhodecode.lib.hooks import log_pull_action
306 307 service = environ['QUERY_STRING'].split('=')
307 308
308 309 if len(service) < 2:
309 310 return
310 311
311 312 from rhodecode.model.db import Repository
312 313 _repo = Repository.get_by_repo_name(repo_name)
313 314 _repo = _repo.scm_instance
314 315 _repo._repo.ui = baseui
315 316
316 317 _hooks = dict(baseui.configitems('hooks')) or {}
317 318 if action == 'pull':
318 319 # stupid git, emulate pre-pull hook !
319 320 pre_pull(ui=baseui, repo=_repo._repo)
320 321 if action == 'pull' and _hooks.get(RhodeCodeUi.HOOK_PULL):
321 322 log_pull_action(ui=baseui, repo=_repo._repo)
322 323
323 324 def __inject_extras(self, repo_path, baseui, extras={}):
324 325 """
325 326 Injects some extra params into baseui instance
326 327
327 328 :param baseui: baseui instance
328 329 :param extras: dict with extra params to put into baseui
329 330 """
330 331
331 332 # make our hgweb quiet so it doesn't print output
332 333 baseui.setconfig('ui', 'quiet', 'true')
333 334
334 335 #inject some additional parameters that will be available in ui
335 336 #for hooks
336 337 for k, v in extras.items():
337 338 baseui.setconfig('rhodecode_extras', k, v)
@@ -1,285 +1,285 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 3 rhodecode.lib.middleware.simplehg
4 4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5 5
6 6 SimpleHG middleware for handling mercurial protocol request
7 7 (push/clone etc.). It's implemented with basic auth function
8 8
9 9 :created_on: Apr 28, 2010
10 10 :author: marcink
11 11 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
12 12 :license: GPLv3, see COPYING for more details.
13 13 """
14 14 # This program is free software: you can redistribute it and/or modify
15 15 # it under the terms of the GNU General Public License as published by
16 16 # the Free Software Foundation, either version 3 of the License, or
17 17 # (at your option) any later version.
18 18 #
19 19 # This program is distributed in the hope that it will be useful,
20 20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
21 21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 22 # GNU General Public License for more details.
23 23 #
24 24 # You should have received a copy of the GNU General Public License
25 25 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26 26
27 27 import os
28 28 import logging
29 29 import traceback
30 import urllib
31 30
32 31 from mercurial.error import RepoError
33 32 from mercurial.hgweb import hgweb_mod
34 33
35 34 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
36 35 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
37 36 HTTPBadRequest, HTTPNotAcceptable
38 37
39 from rhodecode.lib.utils2 import safe_str
38 from rhodecode.lib.utils2 import safe_str, fix_PATH
40 39 from rhodecode.lib.base import BaseVCSController
41 40 from rhodecode.lib.auth import get_container_username
42 41 from rhodecode.lib.utils import make_ui, is_valid_repo, ui_sections
43 42 from rhodecode.lib.compat import json
44 43 from rhodecode.model.db import User
45 44 from rhodecode.lib.exceptions import HTTPLockedRC
46 45
47 46
48 47 log = logging.getLogger(__name__)
49 48
50 49
51 50 def is_mercurial(environ):
52 51 """
53 52 Returns True if request's target is mercurial server - header
54 53 ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
55 54 """
56 55 http_accept = environ.get('HTTP_ACCEPT')
57 56 path_info = environ['PATH_INFO']
58 57 if http_accept and http_accept.startswith('application/mercurial'):
59 58 ishg_path = True
60 59 else:
61 60 ishg_path = False
62 61
63 62 log.debug('pathinfo: %s detected as HG %s' % (
64 63 path_info, ishg_path)
65 64 )
66 65 return ishg_path
67 66
68 67
69 68 class SimpleHg(BaseVCSController):
70 69
71 70 def _handle_request(self, environ, start_response):
72 71 if not is_mercurial(environ):
73 72 return self.application(environ, start_response)
74 73 if not self._check_ssl(environ, start_response):
75 74 return HTTPNotAcceptable('SSL REQUIRED !')(environ, start_response)
76 75
77 76 ipaddr = self._get_ip_addr(environ)
78 77 username = None
79 78 # skip passing error to error controller
80 79 environ['pylons.status_code_redirect'] = True
81 80
82 81 #======================================================================
83 82 # EXTRACT REPOSITORY NAME FROM ENV
84 83 #======================================================================
85 84 try:
86 85 repo_name = environ['REPO_NAME'] = self.__get_repository(environ)
87 86 log.debug('Extracted repo name is %s' % repo_name)
88 87 except:
89 88 return HTTPInternalServerError()(environ, start_response)
90 89
91 90 # quick check if that dir exists...
92 91 if is_valid_repo(repo_name, self.basepath, 'hg') is False:
93 92 return HTTPNotFound()(environ, start_response)
94 93
95 94 #======================================================================
96 95 # GET ACTION PULL or PUSH
97 96 #======================================================================
98 97 action = self.__get_action(environ)
99 98
100 99 #======================================================================
101 100 # CHECK ANONYMOUS PERMISSION
102 101 #======================================================================
103 102 if action in ['pull', 'push']:
104 103 anonymous_user = self.__get_user('default')
105 104 username = anonymous_user.username
106 105 anonymous_perm = self._check_permission(action, anonymous_user,
107 106 repo_name)
108 107
109 108 if anonymous_perm is not True or anonymous_user.active is False:
110 109 if anonymous_perm is not True:
111 110 log.debug('Not enough credentials to access this '
112 111 'repository as anonymous user')
113 112 if anonymous_user.active is False:
114 113 log.debug('Anonymous access is disabled, running '
115 114 'authentication')
116 115 #==============================================================
117 116 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
118 117 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
119 118 #==============================================================
120 119
121 120 # Attempting to retrieve username from the container
122 121 username = get_container_username(environ, self.config)
123 122
124 123 # If not authenticated by the container, running basic auth
125 124 if not username:
126 125 self.authenticate.realm = \
127 126 safe_str(self.config['rhodecode_realm'])
128 127 result = self.authenticate(environ)
129 128 if isinstance(result, str):
130 129 AUTH_TYPE.update(environ, 'basic')
131 130 REMOTE_USER.update(environ, result)
132 131 username = result
133 132 else:
134 133 return result.wsgi_application(environ, start_response)
135 134
136 135 #==============================================================
137 136 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
138 137 #==============================================================
139 138 try:
140 139 user = self.__get_user(username)
141 140 if user is None or not user.active:
142 141 return HTTPForbidden()(environ, start_response)
143 142 username = user.username
144 143 except:
145 144 log.error(traceback.format_exc())
146 145 return HTTPInternalServerError()(environ, start_response)
147 146
148 147 #check permissions for this repository
149 148 perm = self._check_permission(action, user, repo_name)
150 149 if perm is not True:
151 150 return HTTPForbidden()(environ, start_response)
152 151
153 152 # extras are injected into mercurial UI object and later available
154 153 # in hg hooks executed by rhodecode
155 154 from rhodecode import CONFIG
156 155 extras = {
157 156 'ip': ipaddr,
158 157 'username': username,
159 158 'action': action,
160 159 'repository': repo_name,
161 160 'scm': 'hg',
162 161 'config': CONFIG['__file__'],
163 162 'make_lock': None,
164 163 'locked_by': [None, None]
165 164 }
166 165 #======================================================================
167 166 # MERCURIAL REQUEST HANDLING
168 167 #======================================================================
169 168 repo_path = os.path.join(safe_str(self.basepath), safe_str(repo_name))
170 169 log.debug('Repository path is %s' % repo_path)
171 170
172 171 # CHECK LOCKING only if it's not ANONYMOUS USER
173 172 if username != User.DEFAULT_USER:
174 173 log.debug('Checking locking on repository')
175 174 (make_lock,
176 175 locked,
177 176 locked_by) = self._check_locking_state(
178 177 environ=environ, action=action,
179 178 repo=repo_name, user_id=user.user_id
180 179 )
181 180 # store the make_lock for later evaluation in hooks
182 181 extras.update({'make_lock': make_lock,
183 182 'locked_by': locked_by})
184 183
185 184 # set the environ variables for this request
186 185 os.environ['RC_SCM_DATA'] = json.dumps(extras)
186 fix_PATH()
187 187 log.debug('HOOKS extras is %s' % extras)
188 188 baseui = make_ui('db')
189 189 self.__inject_extras(repo_path, baseui, extras)
190 190
191 191 try:
192 192 # invalidate cache on push
193 193 if action == 'push':
194 194 self._invalidate_cache(repo_name)
195 195 log.info('%s action on HG repo "%s"' % (action, repo_name))
196 196 app = self.__make_app(repo_path, baseui, extras)
197 197 return app(environ, start_response)
198 198 except RepoError, e:
199 199 if str(e).find('not found') != -1:
200 200 return HTTPNotFound()(environ, start_response)
201 201 except HTTPLockedRC, e:
202 202 log.debug('Repositry LOCKED ret code 423!')
203 203 return e(environ, start_response)
204 204 except Exception:
205 205 log.error(traceback.format_exc())
206 206 return HTTPInternalServerError()(environ, start_response)
207 207
208 208 def __make_app(self, repo_name, baseui, extras):
209 209 """
210 210 Make an wsgi application using hgweb, and inject generated baseui
211 211 instance, additionally inject some extras into ui object
212 212 """
213 213 return hgweb_mod.hgweb(repo_name, name=repo_name, baseui=baseui)
214 214
215 215 def __get_repository(self, environ):
216 216 """
217 217 Get's repository name out of PATH_INFO header
218 218
219 219 :param environ: environ where PATH_INFO is stored
220 220 """
221 221 try:
222 222 environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
223 223 repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
224 224 if repo_name.endswith('/'):
225 225 repo_name = repo_name.rstrip('/')
226 226 except:
227 227 log.error(traceback.format_exc())
228 228 raise
229 229
230 230 return repo_name
231 231
232 232 def __get_user(self, username):
233 233 return User.get_by_username(username)
234 234
235 235 def __get_action(self, environ):
236 236 """
237 237 Maps mercurial request commands into a clone,pull or push command.
238 238 This should always return a valid command string
239 239
240 240 :param environ:
241 241 """
242 242 mapping = {'changegroup': 'pull',
243 243 'changegroupsubset': 'pull',
244 244 'stream_out': 'pull',
245 245 'listkeys': 'pull',
246 246 'unbundle': 'push',
247 247 'pushkey': 'push', }
248 248 for qry in environ['QUERY_STRING'].split('&'):
249 249 if qry.startswith('cmd'):
250 250 cmd = qry.split('=')[-1]
251 251 if cmd in mapping:
252 252 return mapping[cmd]
253 253
254 254 return 'pull'
255 255
256 256 raise Exception('Unable to detect pull/push action !!'
257 257 'Are you using non standard command or client ?')
258 258
259 259 def __inject_extras(self, repo_path, baseui, extras={}):
260 260 """
261 261 Injects some extra params into baseui instance
262 262
263 263 also overwrites global settings with those takes from local hgrc file
264 264
265 265 :param baseui: baseui instance
266 266 :param extras: dict with extra params to put into baseui
267 267 """
268 268
269 269 hgrc = os.path.join(repo_path, '.hg', 'hgrc')
270 270
271 271 # make our hgweb quiet so it doesn't print output
272 272 baseui.setconfig('ui', 'quiet', 'true')
273 273
274 274 #inject some additional parameters that will be available in ui
275 275 #for hooks
276 276 for k, v in extras.items():
277 277 baseui.setconfig('rhodecode_extras', k, v)
278 278
279 279 repoui = make_ui('file', hgrc, False)
280 280
281 281 if repoui:
282 282 #overwrite our ui instance with the section from hgrc file
283 283 for section in ui_sections:
284 284 for k, v in repoui.configitems(section):
285 285 baseui.setconfig(section, k, v)
@@ -1,483 +1,499 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 3 rhodecode.lib.utils
4 4 ~~~~~~~~~~~~~~~~~~~
5 5
6 6 Some simple helper functions
7 7
8 8 :created_on: Jan 5, 2011
9 9 :author: marcink
10 10 :copyright: (C) 2011-2012 Marcin Kuzminski <marcin@python-works.com>
11 11 :license: GPLv3, see COPYING for more details.
12 12 """
13 13 # This program is free software: you can redistribute it and/or modify
14 14 # it under the terms of the GNU General Public License as published by
15 15 # the Free Software Foundation, either version 3 of the License, or
16 16 # (at your option) any later version.
17 17 #
18 18 # This program is distributed in the hope that it will be useful,
19 19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 21 # GNU General Public License for more details.
22 22 #
23 23 # You should have received a copy of the GNU General Public License
24 24 # along with this program. If not, see <http://www.gnu.org/licenses/>.
25 25
26 26 import re
27 27 import time
28 28 import datetime
29 29 from pylons.i18n.translation import _, ungettext
30 30 from rhodecode.lib.vcs.utils.lazy import LazyProperty
31 31
32 32
33 33 def __get_lem():
34 34 """
35 35 Get language extension map based on what's inside pygments lexers
36 36 """
37 37 from pygments import lexers
38 38 from string import lower
39 39 from collections import defaultdict
40 40
41 41 d = defaultdict(lambda: [])
42 42
43 43 def __clean(s):
44 44 s = s.lstrip('*')
45 45 s = s.lstrip('.')
46 46
47 47 if s.find('[') != -1:
48 48 exts = []
49 49 start, stop = s.find('['), s.find(']')
50 50
51 51 for suffix in s[start + 1:stop]:
52 52 exts.append(s[:s.find('[')] + suffix)
53 53 return map(lower, exts)
54 54 else:
55 55 return map(lower, [s])
56 56
57 57 for lx, t in sorted(lexers.LEXERS.items()):
58 58 m = map(__clean, t[-2])
59 59 if m:
60 60 m = reduce(lambda x, y: x + y, m)
61 61 for ext in m:
62 62 desc = lx.replace('Lexer', '')
63 63 d[ext].append(desc)
64 64
65 65 return dict(d)
66 66
67 67 def str2bool(_str):
68 68 """
69 69 returs True/False value from given string, it tries to translate the
70 70 string into boolean
71 71
72 72 :param _str: string value to translate into boolean
73 73 :rtype: boolean
74 74 :returns: boolean from given string
75 75 """
76 76 if _str is None:
77 77 return False
78 78 if _str in (True, False):
79 79 return _str
80 80 _str = str(_str).strip().lower()
81 81 return _str in ('t', 'true', 'y', 'yes', 'on', '1')
82 82
83 83
84 84 def convert_line_endings(line, mode):
85 85 """
86 86 Converts a given line "line end" accordingly to given mode
87 87
88 88 Available modes are::
89 89 0 - Unix
90 90 1 - Mac
91 91 2 - DOS
92 92
93 93 :param line: given line to convert
94 94 :param mode: mode to convert to
95 95 :rtype: str
96 96 :return: converted line according to mode
97 97 """
98 98 from string import replace
99 99
100 100 if mode == 0:
101 101 line = replace(line, '\r\n', '\n')
102 102 line = replace(line, '\r', '\n')
103 103 elif mode == 1:
104 104 line = replace(line, '\r\n', '\r')
105 105 line = replace(line, '\n', '\r')
106 106 elif mode == 2:
107 107 line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
108 108 return line
109 109
110 110
111 111 def detect_mode(line, default):
112 112 """
113 113 Detects line break for given line, if line break couldn't be found
114 114 given default value is returned
115 115
116 116 :param line: str line
117 117 :param default: default
118 118 :rtype: int
119 119 :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
120 120 """
121 121 if line.endswith('\r\n'):
122 122 return 2
123 123 elif line.endswith('\n'):
124 124 return 0
125 125 elif line.endswith('\r'):
126 126 return 1
127 127 else:
128 128 return default
129 129
130 130
131 131 def generate_api_key(username, salt=None):
132 132 """
133 133 Generates unique API key for given username, if salt is not given
134 134 it'll be generated from some random string
135 135
136 136 :param username: username as string
137 137 :param salt: salt to hash generate KEY
138 138 :rtype: str
139 139 :returns: sha1 hash from username+salt
140 140 """
141 141 from tempfile import _RandomNameSequence
142 142 import hashlib
143 143
144 144 if salt is None:
145 145 salt = _RandomNameSequence().next()
146 146
147 147 return hashlib.sha1(username + salt).hexdigest()
148 148
149 149
150 150 def safe_int(val, default=None):
151 151 """
152 152 Returns int() of val if val is not convertable to int use default
153 153 instead
154 154
155 155 :param val:
156 156 :param default:
157 157 """
158 158
159 159 try:
160 160 val = int(val)
161 161 except ValueError:
162 162 val = default
163 163
164 164 return val
165 165
166 166
167 167 def safe_unicode(str_, from_encoding=None):
168 168 """
169 169 safe unicode function. Does few trick to turn str_ into unicode
170 170
171 171 In case of UnicodeDecode error we try to return it with encoding detected
172 172 by chardet library if it fails fallback to unicode with errors replaced
173 173
174 174 :param str_: string to decode
175 175 :rtype: unicode
176 176 :returns: unicode object
177 177 """
178 178 if isinstance(str_, unicode):
179 179 return str_
180 180
181 181 if not from_encoding:
182 182 import rhodecode
183 183 DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
184 184 from_encoding = DEFAULT_ENCODING
185 185
186 186 try:
187 187 return unicode(str_)
188 188 except UnicodeDecodeError:
189 189 pass
190 190
191 191 try:
192 192 return unicode(str_, from_encoding)
193 193 except UnicodeDecodeError:
194 194 pass
195 195
196 196 try:
197 197 import chardet
198 198 encoding = chardet.detect(str_)['encoding']
199 199 if encoding is None:
200 200 raise Exception()
201 201 return str_.decode(encoding)
202 202 except (ImportError, UnicodeDecodeError, Exception):
203 203 return unicode(str_, from_encoding, 'replace')
204 204
205 205
206 206 def safe_str(unicode_, to_encoding=None):
207 207 """
208 208 safe str function. Does few trick to turn unicode_ into string
209 209
210 210 In case of UnicodeEncodeError we try to return it with encoding detected
211 211 by chardet library if it fails fallback to string with errors replaced
212 212
213 213 :param unicode_: unicode to encode
214 214 :rtype: str
215 215 :returns: str object
216 216 """
217 217
218 218 # if it's not basestr cast to str
219 219 if not isinstance(unicode_, basestring):
220 220 return str(unicode_)
221 221
222 222 if isinstance(unicode_, str):
223 223 return unicode_
224 224
225 225 if not to_encoding:
226 226 import rhodecode
227 227 DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
228 228 to_encoding = DEFAULT_ENCODING
229 229
230 230 try:
231 231 return unicode_.encode(to_encoding)
232 232 except UnicodeEncodeError:
233 233 pass
234 234
235 235 try:
236 236 import chardet
237 237 encoding = chardet.detect(unicode_)['encoding']
238 238 if encoding is None:
239 239 raise UnicodeEncodeError()
240 240
241 241 return unicode_.encode(encoding)
242 242 except (ImportError, UnicodeEncodeError):
243 243 return unicode_.encode(to_encoding, 'replace')
244 244
245 245 return safe_str
246 246
247 247
248 248 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
249 249 """
250 250 Custom engine_from_config functions that makes sure we use NullPool for
251 251 file based sqlite databases. This prevents errors on sqlite. This only
252 252 applies to sqlalchemy versions < 0.7.0
253 253
254 254 """
255 255 import sqlalchemy
256 256 from sqlalchemy import engine_from_config as efc
257 257 import logging
258 258
259 259 if int(sqlalchemy.__version__.split('.')[1]) < 7:
260 260
261 261 # This solution should work for sqlalchemy < 0.7.0, and should use
262 262 # proxy=TimerProxy() for execution time profiling
263 263
264 264 from sqlalchemy.pool import NullPool
265 265 url = configuration[prefix + 'url']
266 266
267 267 if url.startswith('sqlite'):
268 268 kwargs.update({'poolclass': NullPool})
269 269 return efc(configuration, prefix, **kwargs)
270 270 else:
271 271 import time
272 272 from sqlalchemy import event
273 273 from sqlalchemy.engine import Engine
274 274
275 275 log = logging.getLogger('sqlalchemy.engine')
276 276 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
277 277 engine = efc(configuration, prefix, **kwargs)
278 278
279 279 def color_sql(sql):
280 280 COLOR_SEQ = "\033[1;%dm"
281 281 COLOR_SQL = YELLOW
282 282 normal = '\x1b[0m'
283 283 return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal])
284 284
285 285 if configuration['debug']:
286 286 #attach events only for debug configuration
287 287
288 288 def before_cursor_execute(conn, cursor, statement,
289 289 parameters, context, executemany):
290 290 context._query_start_time = time.time()
291 291 log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
292 292
293 293 def after_cursor_execute(conn, cursor, statement,
294 294 parameters, context, executemany):
295 295 total = time.time() - context._query_start_time
296 296 log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total))
297 297
298 298 event.listen(engine, "before_cursor_execute",
299 299 before_cursor_execute)
300 300 event.listen(engine, "after_cursor_execute",
301 301 after_cursor_execute)
302 302
303 303 return engine
304 304
305 305
306 306 def age(prevdate):
307 307 """
308 308 turns a datetime into an age string.
309 309
310 310 :param prevdate: datetime object
311 311 :rtype: unicode
312 312 :returns: unicode words describing age
313 313 """
314 314
315 315 order = ['year', 'month', 'day', 'hour', 'minute', 'second']
316 316 deltas = {}
317 317
318 318 # Get date parts deltas
319 319 now = datetime.datetime.now()
320 320 for part in order:
321 321 deltas[part] = getattr(now, part) - getattr(prevdate, part)
322 322
323 323 # Fix negative offsets (there is 1 second between 10:59:59 and 11:00:00,
324 324 # not 1 hour, -59 minutes and -59 seconds)
325 325
326 326 for num, length in [(5, 60), (4, 60), (3, 24)]: # seconds, minutes, hours
327 327 part = order[num]
328 328 carry_part = order[num - 1]
329 329
330 330 if deltas[part] < 0:
331 331 deltas[part] += length
332 332 deltas[carry_part] -= 1
333 333
334 334 # Same thing for days except that the increment depends on the (variable)
335 335 # number of days in the month
336 336 month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
337 337 if deltas['day'] < 0:
338 338 if prevdate.month == 2 and (prevdate.year % 4 == 0 and
339 339 (prevdate.year % 100 != 0 or prevdate.year % 400 == 0)):
340 340 deltas['day'] += 29
341 341 else:
342 342 deltas['day'] += month_lengths[prevdate.month - 1]
343 343
344 344 deltas['month'] -= 1
345 345
346 346 if deltas['month'] < 0:
347 347 deltas['month'] += 12
348 348 deltas['year'] -= 1
349 349
350 350 # Format the result
351 351 fmt_funcs = {
352 352 'year': lambda d: ungettext(u'%d year', '%d years', d) % d,
353 353 'month': lambda d: ungettext(u'%d month', '%d months', d) % d,
354 354 'day': lambda d: ungettext(u'%d day', '%d days', d) % d,
355 355 'hour': lambda d: ungettext(u'%d hour', '%d hours', d) % d,
356 356 'minute': lambda d: ungettext(u'%d minute', '%d minutes', d) % d,
357 357 'second': lambda d: ungettext(u'%d second', '%d seconds', d) % d,
358 358 }
359 359
360 360 for i, part in enumerate(order):
361 361 value = deltas[part]
362 362 if value == 0:
363 363 continue
364 364
365 365 if i < 5:
366 366 sub_part = order[i + 1]
367 367 sub_value = deltas[sub_part]
368 368 else:
369 369 sub_value = 0
370 370
371 371 if sub_value == 0:
372 372 return _(u'%s ago') % fmt_funcs[part](value)
373 373
374 374 return _(u'%s and %s ago') % (fmt_funcs[part](value),
375 375 fmt_funcs[sub_part](sub_value))
376 376
377 377 return _(u'just now')
378 378
379 379
380 380 def uri_filter(uri):
381 381 """
382 382 Removes user:password from given url string
383 383
384 384 :param uri:
385 385 :rtype: unicode
386 386 :returns: filtered list of strings
387 387 """
388 388 if not uri:
389 389 return ''
390 390
391 391 proto = ''
392 392
393 393 for pat in ('https://', 'http://'):
394 394 if uri.startswith(pat):
395 395 uri = uri[len(pat):]
396 396 proto = pat
397 397 break
398 398
399 399 # remove passwords and username
400 400 uri = uri[uri.find('@') + 1:]
401 401
402 402 # get the port
403 403 cred_pos = uri.find(':')
404 404 if cred_pos == -1:
405 405 host, port = uri, None
406 406 else:
407 407 host, port = uri[:cred_pos], uri[cred_pos + 1:]
408 408
409 409 return filter(None, [proto, host, port])
410 410
411 411
412 412 def credentials_filter(uri):
413 413 """
414 414 Returns a url with removed credentials
415 415
416 416 :param uri:
417 417 """
418 418
419 419 uri = uri_filter(uri)
420 420 #check if we have port
421 421 if len(uri) > 2 and uri[2]:
422 422 uri[2] = ':' + uri[2]
423 423
424 424 return ''.join(uri)
425 425
426 426
427 427 def get_changeset_safe(repo, rev):
428 428 """
429 429 Safe version of get_changeset if this changeset doesn't exists for a
430 430 repo it returns a Dummy one instead
431 431
432 432 :param repo:
433 433 :param rev:
434 434 """
435 435 from rhodecode.lib.vcs.backends.base import BaseRepository
436 436 from rhodecode.lib.vcs.exceptions import RepositoryError
437 437 from rhodecode.lib.vcs.backends.base import EmptyChangeset
438 438 if not isinstance(repo, BaseRepository):
439 439 raise Exception('You must pass an Repository '
440 440 'object as first argument got %s', type(repo))
441 441
442 442 try:
443 443 cs = repo.get_changeset(rev)
444 444 except RepositoryError:
445 445 cs = EmptyChangeset(requested_revision=rev)
446 446 return cs
447 447
448 448
449 449 def datetime_to_time(dt):
450 450 if dt:
451 451 return time.mktime(dt.timetuple())
452 452
453 453
454 454 def time_to_datetime(tm):
455 455 if tm:
456 456 if isinstance(tm, basestring):
457 457 try:
458 458 tm = float(tm)
459 459 except ValueError:
460 460 return
461 461 return datetime.datetime.fromtimestamp(tm)
462 462
463 463 MENTIONS_REGEX = r'(?:^@|\s@)([a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+)(?:\s{1})'
464 464
465 465
466 466 def extract_mentioned_users(s):
467 467 """
468 468 Returns unique usernames from given string s that have @mention
469 469
470 470 :param s: string to get mentions
471 471 """
472 472 usrs = set()
473 473 for username in re.findall(MENTIONS_REGEX, s):
474 474 usrs.add(username)
475 475
476 476 return sorted(list(usrs), key=lambda k: k.lower())
477 477
478 478
479 479 class AttributeDict(dict):
480 480 def __getattr__(self, attr):
481 481 return self.get(attr, None)
482 482 __setattr__ = dict.__setitem__
483 483 __delattr__ = dict.__delitem__
484
485
486 def fix_PATH(os_=None):
487 """
488 Get current active python path, and append it to PATH variable to fix issues
489 of subprocess calls and different python versions
490 """
491 import sys
492 if os_ is None:
493 import os
494 else:
495 os = os_
496
497 cur_path = os.path.split(sys.executable)[0]
498 if not os.environ['PATH'].startswith(cur_path):
499 os.environ['PATH'] = '%s:%s' % (cur_path, os.environ['PATH'])
General Comments 0
You need to be logged in to leave comments. Login now