##// END OF EJS Templates
fix for issue #578 git hooks sometimes cannot be executed due to different python they runned under, this commit tries to fix that by altering the PATH env variable using current python that rhodecode is running
marcink -
r2869:ccbdff90 beta
parent child Browse files
Show More
@@ -1,337 +1,338 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 rhodecode.lib.middleware.simplegit
3 rhodecode.lib.middleware.simplegit
4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5
5
6 SimpleGit middleware for handling git protocol request (push/clone etc.)
6 SimpleGit middleware for handling git protocol request (push/clone etc.)
7 It's implemented with basic auth function
7 It's implemented with basic auth function
8
8
9 :created_on: Apr 28, 2010
9 :created_on: Apr 28, 2010
10 :author: marcink
10 :author: marcink
11 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
11 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
12 :license: GPLv3, see COPYING for more details.
12 :license: GPLv3, see COPYING for more details.
13 """
13 """
14 # This program is free software: you can redistribute it and/or modify
14 # This program is free software: you can redistribute it and/or modify
15 # it under the terms of the GNU General Public License as published by
15 # it under the terms of the GNU General Public License as published by
16 # the Free Software Foundation, either version 3 of the License, or
16 # the Free Software Foundation, either version 3 of the License, or
17 # (at your option) any later version.
17 # (at your option) any later version.
18 #
18 #
19 # This program is distributed in the hope that it will be useful,
19 # This program is distributed in the hope that it will be useful,
20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 # GNU General Public License for more details.
22 # GNU General Public License for more details.
23 #
23 #
24 # You should have received a copy of the GNU General Public License
24 # You should have received a copy of the GNU General Public License
25 # along with this program. If not, see <http://www.gnu.org/licenses/>.
25 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26
26
27 import os
27 import os
28 import re
28 import re
29 import logging
29 import logging
30 import traceback
30 import traceback
31
31
32 from dulwich import server as dulserver
32 from dulwich import server as dulserver
33 from dulwich.web import LimitedInputFilter, GunzipFilter
33 from dulwich.web import LimitedInputFilter, GunzipFilter
34 from rhodecode.lib.exceptions import HTTPLockedRC
34 from rhodecode.lib.exceptions import HTTPLockedRC
35 from rhodecode.lib.hooks import pre_pull
35 from rhodecode.lib.hooks import pre_pull
36
36
37
37
38 class SimpleGitUploadPackHandler(dulserver.UploadPackHandler):
38 class SimpleGitUploadPackHandler(dulserver.UploadPackHandler):
39
39
40 def handle(self):
40 def handle(self):
41 write = lambda x: self.proto.write_sideband(1, x)
41 write = lambda x: self.proto.write_sideband(1, x)
42
42
43 graph_walker = dulserver.ProtocolGraphWalker(self,
43 graph_walker = dulserver.ProtocolGraphWalker(self,
44 self.repo.object_store,
44 self.repo.object_store,
45 self.repo.get_peeled)
45 self.repo.get_peeled)
46 objects_iter = self.repo.fetch_objects(
46 objects_iter = self.repo.fetch_objects(
47 graph_walker.determine_wants, graph_walker, self.progress,
47 graph_walker.determine_wants, graph_walker, self.progress,
48 get_tagged=self.get_tagged)
48 get_tagged=self.get_tagged)
49
49
50 # Did the process short-circuit (e.g. in a stateless RPC call)? Note
50 # Did the process short-circuit (e.g. in a stateless RPC call)? Note
51 # that the client still expects a 0-object pack in most cases.
51 # that the client still expects a 0-object pack in most cases.
52 if objects_iter is None:
52 if objects_iter is None:
53 return
53 return
54
54
55 self.progress("counting objects: %d, done.\n" % len(objects_iter))
55 self.progress("counting objects: %d, done.\n" % len(objects_iter))
56 dulserver.write_pack_objects(dulserver.ProtocolFile(None, write),
56 dulserver.write_pack_objects(dulserver.ProtocolFile(None, write),
57 objects_iter)
57 objects_iter)
58 messages = []
58 messages = []
59 messages.append('thank you for using rhodecode')
59 messages.append('thank you for using rhodecode')
60
60
61 for msg in messages:
61 for msg in messages:
62 self.progress(msg + "\n")
62 self.progress(msg + "\n")
63 # we are done
63 # we are done
64 self.proto.write("0000")
64 self.proto.write("0000")
65
65
66
66
67 dulserver.DEFAULT_HANDLERS = {
67 dulserver.DEFAULT_HANDLERS = {
68 #git-ls-remote, git-clone, git-fetch and git-pull
68 #git-ls-remote, git-clone, git-fetch and git-pull
69 'git-upload-pack': SimpleGitUploadPackHandler,
69 'git-upload-pack': SimpleGitUploadPackHandler,
70 #git-push
70 #git-push
71 'git-receive-pack': dulserver.ReceivePackHandler,
71 'git-receive-pack': dulserver.ReceivePackHandler,
72 }
72 }
73
73
74 # not used for now until dulwich get's fixed
74 # not used for now until dulwich get's fixed
75 #from dulwich.repo import Repo
75 #from dulwich.repo import Repo
76 #from dulwich.web import make_wsgi_chain
76 #from dulwich.web import make_wsgi_chain
77
77
78 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
78 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
79 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
79 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
80 HTTPBadRequest, HTTPNotAcceptable
80 HTTPBadRequest, HTTPNotAcceptable
81
81
82 from rhodecode.lib.utils2 import safe_str
82 from rhodecode.lib.utils2 import safe_str, fix_PATH
83 from rhodecode.lib.base import BaseVCSController
83 from rhodecode.lib.base import BaseVCSController
84 from rhodecode.lib.auth import get_container_username
84 from rhodecode.lib.auth import get_container_username
85 from rhodecode.lib.utils import is_valid_repo, make_ui
85 from rhodecode.lib.utils import is_valid_repo, make_ui
86 from rhodecode.lib.compat import json
86 from rhodecode.lib.compat import json
87 from rhodecode.model.db import User, RhodeCodeUi
87 from rhodecode.model.db import User, RhodeCodeUi
88
88
89 log = logging.getLogger(__name__)
89 log = logging.getLogger(__name__)
90
90
91
91
92 GIT_PROTO_PAT = re.compile(r'^/(.+)/(info/refs|git-upload-pack|git-receive-pack)')
92 GIT_PROTO_PAT = re.compile(r'^/(.+)/(info/refs|git-upload-pack|git-receive-pack)')
93
93
94
94
95 def is_git(environ):
95 def is_git(environ):
96 path_info = environ['PATH_INFO']
96 path_info = environ['PATH_INFO']
97 isgit_path = GIT_PROTO_PAT.match(path_info)
97 isgit_path = GIT_PROTO_PAT.match(path_info)
98 log.debug('pathinfo: %s detected as GIT %s' % (
98 log.debug('pathinfo: %s detected as GIT %s' % (
99 path_info, isgit_path != None)
99 path_info, isgit_path != None)
100 )
100 )
101 return isgit_path
101 return isgit_path
102
102
103
103
104 class SimpleGit(BaseVCSController):
104 class SimpleGit(BaseVCSController):
105
105
106 def _handle_request(self, environ, start_response):
106 def _handle_request(self, environ, start_response):
107 if not is_git(environ):
107 if not is_git(environ):
108 return self.application(environ, start_response)
108 return self.application(environ, start_response)
109 if not self._check_ssl(environ, start_response):
109 if not self._check_ssl(environ, start_response):
110 return HTTPNotAcceptable('SSL REQUIRED !')(environ, start_response)
110 return HTTPNotAcceptable('SSL REQUIRED !')(environ, start_response)
111
111
112 ipaddr = self._get_ip_addr(environ)
112 ipaddr = self._get_ip_addr(environ)
113 username = None
113 username = None
114 self._git_first_op = False
114 self._git_first_op = False
115 # skip passing error to error controller
115 # skip passing error to error controller
116 environ['pylons.status_code_redirect'] = True
116 environ['pylons.status_code_redirect'] = True
117
117
118 #======================================================================
118 #======================================================================
119 # EXTRACT REPOSITORY NAME FROM ENV
119 # EXTRACT REPOSITORY NAME FROM ENV
120 #======================================================================
120 #======================================================================
121 try:
121 try:
122 repo_name = self.__get_repository(environ)
122 repo_name = self.__get_repository(environ)
123 log.debug('Extracted repo name is %s' % repo_name)
123 log.debug('Extracted repo name is %s' % repo_name)
124 except:
124 except:
125 return HTTPInternalServerError()(environ, start_response)
125 return HTTPInternalServerError()(environ, start_response)
126
126
127 # quick check if that dir exists...
127 # quick check if that dir exists...
128 if is_valid_repo(repo_name, self.basepath, 'git') is False:
128 if is_valid_repo(repo_name, self.basepath, 'git') is False:
129 return HTTPNotFound()(environ, start_response)
129 return HTTPNotFound()(environ, start_response)
130
130
131 #======================================================================
131 #======================================================================
132 # GET ACTION PULL or PUSH
132 # GET ACTION PULL or PUSH
133 #======================================================================
133 #======================================================================
134 action = self.__get_action(environ)
134 action = self.__get_action(environ)
135
135
136 #======================================================================
136 #======================================================================
137 # CHECK ANONYMOUS PERMISSION
137 # CHECK ANONYMOUS PERMISSION
138 #======================================================================
138 #======================================================================
139 if action in ['pull', 'push']:
139 if action in ['pull', 'push']:
140 anonymous_user = self.__get_user('default')
140 anonymous_user = self.__get_user('default')
141 username = anonymous_user.username
141 username = anonymous_user.username
142 anonymous_perm = self._check_permission(action, anonymous_user,
142 anonymous_perm = self._check_permission(action, anonymous_user,
143 repo_name)
143 repo_name)
144
144
145 if anonymous_perm is not True or anonymous_user.active is False:
145 if anonymous_perm is not True or anonymous_user.active is False:
146 if anonymous_perm is not True:
146 if anonymous_perm is not True:
147 log.debug('Not enough credentials to access this '
147 log.debug('Not enough credentials to access this '
148 'repository as anonymous user')
148 'repository as anonymous user')
149 if anonymous_user.active is False:
149 if anonymous_user.active is False:
150 log.debug('Anonymous access is disabled, running '
150 log.debug('Anonymous access is disabled, running '
151 'authentication')
151 'authentication')
152 #==============================================================
152 #==============================================================
153 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
153 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
154 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
154 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
155 #==============================================================
155 #==============================================================
156
156
157 # Attempting to retrieve username from the container
157 # Attempting to retrieve username from the container
158 username = get_container_username(environ, self.config)
158 username = get_container_username(environ, self.config)
159
159
160 # If not authenticated by the container, running basic auth
160 # If not authenticated by the container, running basic auth
161 if not username:
161 if not username:
162 self.authenticate.realm = \
162 self.authenticate.realm = \
163 safe_str(self.config['rhodecode_realm'])
163 safe_str(self.config['rhodecode_realm'])
164 result = self.authenticate(environ)
164 result = self.authenticate(environ)
165 if isinstance(result, str):
165 if isinstance(result, str):
166 AUTH_TYPE.update(environ, 'basic')
166 AUTH_TYPE.update(environ, 'basic')
167 REMOTE_USER.update(environ, result)
167 REMOTE_USER.update(environ, result)
168 username = result
168 username = result
169 else:
169 else:
170 return result.wsgi_application(environ, start_response)
170 return result.wsgi_application(environ, start_response)
171
171
172 #==============================================================
172 #==============================================================
173 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
173 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
174 #==============================================================
174 #==============================================================
175 try:
175 try:
176 user = self.__get_user(username)
176 user = self.__get_user(username)
177 if user is None or not user.active:
177 if user is None or not user.active:
178 return HTTPForbidden()(environ, start_response)
178 return HTTPForbidden()(environ, start_response)
179 username = user.username
179 username = user.username
180 except:
180 except:
181 log.error(traceback.format_exc())
181 log.error(traceback.format_exc())
182 return HTTPInternalServerError()(environ, start_response)
182 return HTTPInternalServerError()(environ, start_response)
183
183
184 #check permissions for this repository
184 #check permissions for this repository
185 perm = self._check_permission(action, user, repo_name)
185 perm = self._check_permission(action, user, repo_name)
186 if perm is not True:
186 if perm is not True:
187 return HTTPForbidden()(environ, start_response)
187 return HTTPForbidden()(environ, start_response)
188
188
189 # extras are injected into UI object and later available
189 # extras are injected into UI object and later available
190 # in hooks executed by rhodecode
190 # in hooks executed by rhodecode
191 from rhodecode import CONFIG
191 from rhodecode import CONFIG
192 extras = {
192 extras = {
193 'ip': ipaddr,
193 'ip': ipaddr,
194 'username': username,
194 'username': username,
195 'action': action,
195 'action': action,
196 'repository': repo_name,
196 'repository': repo_name,
197 'scm': 'git',
197 'scm': 'git',
198 'config': CONFIG['__file__'],
198 'config': CONFIG['__file__'],
199 'make_lock': None,
199 'make_lock': None,
200 'locked_by': [None, None]
200 'locked_by': [None, None]
201 }
201 }
202
202
203 #===================================================================
203 #===================================================================
204 # GIT REQUEST HANDLING
204 # GIT REQUEST HANDLING
205 #===================================================================
205 #===================================================================
206 repo_path = os.path.join(safe_str(self.basepath), safe_str(repo_name))
206 repo_path = os.path.join(safe_str(self.basepath), safe_str(repo_name))
207 log.debug('Repository path is %s' % repo_path)
207 log.debug('Repository path is %s' % repo_path)
208
208
209 # CHECK LOCKING only if it's not ANONYMOUS USER
209 # CHECK LOCKING only if it's not ANONYMOUS USER
210 if username != User.DEFAULT_USER:
210 if username != User.DEFAULT_USER:
211 log.debug('Checking locking on repository')
211 log.debug('Checking locking on repository')
212 (make_lock,
212 (make_lock,
213 locked,
213 locked,
214 locked_by) = self._check_locking_state(
214 locked_by) = self._check_locking_state(
215 environ=environ, action=action,
215 environ=environ, action=action,
216 repo=repo_name, user_id=user.user_id
216 repo=repo_name, user_id=user.user_id
217 )
217 )
218 # store the make_lock for later evaluation in hooks
218 # store the make_lock for later evaluation in hooks
219 extras.update({'make_lock': make_lock,
219 extras.update({'make_lock': make_lock,
220 'locked_by': locked_by})
220 'locked_by': locked_by})
221 # set the environ variables for this request
221 # set the environ variables for this request
222 os.environ['RC_SCM_DATA'] = json.dumps(extras)
222 os.environ['RC_SCM_DATA'] = json.dumps(extras)
223 fix_PATH()
223 log.debug('HOOKS extras is %s' % extras)
224 log.debug('HOOKS extras is %s' % extras)
224 baseui = make_ui('db')
225 baseui = make_ui('db')
225 self.__inject_extras(repo_path, baseui, extras)
226 self.__inject_extras(repo_path, baseui, extras)
226
227
227 try:
228 try:
228 # invalidate cache on push
229 # invalidate cache on push
229 if action == 'push':
230 if action == 'push':
230 self._invalidate_cache(repo_name)
231 self._invalidate_cache(repo_name)
231 self._handle_githooks(repo_name, action, baseui, environ)
232 self._handle_githooks(repo_name, action, baseui, environ)
232
233
233 log.info('%s action on GIT repo "%s"' % (action, repo_name))
234 log.info('%s action on GIT repo "%s"' % (action, repo_name))
234 app = self.__make_app(repo_name, repo_path, extras)
235 app = self.__make_app(repo_name, repo_path, extras)
235 return app(environ, start_response)
236 return app(environ, start_response)
236 except HTTPLockedRC, e:
237 except HTTPLockedRC, e:
237 log.debug('Repositry LOCKED ret code 423!')
238 log.debug('Repositry LOCKED ret code 423!')
238 return e(environ, start_response)
239 return e(environ, start_response)
239 except Exception:
240 except Exception:
240 log.error(traceback.format_exc())
241 log.error(traceback.format_exc())
241 return HTTPInternalServerError()(environ, start_response)
242 return HTTPInternalServerError()(environ, start_response)
242
243
243 def __make_app(self, repo_name, repo_path, extras):
244 def __make_app(self, repo_name, repo_path, extras):
244 """
245 """
245 Make an wsgi application using dulserver
246 Make an wsgi application using dulserver
246
247
247 :param repo_name: name of the repository
248 :param repo_name: name of the repository
248 :param repo_path: full path to the repository
249 :param repo_path: full path to the repository
249 """
250 """
250
251
251 from rhodecode.lib.middleware.pygrack import make_wsgi_app
252 from rhodecode.lib.middleware.pygrack import make_wsgi_app
252 app = make_wsgi_app(
253 app = make_wsgi_app(
253 repo_root=safe_str(self.basepath),
254 repo_root=safe_str(self.basepath),
254 repo_name=repo_name,
255 repo_name=repo_name,
255 extras=extras,
256 extras=extras,
256 )
257 )
257 app = GunzipFilter(LimitedInputFilter(app))
258 app = GunzipFilter(LimitedInputFilter(app))
258 return app
259 return app
259
260
260 def __get_repository(self, environ):
261 def __get_repository(self, environ):
261 """
262 """
262 Get's repository name out of PATH_INFO header
263 Get's repository name out of PATH_INFO header
263
264
264 :param environ: environ where PATH_INFO is stored
265 :param environ: environ where PATH_INFO is stored
265 """
266 """
266 try:
267 try:
267 environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
268 environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
268 repo_name = GIT_PROTO_PAT.match(environ['PATH_INFO']).group(1)
269 repo_name = GIT_PROTO_PAT.match(environ['PATH_INFO']).group(1)
269 except:
270 except:
270 log.error(traceback.format_exc())
271 log.error(traceback.format_exc())
271 raise
272 raise
272
273
273 return repo_name
274 return repo_name
274
275
275 def __get_user(self, username):
276 def __get_user(self, username):
276 return User.get_by_username(username)
277 return User.get_by_username(username)
277
278
278 def __get_action(self, environ):
279 def __get_action(self, environ):
279 """
280 """
280 Maps git request commands into a pull or push command.
281 Maps git request commands into a pull or push command.
281
282
282 :param environ:
283 :param environ:
283 """
284 """
284 service = environ['QUERY_STRING'].split('=')
285 service = environ['QUERY_STRING'].split('=')
285
286
286 if len(service) > 1:
287 if len(service) > 1:
287 service_cmd = service[1]
288 service_cmd = service[1]
288 mapping = {
289 mapping = {
289 'git-receive-pack': 'push',
290 'git-receive-pack': 'push',
290 'git-upload-pack': 'pull',
291 'git-upload-pack': 'pull',
291 }
292 }
292 op = mapping[service_cmd]
293 op = mapping[service_cmd]
293 self._git_stored_op = op
294 self._git_stored_op = op
294 return op
295 return op
295 else:
296 else:
296 # try to fallback to stored variable as we don't know if the last
297 # try to fallback to stored variable as we don't know if the last
297 # operation is pull/push
298 # operation is pull/push
298 op = getattr(self, '_git_stored_op', 'pull')
299 op = getattr(self, '_git_stored_op', 'pull')
299 return op
300 return op
300
301
301 def _handle_githooks(self, repo_name, action, baseui, environ):
302 def _handle_githooks(self, repo_name, action, baseui, environ):
302 """
303 """
303 Handles pull action, push is handled by post-receive hook
304 Handles pull action, push is handled by post-receive hook
304 """
305 """
305 from rhodecode.lib.hooks import log_pull_action
306 from rhodecode.lib.hooks import log_pull_action
306 service = environ['QUERY_STRING'].split('=')
307 service = environ['QUERY_STRING'].split('=')
307
308
308 if len(service) < 2:
309 if len(service) < 2:
309 return
310 return
310
311
311 from rhodecode.model.db import Repository
312 from rhodecode.model.db import Repository
312 _repo = Repository.get_by_repo_name(repo_name)
313 _repo = Repository.get_by_repo_name(repo_name)
313 _repo = _repo.scm_instance
314 _repo = _repo.scm_instance
314 _repo._repo.ui = baseui
315 _repo._repo.ui = baseui
315
316
316 _hooks = dict(baseui.configitems('hooks')) or {}
317 _hooks = dict(baseui.configitems('hooks')) or {}
317 if action == 'pull':
318 if action == 'pull':
318 # stupid git, emulate pre-pull hook !
319 # stupid git, emulate pre-pull hook !
319 pre_pull(ui=baseui, repo=_repo._repo)
320 pre_pull(ui=baseui, repo=_repo._repo)
320 if action == 'pull' and _hooks.get(RhodeCodeUi.HOOK_PULL):
321 if action == 'pull' and _hooks.get(RhodeCodeUi.HOOK_PULL):
321 log_pull_action(ui=baseui, repo=_repo._repo)
322 log_pull_action(ui=baseui, repo=_repo._repo)
322
323
323 def __inject_extras(self, repo_path, baseui, extras={}):
324 def __inject_extras(self, repo_path, baseui, extras={}):
324 """
325 """
325 Injects some extra params into baseui instance
326 Injects some extra params into baseui instance
326
327
327 :param baseui: baseui instance
328 :param baseui: baseui instance
328 :param extras: dict with extra params to put into baseui
329 :param extras: dict with extra params to put into baseui
329 """
330 """
330
331
331 # make our hgweb quiet so it doesn't print output
332 # make our hgweb quiet so it doesn't print output
332 baseui.setconfig('ui', 'quiet', 'true')
333 baseui.setconfig('ui', 'quiet', 'true')
333
334
334 #inject some additional parameters that will be available in ui
335 #inject some additional parameters that will be available in ui
335 #for hooks
336 #for hooks
336 for k, v in extras.items():
337 for k, v in extras.items():
337 baseui.setconfig('rhodecode_extras', k, v)
338 baseui.setconfig('rhodecode_extras', k, v)
@@ -1,285 +1,285 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 rhodecode.lib.middleware.simplehg
3 rhodecode.lib.middleware.simplehg
4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
5
5
6 SimpleHG middleware for handling mercurial protocol request
6 SimpleHG middleware for handling mercurial protocol request
7 (push/clone etc.). It's implemented with basic auth function
7 (push/clone etc.). It's implemented with basic auth function
8
8
9 :created_on: Apr 28, 2010
9 :created_on: Apr 28, 2010
10 :author: marcink
10 :author: marcink
11 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
11 :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
12 :license: GPLv3, see COPYING for more details.
12 :license: GPLv3, see COPYING for more details.
13 """
13 """
14 # This program is free software: you can redistribute it and/or modify
14 # This program is free software: you can redistribute it and/or modify
15 # it under the terms of the GNU General Public License as published by
15 # it under the terms of the GNU General Public License as published by
16 # the Free Software Foundation, either version 3 of the License, or
16 # the Free Software Foundation, either version 3 of the License, or
17 # (at your option) any later version.
17 # (at your option) any later version.
18 #
18 #
19 # This program is distributed in the hope that it will be useful,
19 # This program is distributed in the hope that it will be useful,
20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 # GNU General Public License for more details.
22 # GNU General Public License for more details.
23 #
23 #
24 # You should have received a copy of the GNU General Public License
24 # You should have received a copy of the GNU General Public License
25 # along with this program. If not, see <http://www.gnu.org/licenses/>.
25 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26
26
27 import os
27 import os
28 import logging
28 import logging
29 import traceback
29 import traceback
30 import urllib
31
30
32 from mercurial.error import RepoError
31 from mercurial.error import RepoError
33 from mercurial.hgweb import hgweb_mod
32 from mercurial.hgweb import hgweb_mod
34
33
35 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
34 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
36 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
35 from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError, \
37 HTTPBadRequest, HTTPNotAcceptable
36 HTTPBadRequest, HTTPNotAcceptable
38
37
39 from rhodecode.lib.utils2 import safe_str
38 from rhodecode.lib.utils2 import safe_str, fix_PATH
40 from rhodecode.lib.base import BaseVCSController
39 from rhodecode.lib.base import BaseVCSController
41 from rhodecode.lib.auth import get_container_username
40 from rhodecode.lib.auth import get_container_username
42 from rhodecode.lib.utils import make_ui, is_valid_repo, ui_sections
41 from rhodecode.lib.utils import make_ui, is_valid_repo, ui_sections
43 from rhodecode.lib.compat import json
42 from rhodecode.lib.compat import json
44 from rhodecode.model.db import User
43 from rhodecode.model.db import User
45 from rhodecode.lib.exceptions import HTTPLockedRC
44 from rhodecode.lib.exceptions import HTTPLockedRC
46
45
47
46
48 log = logging.getLogger(__name__)
47 log = logging.getLogger(__name__)
49
48
50
49
51 def is_mercurial(environ):
50 def is_mercurial(environ):
52 """
51 """
53 Returns True if request's target is mercurial server - header
52 Returns True if request's target is mercurial server - header
54 ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
53 ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
55 """
54 """
56 http_accept = environ.get('HTTP_ACCEPT')
55 http_accept = environ.get('HTTP_ACCEPT')
57 path_info = environ['PATH_INFO']
56 path_info = environ['PATH_INFO']
58 if http_accept and http_accept.startswith('application/mercurial'):
57 if http_accept and http_accept.startswith('application/mercurial'):
59 ishg_path = True
58 ishg_path = True
60 else:
59 else:
61 ishg_path = False
60 ishg_path = False
62
61
63 log.debug('pathinfo: %s detected as HG %s' % (
62 log.debug('pathinfo: %s detected as HG %s' % (
64 path_info, ishg_path)
63 path_info, ishg_path)
65 )
64 )
66 return ishg_path
65 return ishg_path
67
66
68
67
69 class SimpleHg(BaseVCSController):
68 class SimpleHg(BaseVCSController):
70
69
71 def _handle_request(self, environ, start_response):
70 def _handle_request(self, environ, start_response):
72 if not is_mercurial(environ):
71 if not is_mercurial(environ):
73 return self.application(environ, start_response)
72 return self.application(environ, start_response)
74 if not self._check_ssl(environ, start_response):
73 if not self._check_ssl(environ, start_response):
75 return HTTPNotAcceptable('SSL REQUIRED !')(environ, start_response)
74 return HTTPNotAcceptable('SSL REQUIRED !')(environ, start_response)
76
75
77 ipaddr = self._get_ip_addr(environ)
76 ipaddr = self._get_ip_addr(environ)
78 username = None
77 username = None
79 # skip passing error to error controller
78 # skip passing error to error controller
80 environ['pylons.status_code_redirect'] = True
79 environ['pylons.status_code_redirect'] = True
81
80
82 #======================================================================
81 #======================================================================
83 # EXTRACT REPOSITORY NAME FROM ENV
82 # EXTRACT REPOSITORY NAME FROM ENV
84 #======================================================================
83 #======================================================================
85 try:
84 try:
86 repo_name = environ['REPO_NAME'] = self.__get_repository(environ)
85 repo_name = environ['REPO_NAME'] = self.__get_repository(environ)
87 log.debug('Extracted repo name is %s' % repo_name)
86 log.debug('Extracted repo name is %s' % repo_name)
88 except:
87 except:
89 return HTTPInternalServerError()(environ, start_response)
88 return HTTPInternalServerError()(environ, start_response)
90
89
91 # quick check if that dir exists...
90 # quick check if that dir exists...
92 if is_valid_repo(repo_name, self.basepath, 'hg') is False:
91 if is_valid_repo(repo_name, self.basepath, 'hg') is False:
93 return HTTPNotFound()(environ, start_response)
92 return HTTPNotFound()(environ, start_response)
94
93
95 #======================================================================
94 #======================================================================
96 # GET ACTION PULL or PUSH
95 # GET ACTION PULL or PUSH
97 #======================================================================
96 #======================================================================
98 action = self.__get_action(environ)
97 action = self.__get_action(environ)
99
98
100 #======================================================================
99 #======================================================================
101 # CHECK ANONYMOUS PERMISSION
100 # CHECK ANONYMOUS PERMISSION
102 #======================================================================
101 #======================================================================
103 if action in ['pull', 'push']:
102 if action in ['pull', 'push']:
104 anonymous_user = self.__get_user('default')
103 anonymous_user = self.__get_user('default')
105 username = anonymous_user.username
104 username = anonymous_user.username
106 anonymous_perm = self._check_permission(action, anonymous_user,
105 anonymous_perm = self._check_permission(action, anonymous_user,
107 repo_name)
106 repo_name)
108
107
109 if anonymous_perm is not True or anonymous_user.active is False:
108 if anonymous_perm is not True or anonymous_user.active is False:
110 if anonymous_perm is not True:
109 if anonymous_perm is not True:
111 log.debug('Not enough credentials to access this '
110 log.debug('Not enough credentials to access this '
112 'repository as anonymous user')
111 'repository as anonymous user')
113 if anonymous_user.active is False:
112 if anonymous_user.active is False:
114 log.debug('Anonymous access is disabled, running '
113 log.debug('Anonymous access is disabled, running '
115 'authentication')
114 'authentication')
116 #==============================================================
115 #==============================================================
117 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
116 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
118 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
117 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
119 #==============================================================
118 #==============================================================
120
119
121 # Attempting to retrieve username from the container
120 # Attempting to retrieve username from the container
122 username = get_container_username(environ, self.config)
121 username = get_container_username(environ, self.config)
123
122
124 # If not authenticated by the container, running basic auth
123 # If not authenticated by the container, running basic auth
125 if not username:
124 if not username:
126 self.authenticate.realm = \
125 self.authenticate.realm = \
127 safe_str(self.config['rhodecode_realm'])
126 safe_str(self.config['rhodecode_realm'])
128 result = self.authenticate(environ)
127 result = self.authenticate(environ)
129 if isinstance(result, str):
128 if isinstance(result, str):
130 AUTH_TYPE.update(environ, 'basic')
129 AUTH_TYPE.update(environ, 'basic')
131 REMOTE_USER.update(environ, result)
130 REMOTE_USER.update(environ, result)
132 username = result
131 username = result
133 else:
132 else:
134 return result.wsgi_application(environ, start_response)
133 return result.wsgi_application(environ, start_response)
135
134
136 #==============================================================
135 #==============================================================
137 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
136 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
138 #==============================================================
137 #==============================================================
139 try:
138 try:
140 user = self.__get_user(username)
139 user = self.__get_user(username)
141 if user is None or not user.active:
140 if user is None or not user.active:
142 return HTTPForbidden()(environ, start_response)
141 return HTTPForbidden()(environ, start_response)
143 username = user.username
142 username = user.username
144 except:
143 except:
145 log.error(traceback.format_exc())
144 log.error(traceback.format_exc())
146 return HTTPInternalServerError()(environ, start_response)
145 return HTTPInternalServerError()(environ, start_response)
147
146
148 #check permissions for this repository
147 #check permissions for this repository
149 perm = self._check_permission(action, user, repo_name)
148 perm = self._check_permission(action, user, repo_name)
150 if perm is not True:
149 if perm is not True:
151 return HTTPForbidden()(environ, start_response)
150 return HTTPForbidden()(environ, start_response)
152
151
153 # extras are injected into mercurial UI object and later available
152 # extras are injected into mercurial UI object and later available
154 # in hg hooks executed by rhodecode
153 # in hg hooks executed by rhodecode
155 from rhodecode import CONFIG
154 from rhodecode import CONFIG
156 extras = {
155 extras = {
157 'ip': ipaddr,
156 'ip': ipaddr,
158 'username': username,
157 'username': username,
159 'action': action,
158 'action': action,
160 'repository': repo_name,
159 'repository': repo_name,
161 'scm': 'hg',
160 'scm': 'hg',
162 'config': CONFIG['__file__'],
161 'config': CONFIG['__file__'],
163 'make_lock': None,
162 'make_lock': None,
164 'locked_by': [None, None]
163 'locked_by': [None, None]
165 }
164 }
166 #======================================================================
165 #======================================================================
167 # MERCURIAL REQUEST HANDLING
166 # MERCURIAL REQUEST HANDLING
168 #======================================================================
167 #======================================================================
169 repo_path = os.path.join(safe_str(self.basepath), safe_str(repo_name))
168 repo_path = os.path.join(safe_str(self.basepath), safe_str(repo_name))
170 log.debug('Repository path is %s' % repo_path)
169 log.debug('Repository path is %s' % repo_path)
171
170
172 # CHECK LOCKING only if it's not ANONYMOUS USER
171 # CHECK LOCKING only if it's not ANONYMOUS USER
173 if username != User.DEFAULT_USER:
172 if username != User.DEFAULT_USER:
174 log.debug('Checking locking on repository')
173 log.debug('Checking locking on repository')
175 (make_lock,
174 (make_lock,
176 locked,
175 locked,
177 locked_by) = self._check_locking_state(
176 locked_by) = self._check_locking_state(
178 environ=environ, action=action,
177 environ=environ, action=action,
179 repo=repo_name, user_id=user.user_id
178 repo=repo_name, user_id=user.user_id
180 )
179 )
181 # store the make_lock for later evaluation in hooks
180 # store the make_lock for later evaluation in hooks
182 extras.update({'make_lock': make_lock,
181 extras.update({'make_lock': make_lock,
183 'locked_by': locked_by})
182 'locked_by': locked_by})
184
183
185 # set the environ variables for this request
184 # set the environ variables for this request
186 os.environ['RC_SCM_DATA'] = json.dumps(extras)
185 os.environ['RC_SCM_DATA'] = json.dumps(extras)
186 fix_PATH()
187 log.debug('HOOKS extras is %s' % extras)
187 log.debug('HOOKS extras is %s' % extras)
188 baseui = make_ui('db')
188 baseui = make_ui('db')
189 self.__inject_extras(repo_path, baseui, extras)
189 self.__inject_extras(repo_path, baseui, extras)
190
190
191 try:
191 try:
192 # invalidate cache on push
192 # invalidate cache on push
193 if action == 'push':
193 if action == 'push':
194 self._invalidate_cache(repo_name)
194 self._invalidate_cache(repo_name)
195 log.info('%s action on HG repo "%s"' % (action, repo_name))
195 log.info('%s action on HG repo "%s"' % (action, repo_name))
196 app = self.__make_app(repo_path, baseui, extras)
196 app = self.__make_app(repo_path, baseui, extras)
197 return app(environ, start_response)
197 return app(environ, start_response)
198 except RepoError, e:
198 except RepoError, e:
199 if str(e).find('not found') != -1:
199 if str(e).find('not found') != -1:
200 return HTTPNotFound()(environ, start_response)
200 return HTTPNotFound()(environ, start_response)
201 except HTTPLockedRC, e:
201 except HTTPLockedRC, e:
202 log.debug('Repositry LOCKED ret code 423!')
202 log.debug('Repositry LOCKED ret code 423!')
203 return e(environ, start_response)
203 return e(environ, start_response)
204 except Exception:
204 except Exception:
205 log.error(traceback.format_exc())
205 log.error(traceback.format_exc())
206 return HTTPInternalServerError()(environ, start_response)
206 return HTTPInternalServerError()(environ, start_response)
207
207
208 def __make_app(self, repo_name, baseui, extras):
208 def __make_app(self, repo_name, baseui, extras):
209 """
209 """
210 Make an wsgi application using hgweb, and inject generated baseui
210 Make an wsgi application using hgweb, and inject generated baseui
211 instance, additionally inject some extras into ui object
211 instance, additionally inject some extras into ui object
212 """
212 """
213 return hgweb_mod.hgweb(repo_name, name=repo_name, baseui=baseui)
213 return hgweb_mod.hgweb(repo_name, name=repo_name, baseui=baseui)
214
214
215 def __get_repository(self, environ):
215 def __get_repository(self, environ):
216 """
216 """
217 Get's repository name out of PATH_INFO header
217 Get's repository name out of PATH_INFO header
218
218
219 :param environ: environ where PATH_INFO is stored
219 :param environ: environ where PATH_INFO is stored
220 """
220 """
221 try:
221 try:
222 environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
222 environ['PATH_INFO'] = self._get_by_id(environ['PATH_INFO'])
223 repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
223 repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])
224 if repo_name.endswith('/'):
224 if repo_name.endswith('/'):
225 repo_name = repo_name.rstrip('/')
225 repo_name = repo_name.rstrip('/')
226 except:
226 except:
227 log.error(traceback.format_exc())
227 log.error(traceback.format_exc())
228 raise
228 raise
229
229
230 return repo_name
230 return repo_name
231
231
232 def __get_user(self, username):
232 def __get_user(self, username):
233 return User.get_by_username(username)
233 return User.get_by_username(username)
234
234
235 def __get_action(self, environ):
235 def __get_action(self, environ):
236 """
236 """
237 Maps mercurial request commands into a clone,pull or push command.
237 Maps mercurial request commands into a clone,pull or push command.
238 This should always return a valid command string
238 This should always return a valid command string
239
239
240 :param environ:
240 :param environ:
241 """
241 """
242 mapping = {'changegroup': 'pull',
242 mapping = {'changegroup': 'pull',
243 'changegroupsubset': 'pull',
243 'changegroupsubset': 'pull',
244 'stream_out': 'pull',
244 'stream_out': 'pull',
245 'listkeys': 'pull',
245 'listkeys': 'pull',
246 'unbundle': 'push',
246 'unbundle': 'push',
247 'pushkey': 'push', }
247 'pushkey': 'push', }
248 for qry in environ['QUERY_STRING'].split('&'):
248 for qry in environ['QUERY_STRING'].split('&'):
249 if qry.startswith('cmd'):
249 if qry.startswith('cmd'):
250 cmd = qry.split('=')[-1]
250 cmd = qry.split('=')[-1]
251 if cmd in mapping:
251 if cmd in mapping:
252 return mapping[cmd]
252 return mapping[cmd]
253
253
254 return 'pull'
254 return 'pull'
255
255
256 raise Exception('Unable to detect pull/push action !!'
256 raise Exception('Unable to detect pull/push action !!'
257 'Are you using non standard command or client ?')
257 'Are you using non standard command or client ?')
258
258
259 def __inject_extras(self, repo_path, baseui, extras={}):
259 def __inject_extras(self, repo_path, baseui, extras={}):
260 """
260 """
261 Injects some extra params into baseui instance
261 Injects some extra params into baseui instance
262
262
263 also overwrites global settings with those takes from local hgrc file
263 also overwrites global settings with those takes from local hgrc file
264
264
265 :param baseui: baseui instance
265 :param baseui: baseui instance
266 :param extras: dict with extra params to put into baseui
266 :param extras: dict with extra params to put into baseui
267 """
267 """
268
268
269 hgrc = os.path.join(repo_path, '.hg', 'hgrc')
269 hgrc = os.path.join(repo_path, '.hg', 'hgrc')
270
270
271 # make our hgweb quiet so it doesn't print output
271 # make our hgweb quiet so it doesn't print output
272 baseui.setconfig('ui', 'quiet', 'true')
272 baseui.setconfig('ui', 'quiet', 'true')
273
273
274 #inject some additional parameters that will be available in ui
274 #inject some additional parameters that will be available in ui
275 #for hooks
275 #for hooks
276 for k, v in extras.items():
276 for k, v in extras.items():
277 baseui.setconfig('rhodecode_extras', k, v)
277 baseui.setconfig('rhodecode_extras', k, v)
278
278
279 repoui = make_ui('file', hgrc, False)
279 repoui = make_ui('file', hgrc, False)
280
280
281 if repoui:
281 if repoui:
282 #overwrite our ui instance with the section from hgrc file
282 #overwrite our ui instance with the section from hgrc file
283 for section in ui_sections:
283 for section in ui_sections:
284 for k, v in repoui.configitems(section):
284 for k, v in repoui.configitems(section):
285 baseui.setconfig(section, k, v)
285 baseui.setconfig(section, k, v)
@@ -1,483 +1,499 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 rhodecode.lib.utils
3 rhodecode.lib.utils
4 ~~~~~~~~~~~~~~~~~~~
4 ~~~~~~~~~~~~~~~~~~~
5
5
6 Some simple helper functions
6 Some simple helper functions
7
7
8 :created_on: Jan 5, 2011
8 :created_on: Jan 5, 2011
9 :author: marcink
9 :author: marcink
10 :copyright: (C) 2011-2012 Marcin Kuzminski <marcin@python-works.com>
10 :copyright: (C) 2011-2012 Marcin Kuzminski <marcin@python-works.com>
11 :license: GPLv3, see COPYING for more details.
11 :license: GPLv3, see COPYING for more details.
12 """
12 """
13 # This program is free software: you can redistribute it and/or modify
13 # This program is free software: you can redistribute it and/or modify
14 # it under the terms of the GNU General Public License as published by
14 # it under the terms of the GNU General Public License as published by
15 # the Free Software Foundation, either version 3 of the License, or
15 # the Free Software Foundation, either version 3 of the License, or
16 # (at your option) any later version.
16 # (at your option) any later version.
17 #
17 #
18 # This program is distributed in the hope that it will be useful,
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # GNU General Public License for more details.
21 # GNU General Public License for more details.
22 #
22 #
23 # You should have received a copy of the GNU General Public License
23 # You should have received a copy of the GNU General Public License
24 # along with this program. If not, see <http://www.gnu.org/licenses/>.
24 # along with this program. If not, see <http://www.gnu.org/licenses/>.
25
25
26 import re
26 import re
27 import time
27 import time
28 import datetime
28 import datetime
29 from pylons.i18n.translation import _, ungettext
29 from pylons.i18n.translation import _, ungettext
30 from rhodecode.lib.vcs.utils.lazy import LazyProperty
30 from rhodecode.lib.vcs.utils.lazy import LazyProperty
31
31
32
32
33 def __get_lem():
33 def __get_lem():
34 """
34 """
35 Get language extension map based on what's inside pygments lexers
35 Get language extension map based on what's inside pygments lexers
36 """
36 """
37 from pygments import lexers
37 from pygments import lexers
38 from string import lower
38 from string import lower
39 from collections import defaultdict
39 from collections import defaultdict
40
40
41 d = defaultdict(lambda: [])
41 d = defaultdict(lambda: [])
42
42
43 def __clean(s):
43 def __clean(s):
44 s = s.lstrip('*')
44 s = s.lstrip('*')
45 s = s.lstrip('.')
45 s = s.lstrip('.')
46
46
47 if s.find('[') != -1:
47 if s.find('[') != -1:
48 exts = []
48 exts = []
49 start, stop = s.find('['), s.find(']')
49 start, stop = s.find('['), s.find(']')
50
50
51 for suffix in s[start + 1:stop]:
51 for suffix in s[start + 1:stop]:
52 exts.append(s[:s.find('[')] + suffix)
52 exts.append(s[:s.find('[')] + suffix)
53 return map(lower, exts)
53 return map(lower, exts)
54 else:
54 else:
55 return map(lower, [s])
55 return map(lower, [s])
56
56
57 for lx, t in sorted(lexers.LEXERS.items()):
57 for lx, t in sorted(lexers.LEXERS.items()):
58 m = map(__clean, t[-2])
58 m = map(__clean, t[-2])
59 if m:
59 if m:
60 m = reduce(lambda x, y: x + y, m)
60 m = reduce(lambda x, y: x + y, m)
61 for ext in m:
61 for ext in m:
62 desc = lx.replace('Lexer', '')
62 desc = lx.replace('Lexer', '')
63 d[ext].append(desc)
63 d[ext].append(desc)
64
64
65 return dict(d)
65 return dict(d)
66
66
67 def str2bool(_str):
67 def str2bool(_str):
68 """
68 """
69 returs True/False value from given string, it tries to translate the
69 returs True/False value from given string, it tries to translate the
70 string into boolean
70 string into boolean
71
71
72 :param _str: string value to translate into boolean
72 :param _str: string value to translate into boolean
73 :rtype: boolean
73 :rtype: boolean
74 :returns: boolean from given string
74 :returns: boolean from given string
75 """
75 """
76 if _str is None:
76 if _str is None:
77 return False
77 return False
78 if _str in (True, False):
78 if _str in (True, False):
79 return _str
79 return _str
80 _str = str(_str).strip().lower()
80 _str = str(_str).strip().lower()
81 return _str in ('t', 'true', 'y', 'yes', 'on', '1')
81 return _str in ('t', 'true', 'y', 'yes', 'on', '1')
82
82
83
83
84 def convert_line_endings(line, mode):
84 def convert_line_endings(line, mode):
85 """
85 """
86 Converts a given line "line end" accordingly to given mode
86 Converts a given line "line end" accordingly to given mode
87
87
88 Available modes are::
88 Available modes are::
89 0 - Unix
89 0 - Unix
90 1 - Mac
90 1 - Mac
91 2 - DOS
91 2 - DOS
92
92
93 :param line: given line to convert
93 :param line: given line to convert
94 :param mode: mode to convert to
94 :param mode: mode to convert to
95 :rtype: str
95 :rtype: str
96 :return: converted line according to mode
96 :return: converted line according to mode
97 """
97 """
98 from string import replace
98 from string import replace
99
99
100 if mode == 0:
100 if mode == 0:
101 line = replace(line, '\r\n', '\n')
101 line = replace(line, '\r\n', '\n')
102 line = replace(line, '\r', '\n')
102 line = replace(line, '\r', '\n')
103 elif mode == 1:
103 elif mode == 1:
104 line = replace(line, '\r\n', '\r')
104 line = replace(line, '\r\n', '\r')
105 line = replace(line, '\n', '\r')
105 line = replace(line, '\n', '\r')
106 elif mode == 2:
106 elif mode == 2:
107 line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
107 line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
108 return line
108 return line
109
109
110
110
111 def detect_mode(line, default):
111 def detect_mode(line, default):
112 """
112 """
113 Detects line break for given line, if line break couldn't be found
113 Detects line break for given line, if line break couldn't be found
114 given default value is returned
114 given default value is returned
115
115
116 :param line: str line
116 :param line: str line
117 :param default: default
117 :param default: default
118 :rtype: int
118 :rtype: int
119 :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
119 :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
120 """
120 """
121 if line.endswith('\r\n'):
121 if line.endswith('\r\n'):
122 return 2
122 return 2
123 elif line.endswith('\n'):
123 elif line.endswith('\n'):
124 return 0
124 return 0
125 elif line.endswith('\r'):
125 elif line.endswith('\r'):
126 return 1
126 return 1
127 else:
127 else:
128 return default
128 return default
129
129
130
130
131 def generate_api_key(username, salt=None):
131 def generate_api_key(username, salt=None):
132 """
132 """
133 Generates unique API key for given username, if salt is not given
133 Generates unique API key for given username, if salt is not given
134 it'll be generated from some random string
134 it'll be generated from some random string
135
135
136 :param username: username as string
136 :param username: username as string
137 :param salt: salt to hash generate KEY
137 :param salt: salt to hash generate KEY
138 :rtype: str
138 :rtype: str
139 :returns: sha1 hash from username+salt
139 :returns: sha1 hash from username+salt
140 """
140 """
141 from tempfile import _RandomNameSequence
141 from tempfile import _RandomNameSequence
142 import hashlib
142 import hashlib
143
143
144 if salt is None:
144 if salt is None:
145 salt = _RandomNameSequence().next()
145 salt = _RandomNameSequence().next()
146
146
147 return hashlib.sha1(username + salt).hexdigest()
147 return hashlib.sha1(username + salt).hexdigest()
148
148
149
149
150 def safe_int(val, default=None):
150 def safe_int(val, default=None):
151 """
151 """
152 Returns int() of val if val is not convertable to int use default
152 Returns int() of val if val is not convertable to int use default
153 instead
153 instead
154
154
155 :param val:
155 :param val:
156 :param default:
156 :param default:
157 """
157 """
158
158
159 try:
159 try:
160 val = int(val)
160 val = int(val)
161 except ValueError:
161 except ValueError:
162 val = default
162 val = default
163
163
164 return val
164 return val
165
165
166
166
167 def safe_unicode(str_, from_encoding=None):
167 def safe_unicode(str_, from_encoding=None):
168 """
168 """
169 safe unicode function. Does few trick to turn str_ into unicode
169 safe unicode function. Does few trick to turn str_ into unicode
170
170
171 In case of UnicodeDecode error we try to return it with encoding detected
171 In case of UnicodeDecode error we try to return it with encoding detected
172 by chardet library if it fails fallback to unicode with errors replaced
172 by chardet library if it fails fallback to unicode with errors replaced
173
173
174 :param str_: string to decode
174 :param str_: string to decode
175 :rtype: unicode
175 :rtype: unicode
176 :returns: unicode object
176 :returns: unicode object
177 """
177 """
178 if isinstance(str_, unicode):
178 if isinstance(str_, unicode):
179 return str_
179 return str_
180
180
181 if not from_encoding:
181 if not from_encoding:
182 import rhodecode
182 import rhodecode
183 DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
183 DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
184 from_encoding = DEFAULT_ENCODING
184 from_encoding = DEFAULT_ENCODING
185
185
186 try:
186 try:
187 return unicode(str_)
187 return unicode(str_)
188 except UnicodeDecodeError:
188 except UnicodeDecodeError:
189 pass
189 pass
190
190
191 try:
191 try:
192 return unicode(str_, from_encoding)
192 return unicode(str_, from_encoding)
193 except UnicodeDecodeError:
193 except UnicodeDecodeError:
194 pass
194 pass
195
195
196 try:
196 try:
197 import chardet
197 import chardet
198 encoding = chardet.detect(str_)['encoding']
198 encoding = chardet.detect(str_)['encoding']
199 if encoding is None:
199 if encoding is None:
200 raise Exception()
200 raise Exception()
201 return str_.decode(encoding)
201 return str_.decode(encoding)
202 except (ImportError, UnicodeDecodeError, Exception):
202 except (ImportError, UnicodeDecodeError, Exception):
203 return unicode(str_, from_encoding, 'replace')
203 return unicode(str_, from_encoding, 'replace')
204
204
205
205
206 def safe_str(unicode_, to_encoding=None):
206 def safe_str(unicode_, to_encoding=None):
207 """
207 """
208 safe str function. Does few trick to turn unicode_ into string
208 safe str function. Does few trick to turn unicode_ into string
209
209
210 In case of UnicodeEncodeError we try to return it with encoding detected
210 In case of UnicodeEncodeError we try to return it with encoding detected
211 by chardet library if it fails fallback to string with errors replaced
211 by chardet library if it fails fallback to string with errors replaced
212
212
213 :param unicode_: unicode to encode
213 :param unicode_: unicode to encode
214 :rtype: str
214 :rtype: str
215 :returns: str object
215 :returns: str object
216 """
216 """
217
217
218 # if it's not basestr cast to str
218 # if it's not basestr cast to str
219 if not isinstance(unicode_, basestring):
219 if not isinstance(unicode_, basestring):
220 return str(unicode_)
220 return str(unicode_)
221
221
222 if isinstance(unicode_, str):
222 if isinstance(unicode_, str):
223 return unicode_
223 return unicode_
224
224
225 if not to_encoding:
225 if not to_encoding:
226 import rhodecode
226 import rhodecode
227 DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
227 DEFAULT_ENCODING = rhodecode.CONFIG.get('default_encoding','utf8')
228 to_encoding = DEFAULT_ENCODING
228 to_encoding = DEFAULT_ENCODING
229
229
230 try:
230 try:
231 return unicode_.encode(to_encoding)
231 return unicode_.encode(to_encoding)
232 except UnicodeEncodeError:
232 except UnicodeEncodeError:
233 pass
233 pass
234
234
235 try:
235 try:
236 import chardet
236 import chardet
237 encoding = chardet.detect(unicode_)['encoding']
237 encoding = chardet.detect(unicode_)['encoding']
238 if encoding is None:
238 if encoding is None:
239 raise UnicodeEncodeError()
239 raise UnicodeEncodeError()
240
240
241 return unicode_.encode(encoding)
241 return unicode_.encode(encoding)
242 except (ImportError, UnicodeEncodeError):
242 except (ImportError, UnicodeEncodeError):
243 return unicode_.encode(to_encoding, 'replace')
243 return unicode_.encode(to_encoding, 'replace')
244
244
245 return safe_str
245 return safe_str
246
246
247
247
248 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
248 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
249 """
249 """
250 Custom engine_from_config functions that makes sure we use NullPool for
250 Custom engine_from_config functions that makes sure we use NullPool for
251 file based sqlite databases. This prevents errors on sqlite. This only
251 file based sqlite databases. This prevents errors on sqlite. This only
252 applies to sqlalchemy versions < 0.7.0
252 applies to sqlalchemy versions < 0.7.0
253
253
254 """
254 """
255 import sqlalchemy
255 import sqlalchemy
256 from sqlalchemy import engine_from_config as efc
256 from sqlalchemy import engine_from_config as efc
257 import logging
257 import logging
258
258
259 if int(sqlalchemy.__version__.split('.')[1]) < 7:
259 if int(sqlalchemy.__version__.split('.')[1]) < 7:
260
260
261 # This solution should work for sqlalchemy < 0.7.0, and should use
261 # This solution should work for sqlalchemy < 0.7.0, and should use
262 # proxy=TimerProxy() for execution time profiling
262 # proxy=TimerProxy() for execution time profiling
263
263
264 from sqlalchemy.pool import NullPool
264 from sqlalchemy.pool import NullPool
265 url = configuration[prefix + 'url']
265 url = configuration[prefix + 'url']
266
266
267 if url.startswith('sqlite'):
267 if url.startswith('sqlite'):
268 kwargs.update({'poolclass': NullPool})
268 kwargs.update({'poolclass': NullPool})
269 return efc(configuration, prefix, **kwargs)
269 return efc(configuration, prefix, **kwargs)
270 else:
270 else:
271 import time
271 import time
272 from sqlalchemy import event
272 from sqlalchemy import event
273 from sqlalchemy.engine import Engine
273 from sqlalchemy.engine import Engine
274
274
275 log = logging.getLogger('sqlalchemy.engine')
275 log = logging.getLogger('sqlalchemy.engine')
276 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
276 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
277 engine = efc(configuration, prefix, **kwargs)
277 engine = efc(configuration, prefix, **kwargs)
278
278
279 def color_sql(sql):
279 def color_sql(sql):
280 COLOR_SEQ = "\033[1;%dm"
280 COLOR_SEQ = "\033[1;%dm"
281 COLOR_SQL = YELLOW
281 COLOR_SQL = YELLOW
282 normal = '\x1b[0m'
282 normal = '\x1b[0m'
283 return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal])
283 return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal])
284
284
285 if configuration['debug']:
285 if configuration['debug']:
286 #attach events only for debug configuration
286 #attach events only for debug configuration
287
287
288 def before_cursor_execute(conn, cursor, statement,
288 def before_cursor_execute(conn, cursor, statement,
289 parameters, context, executemany):
289 parameters, context, executemany):
290 context._query_start_time = time.time()
290 context._query_start_time = time.time()
291 log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
291 log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
292
292
293 def after_cursor_execute(conn, cursor, statement,
293 def after_cursor_execute(conn, cursor, statement,
294 parameters, context, executemany):
294 parameters, context, executemany):
295 total = time.time() - context._query_start_time
295 total = time.time() - context._query_start_time
296 log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total))
296 log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total))
297
297
298 event.listen(engine, "before_cursor_execute",
298 event.listen(engine, "before_cursor_execute",
299 before_cursor_execute)
299 before_cursor_execute)
300 event.listen(engine, "after_cursor_execute",
300 event.listen(engine, "after_cursor_execute",
301 after_cursor_execute)
301 after_cursor_execute)
302
302
303 return engine
303 return engine
304
304
305
305
306 def age(prevdate):
306 def age(prevdate):
307 """
307 """
308 turns a datetime into an age string.
308 turns a datetime into an age string.
309
309
310 :param prevdate: datetime object
310 :param prevdate: datetime object
311 :rtype: unicode
311 :rtype: unicode
312 :returns: unicode words describing age
312 :returns: unicode words describing age
313 """
313 """
314
314
315 order = ['year', 'month', 'day', 'hour', 'minute', 'second']
315 order = ['year', 'month', 'day', 'hour', 'minute', 'second']
316 deltas = {}
316 deltas = {}
317
317
318 # Get date parts deltas
318 # Get date parts deltas
319 now = datetime.datetime.now()
319 now = datetime.datetime.now()
320 for part in order:
320 for part in order:
321 deltas[part] = getattr(now, part) - getattr(prevdate, part)
321 deltas[part] = getattr(now, part) - getattr(prevdate, part)
322
322
323 # Fix negative offsets (there is 1 second between 10:59:59 and 11:00:00,
323 # Fix negative offsets (there is 1 second between 10:59:59 and 11:00:00,
324 # not 1 hour, -59 minutes and -59 seconds)
324 # not 1 hour, -59 minutes and -59 seconds)
325
325
326 for num, length in [(5, 60), (4, 60), (3, 24)]: # seconds, minutes, hours
326 for num, length in [(5, 60), (4, 60), (3, 24)]: # seconds, minutes, hours
327 part = order[num]
327 part = order[num]
328 carry_part = order[num - 1]
328 carry_part = order[num - 1]
329
329
330 if deltas[part] < 0:
330 if deltas[part] < 0:
331 deltas[part] += length
331 deltas[part] += length
332 deltas[carry_part] -= 1
332 deltas[carry_part] -= 1
333
333
334 # Same thing for days except that the increment depends on the (variable)
334 # Same thing for days except that the increment depends on the (variable)
335 # number of days in the month
335 # number of days in the month
336 month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
336 month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
337 if deltas['day'] < 0:
337 if deltas['day'] < 0:
338 if prevdate.month == 2 and (prevdate.year % 4 == 0 and
338 if prevdate.month == 2 and (prevdate.year % 4 == 0 and
339 (prevdate.year % 100 != 0 or prevdate.year % 400 == 0)):
339 (prevdate.year % 100 != 0 or prevdate.year % 400 == 0)):
340 deltas['day'] += 29
340 deltas['day'] += 29
341 else:
341 else:
342 deltas['day'] += month_lengths[prevdate.month - 1]
342 deltas['day'] += month_lengths[prevdate.month - 1]
343
343
344 deltas['month'] -= 1
344 deltas['month'] -= 1
345
345
346 if deltas['month'] < 0:
346 if deltas['month'] < 0:
347 deltas['month'] += 12
347 deltas['month'] += 12
348 deltas['year'] -= 1
348 deltas['year'] -= 1
349
349
350 # Format the result
350 # Format the result
351 fmt_funcs = {
351 fmt_funcs = {
352 'year': lambda d: ungettext(u'%d year', '%d years', d) % d,
352 'year': lambda d: ungettext(u'%d year', '%d years', d) % d,
353 'month': lambda d: ungettext(u'%d month', '%d months', d) % d,
353 'month': lambda d: ungettext(u'%d month', '%d months', d) % d,
354 'day': lambda d: ungettext(u'%d day', '%d days', d) % d,
354 'day': lambda d: ungettext(u'%d day', '%d days', d) % d,
355 'hour': lambda d: ungettext(u'%d hour', '%d hours', d) % d,
355 'hour': lambda d: ungettext(u'%d hour', '%d hours', d) % d,
356 'minute': lambda d: ungettext(u'%d minute', '%d minutes', d) % d,
356 'minute': lambda d: ungettext(u'%d minute', '%d minutes', d) % d,
357 'second': lambda d: ungettext(u'%d second', '%d seconds', d) % d,
357 'second': lambda d: ungettext(u'%d second', '%d seconds', d) % d,
358 }
358 }
359
359
360 for i, part in enumerate(order):
360 for i, part in enumerate(order):
361 value = deltas[part]
361 value = deltas[part]
362 if value == 0:
362 if value == 0:
363 continue
363 continue
364
364
365 if i < 5:
365 if i < 5:
366 sub_part = order[i + 1]
366 sub_part = order[i + 1]
367 sub_value = deltas[sub_part]
367 sub_value = deltas[sub_part]
368 else:
368 else:
369 sub_value = 0
369 sub_value = 0
370
370
371 if sub_value == 0:
371 if sub_value == 0:
372 return _(u'%s ago') % fmt_funcs[part](value)
372 return _(u'%s ago') % fmt_funcs[part](value)
373
373
374 return _(u'%s and %s ago') % (fmt_funcs[part](value),
374 return _(u'%s and %s ago') % (fmt_funcs[part](value),
375 fmt_funcs[sub_part](sub_value))
375 fmt_funcs[sub_part](sub_value))
376
376
377 return _(u'just now')
377 return _(u'just now')
378
378
379
379
380 def uri_filter(uri):
380 def uri_filter(uri):
381 """
381 """
382 Removes user:password from given url string
382 Removes user:password from given url string
383
383
384 :param uri:
384 :param uri:
385 :rtype: unicode
385 :rtype: unicode
386 :returns: filtered list of strings
386 :returns: filtered list of strings
387 """
387 """
388 if not uri:
388 if not uri:
389 return ''
389 return ''
390
390
391 proto = ''
391 proto = ''
392
392
393 for pat in ('https://', 'http://'):
393 for pat in ('https://', 'http://'):
394 if uri.startswith(pat):
394 if uri.startswith(pat):
395 uri = uri[len(pat):]
395 uri = uri[len(pat):]
396 proto = pat
396 proto = pat
397 break
397 break
398
398
399 # remove passwords and username
399 # remove passwords and username
400 uri = uri[uri.find('@') + 1:]
400 uri = uri[uri.find('@') + 1:]
401
401
402 # get the port
402 # get the port
403 cred_pos = uri.find(':')
403 cred_pos = uri.find(':')
404 if cred_pos == -1:
404 if cred_pos == -1:
405 host, port = uri, None
405 host, port = uri, None
406 else:
406 else:
407 host, port = uri[:cred_pos], uri[cred_pos + 1:]
407 host, port = uri[:cred_pos], uri[cred_pos + 1:]
408
408
409 return filter(None, [proto, host, port])
409 return filter(None, [proto, host, port])
410
410
411
411
412 def credentials_filter(uri):
412 def credentials_filter(uri):
413 """
413 """
414 Returns a url with removed credentials
414 Returns a url with removed credentials
415
415
416 :param uri:
416 :param uri:
417 """
417 """
418
418
419 uri = uri_filter(uri)
419 uri = uri_filter(uri)
420 #check if we have port
420 #check if we have port
421 if len(uri) > 2 and uri[2]:
421 if len(uri) > 2 and uri[2]:
422 uri[2] = ':' + uri[2]
422 uri[2] = ':' + uri[2]
423
423
424 return ''.join(uri)
424 return ''.join(uri)
425
425
426
426
427 def get_changeset_safe(repo, rev):
427 def get_changeset_safe(repo, rev):
428 """
428 """
429 Safe version of get_changeset if this changeset doesn't exists for a
429 Safe version of get_changeset if this changeset doesn't exists for a
430 repo it returns a Dummy one instead
430 repo it returns a Dummy one instead
431
431
432 :param repo:
432 :param repo:
433 :param rev:
433 :param rev:
434 """
434 """
435 from rhodecode.lib.vcs.backends.base import BaseRepository
435 from rhodecode.lib.vcs.backends.base import BaseRepository
436 from rhodecode.lib.vcs.exceptions import RepositoryError
436 from rhodecode.lib.vcs.exceptions import RepositoryError
437 from rhodecode.lib.vcs.backends.base import EmptyChangeset
437 from rhodecode.lib.vcs.backends.base import EmptyChangeset
438 if not isinstance(repo, BaseRepository):
438 if not isinstance(repo, BaseRepository):
439 raise Exception('You must pass an Repository '
439 raise Exception('You must pass an Repository '
440 'object as first argument got %s', type(repo))
440 'object as first argument got %s', type(repo))
441
441
442 try:
442 try:
443 cs = repo.get_changeset(rev)
443 cs = repo.get_changeset(rev)
444 except RepositoryError:
444 except RepositoryError:
445 cs = EmptyChangeset(requested_revision=rev)
445 cs = EmptyChangeset(requested_revision=rev)
446 return cs
446 return cs
447
447
448
448
449 def datetime_to_time(dt):
449 def datetime_to_time(dt):
450 if dt:
450 if dt:
451 return time.mktime(dt.timetuple())
451 return time.mktime(dt.timetuple())
452
452
453
453
454 def time_to_datetime(tm):
454 def time_to_datetime(tm):
455 if tm:
455 if tm:
456 if isinstance(tm, basestring):
456 if isinstance(tm, basestring):
457 try:
457 try:
458 tm = float(tm)
458 tm = float(tm)
459 except ValueError:
459 except ValueError:
460 return
460 return
461 return datetime.datetime.fromtimestamp(tm)
461 return datetime.datetime.fromtimestamp(tm)
462
462
463 MENTIONS_REGEX = r'(?:^@|\s@)([a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+)(?:\s{1})'
463 MENTIONS_REGEX = r'(?:^@|\s@)([a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+)(?:\s{1})'
464
464
465
465
466 def extract_mentioned_users(s):
466 def extract_mentioned_users(s):
467 """
467 """
468 Returns unique usernames from given string s that have @mention
468 Returns unique usernames from given string s that have @mention
469
469
470 :param s: string to get mentions
470 :param s: string to get mentions
471 """
471 """
472 usrs = set()
472 usrs = set()
473 for username in re.findall(MENTIONS_REGEX, s):
473 for username in re.findall(MENTIONS_REGEX, s):
474 usrs.add(username)
474 usrs.add(username)
475
475
476 return sorted(list(usrs), key=lambda k: k.lower())
476 return sorted(list(usrs), key=lambda k: k.lower())
477
477
478
478
479 class AttributeDict(dict):
479 class AttributeDict(dict):
480 def __getattr__(self, attr):
480 def __getattr__(self, attr):
481 return self.get(attr, None)
481 return self.get(attr, None)
482 __setattr__ = dict.__setitem__
482 __setattr__ = dict.__setitem__
483 __delattr__ = dict.__delitem__
483 __delattr__ = dict.__delitem__
484
485
486 def fix_PATH(os_=None):
487 """
488 Get current active python path, and append it to PATH variable to fix issues
489 of subprocess calls and different python versions
490 """
491 import sys
492 if os_ is None:
493 import os
494 else:
495 os = os_
496
497 cur_path = os.path.split(sys.executable)[0]
498 if not os.environ['PATH'].startswith(cur_path):
499 os.environ['PATH'] = '%s:%s' % (cur_path, os.environ['PATH'])
General Comments 0
You need to be logged in to leave comments. Login now