##// END OF EJS Templates
middlewares: all porting for python3
super-admin -
r5082:35aadafc default
parent child Browse files
Show More
@@ -1,100 +1,105 b''
1
1
2 # Copyright (C) 2010-2020 RhodeCode GmbH
2 # Copyright (C) 2010-2020 RhodeCode GmbH
3 #
3 #
4 # This program is free software: you can redistribute it and/or modify
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License, version 3
5 # it under the terms of the GNU Affero General Public License, version 3
6 # (only), as published by the Free Software Foundation.
6 # (only), as published by the Free Software Foundation.
7 #
7 #
8 # This program is distributed in the hope that it will be useful,
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
11 # GNU General Public License for more details.
12 #
12 #
13 # You should have received a copy of the GNU Affero General Public License
13 # You should have received a copy of the GNU Affero General Public License
14 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 #
15 #
16 # This program is dual-licensed. If you wish to learn more about the
16 # This program is dual-licensed. If you wish to learn more about the
17 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # and proprietary license terms, please see https://rhodecode.com/licenses/
18 # and proprietary license terms, please see https://rhodecode.com/licenses/
19
19
20 """
20 """
21 middleware to handle appenlight publishing of errors
21 middleware to handle appenlight publishing of errors
22 """
22 """
23 import logging
23 import logging
24
24
25 log = logging.getLogger(__name__)
25 log = logging.getLogger(__name__)
26
26
27 try:
28 from appenlight_client.exceptions import get_current_traceback
29 except ImportError:
30 def get_current_traceback(*args, **kwargs):
31 return
32
27
33
28 def track_exception(environ):
34 def track_exception(environ):
29 from appenlight_client.exceptions import get_current_traceback
30
35
31 if 'appenlight.client' not in environ:
36 if 'appenlight.client' not in environ:
32 return
37 return
33
38
34 # pass the traceback object to middleware
39 # pass the traceback object to middleware
35 environ['appenlight.__traceback'] = get_current_traceback(
40 environ['appenlight.__traceback'] = get_current_traceback(
36 skip=1,
41 skip=1,
37 show_hidden_frames=True,
42 show_hidden_frames=True,
38 ignore_system_exceptions=True
43 ignore_system_exceptions=True
39 )
44 )
40
45
41
46
42 def track_extra_information(environ, section, value):
47 def track_extra_information(environ, section, value):
43 """
48 """
44 Utility function to attach extra information in case of an error condition.
49 Utility function to attach extra information in case of an error condition.
45
50
46 It will take care of attaching this information to the right place inside
51 It will take care of attaching this information to the right place inside
47 of `environ`, so that the appenight client can pick it up.
52 of `environ`, so that the appenight client can pick it up.
48 """
53 """
49 environ.setdefault('appenlight.extra', {})
54 environ.setdefault('appenlight.extra', {})
50 environ['appenlight.extra'][section] = value
55 environ['appenlight.extra'][section] = value
51
56
52
57
53 def wrap_in_appenlight_if_enabled(app, settings, appenlight_client=None):
58 def wrap_in_appenlight_if_enabled(app, settings, appenlight_client=None):
54 """
59 """
55 Wraps the given `app` for appenlight support.
60 Wraps the given `app` for appenlight support.
56
61
57 .. important::
62 .. important::
58
63
59 Appenlight expects that the wrapper is executed only once, that's why
64 Appenlight expects that the wrapper is executed only once, that's why
60 the parameter `appenlight_client` can be used to pass in an already
65 the parameter `appenlight_client` can be used to pass in an already
61 existing client instance to avoid that decorators are applied more than
66 existing client instance to avoid that decorators are applied more than
62 once.
67 once.
63
68
64 This is in use to support our setup of the vcs related middlewares.
69 This is in use to support our setup of the vcs related middlewares.
65
70
66 """
71 """
67 if settings['appenlight']:
72 if settings['appenlight']:
68 try:
73 try:
69 from appenlight_client import make_appenlight_middleware
74 from appenlight_client import make_appenlight_middleware
70 from appenlight_client.wsgi import AppenlightWSGIWrapper
75 from appenlight_client.wsgi import AppenlightWSGIWrapper
71 except ImportError:
76 except ImportError:
72 log.info('Appenlight packages not present, skipping appenlight setup')
77 log.info('Appenlight packages not present, skipping appenlight setup')
73 return app, appenlight_client
78 return app, appenlight_client
74
79
75 app = RemoteTracebackTracker(app)
80 app = RemoteTracebackTracker(app)
76 if not appenlight_client:
81 if not appenlight_client:
77 app = make_appenlight_middleware(app, settings)
82 app = make_appenlight_middleware(app, settings)
78 appenlight_client = app.appenlight_client
83 appenlight_client = app.appenlight_client
79 else:
84 else:
80 app = AppenlightWSGIWrapper(app, appenlight_client)
85 app = AppenlightWSGIWrapper(app, appenlight_client)
81
86
82 return app, appenlight_client
87 return app, appenlight_client
83
88
84
89
85 class RemoteTracebackTracker(object):
90 class RemoteTracebackTracker(object):
86 """
91 """
87 Utility middleware which forwards VCSServer remote traceback information.
92 Utility middleware which forwards VCSServer remote traceback information.
88 """
93 """
89
94
90 def __init__(self, app):
95 def __init__(self, app):
91 self.application = app
96 self.application = app
92
97
93 def __call__(self, environ, start_response):
98 def __call__(self, environ, start_response):
94 try:
99 try:
95 return self.application(environ, start_response)
100 return self.application(environ, start_response)
96 except Exception as e:
101 except Exception as e:
97 if hasattr(e, '_vcs_server_traceback'):
102 if hasattr(e, '_vcs_server_traceback'):
98 track_extra_information(
103 track_extra_information(
99 environ, 'remote_traceback', e._vcs_server_traceback)
104 environ, 'remote_traceback', e._vcs_server_traceback)
100 raise
105 raise
@@ -1,94 +1,96 b''
1
1
2
2
3 # Copyright (C) 2016-2020 RhodeCode GmbH
3 # Copyright (C) 2016-2020 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import time
21 import time
22 import logging
22 import logging
23
23
24 import rhodecode
24 import rhodecode
25 from rhodecode.lib.auth import AuthUser
25 from rhodecode.lib.auth import AuthUser
26 from rhodecode.lib.base import get_ip_addr, get_user_agent
26 from rhodecode.lib.base import get_ip_addr, get_user_agent
27 from rhodecode.lib.middleware.utils import get_path_info
27 from rhodecode.lib.middleware.utils import get_path_info
28 from rhodecode.lib.utils2 import safe_str, get_current_rhodecode_user
28 from rhodecode.lib.utils2 import safe_str, get_current_rhodecode_user
29
29
30
30
31 log = logging.getLogger(__name__)
31 log = logging.getLogger(__name__)
32
32
33
33
34 class RequestWrapperTween(object):
34 class RequestWrapperTween(object):
35 def __init__(self, handler, registry):
35 def __init__(self, handler, registry):
36 self.handler = handler
36 self.handler = handler
37 self.registry = registry
37 self.registry = registry
38
38
39 # one-time configuration code goes here
39 # one-time configuration code goes here
40
40
41 def _get_user_info(self, request):
41 def _get_user_info(self, request):
42 user = get_current_rhodecode_user(request)
42 user = get_current_rhodecode_user(request)
43 if not user:
43 if not user:
44 user = AuthUser.repr_user(ip=get_ip_addr(request.environ))
44 user = AuthUser.repr_user(ip=get_ip_addr(request.environ))
45 return user
45 return user
46
46
47 def __call__(self, request):
47 def __call__(self, request):
48 start = time.time()
48 start = time.time()
49 log.debug('Starting request time measurement')
49 log.debug('Starting request time measurement')
50 response = None
50 response = None
51 request.req_wrapper_start = start
52
51 try:
53 try:
52 response = self.handler(request)
54 response = self.handler(request)
53 finally:
55 finally:
54 count = request.request_count()
56 count = request.request_count()
55 _ver_ = rhodecode.__version__
57 _ver_ = rhodecode.__version__
56 _path = get_path_info(request.environ)
58 _path = get_path_info(request.environ)
57 _auth_user = self._get_user_info(request)
59 _auth_user = self._get_user_info(request)
58 ip = get_ip_addr(request.environ)
60 ip = get_ip_addr(request.environ)
59 match_route = request.matched_route.name if request.matched_route else "NOT_FOUND"
61 match_route = request.matched_route.name if request.matched_route else "NOT_FOUND"
60 resp_code = getattr(response, 'status_code', 'UNDEFINED')
62 resp_code = getattr(response, 'status_code', 'UNDEFINED')
61
63
62 total = time.time() - start
64 total = time.time() - start
63 log.info(
65 log.info(
64 'Req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s',
66 'Req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s',
65 count, _auth_user, request.environ.get('REQUEST_METHOD'),
67 count, _auth_user, request.environ.get('REQUEST_METHOD'),
66 _path, total, get_user_agent(request. environ), _ver_,
68 _path, total, get_user_agent(request. environ), _ver_,
67 extra={"time": total, "ver": _ver_, "ip": ip,
69 extra={"time": total, "ver": _ver_, "ip": ip,
68 "path": _path, "view_name": match_route, "code": resp_code}
70 "path": _path, "view_name": match_route, "code": resp_code}
69 )
71 )
70
72
71 statsd = request.registry.statsd
73 statsd = request.registry.statsd
72 if statsd:
74 if statsd:
73 elapsed_time_ms = round(1000.0 * total) # use ms only
75 elapsed_time_ms = round(1000.0 * total) # use ms only
74 statsd.timing(
76 statsd.timing(
75 "rhodecode_req_timing.histogram", elapsed_time_ms,
77 "rhodecode_req_timing.histogram", elapsed_time_ms,
76 tags=[
78 tags=[
77 "view_name:{}".format(match_route),
79 "view_name:{}".format(match_route),
78 "code:{}".format(resp_code)
80 "code:{}".format(resp_code)
79 ],
81 ],
80 use_decimals=False
82 use_decimals=False
81 )
83 )
82 statsd.incr(
84 statsd.incr(
83 'rhodecode_req_total', tags=[
85 'rhodecode_req_total', tags=[
84 "view_name:{}".format(match_route),
86 "view_name:{}".format(match_route),
85 "code:{}".format(resp_code)
87 "code:{}".format(resp_code)
86 ])
88 ])
87
89
88 return response
90 return response
89
91
90
92
91 def includeme(config):
93 def includeme(config):
92 config.add_tween(
94 config.add_tween(
93 'rhodecode.lib.middleware.request_wrapper.RequestWrapperTween',
95 'rhodecode.lib.middleware.request_wrapper.RequestWrapperTween',
94 )
96 )
@@ -1,159 +1,161 b''
1
1
2 # Copyright (C) 2010-2020 RhodeCode GmbH
2 # Copyright (C) 2010-2020 RhodeCode GmbH
3 #
3 #
4 # This program is free software: you can redistribute it and/or modify
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License, version 3
5 # it under the terms of the GNU Affero General Public License, version 3
6 # (only), as published by the Free Software Foundation.
6 # (only), as published by the Free Software Foundation.
7 #
7 #
8 # This program is distributed in the hope that it will be useful,
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
11 # GNU General Public License for more details.
12 #
12 #
13 # You should have received a copy of the GNU Affero General Public License
13 # You should have received a copy of the GNU Affero General Public License
14 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 #
15 #
16 # This program is dual-licensed. If you wish to learn more about the
16 # This program is dual-licensed. If you wish to learn more about the
17 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # and proprietary license terms, please see https://rhodecode.com/licenses/
18 # and proprietary license terms, please see https://rhodecode.com/licenses/
19
19
20 """
20 """
21 SimpleHG middleware for handling mercurial protocol request
21 SimpleHG middleware for handling mercurial protocol request
22 (push/clone etc.). It's implemented with basic auth function
22 (push/clone etc.). It's implemented with basic auth function
23 """
23 """
24
24
25 import logging
25 import logging
26 import urllib.parse
26 import urllib.parse
27 import urllib.request, urllib.parse, urllib.error
27 import urllib.request
28 import urllib.parse
29 import urllib.error
28
30
29 from rhodecode.lib import utils
31 from rhodecode.lib import utils
30 from rhodecode.lib.ext_json import json
32 from rhodecode.lib.ext_json import json
31 from rhodecode.lib.middleware import simplevcs
33 from rhodecode.lib.middleware import simplevcs
32 from rhodecode.lib.middleware.utils import get_path_info
34 from rhodecode.lib.middleware.utils import get_path_info
33
35
34 log = logging.getLogger(__name__)
36 log = logging.getLogger(__name__)
35
37
36
38
37 class SimpleHg(simplevcs.SimpleVCS):
39 class SimpleHg(simplevcs.SimpleVCS):
38
40
39 SCM = 'hg'
41 SCM = 'hg'
40
42
41 def _get_repository_name(self, environ):
43 def _get_repository_name(self, environ):
42 """
44 """
43 Gets repository name out of PATH_INFO header
45 Gets repository name out of PATH_INFO header
44
46
45 :param environ: environ where PATH_INFO is stored
47 :param environ: environ where PATH_INFO is stored
46 """
48 """
47 repo_name = get_path_info(environ)
49 repo_name = get_path_info(environ)
48 if repo_name and repo_name.startswith('/'):
50 if repo_name and repo_name.startswith('/'):
49 # remove only the first leading /
51 # remove only the first leading /
50 repo_name = repo_name[1:]
52 repo_name = repo_name[1:]
51 return repo_name.rstrip('/')
53 return repo_name.rstrip('/')
52
54
53 _ACTION_MAPPING = {
55 _ACTION_MAPPING = {
54 'changegroup': 'pull',
56 'changegroup': 'pull',
55 'changegroupsubset': 'pull',
57 'changegroupsubset': 'pull',
56 'getbundle': 'pull',
58 'getbundle': 'pull',
57 'stream_out': 'pull',
59 'stream_out': 'pull',
58 'listkeys': 'pull',
60 'listkeys': 'pull',
59 'between': 'pull',
61 'between': 'pull',
60 'branchmap': 'pull',
62 'branchmap': 'pull',
61 'branches': 'pull',
63 'branches': 'pull',
62 'clonebundles': 'pull',
64 'clonebundles': 'pull',
63 'capabilities': 'pull',
65 'capabilities': 'pull',
64 'debugwireargs': 'pull',
66 'debugwireargs': 'pull',
65 'heads': 'pull',
67 'heads': 'pull',
66 'lookup': 'pull',
68 'lookup': 'pull',
67 'hello': 'pull',
69 'hello': 'pull',
68 'known': 'pull',
70 'known': 'pull',
69
71
70 # largefiles
72 # largefiles
71 'putlfile': 'push',
73 'putlfile': 'push',
72 'getlfile': 'pull',
74 'getlfile': 'pull',
73 'statlfile': 'pull',
75 'statlfile': 'pull',
74 'lheads': 'pull',
76 'lheads': 'pull',
75
77
76 # evolve
78 # evolve
77 'evoext_obshashrange_v1': 'pull',
79 'evoext_obshashrange_v1': 'pull',
78 'evoext_obshash': 'pull',
80 'evoext_obshash': 'pull',
79 'evoext_obshash1': 'pull',
81 'evoext_obshash1': 'pull',
80
82
81 'unbundle': 'push',
83 'unbundle': 'push',
82 'pushkey': 'push',
84 'pushkey': 'push',
83 }
85 }
84
86
85 @classmethod
87 @classmethod
86 def _get_xarg_headers(cls, environ):
88 def _get_xarg_headers(cls, environ):
87 i = 1
89 i = 1
88 chunks = [] # gather chunks stored in multiple 'hgarg_N'
90 chunks = [] # gather chunks stored in multiple 'hgarg_N'
89 while True:
91 while True:
90 head = environ.get('HTTP_X_HGARG_{}'.format(i))
92 head = environ.get('HTTP_X_HGARG_{}'.format(i))
91 if not head:
93 if not head:
92 break
94 break
93 i += 1
95 i += 1
94 chunks.append(urllib.parse.unquote_plus(head))
96 chunks.append(urllib.parse.unquote_plus(head))
95 full_arg = ''.join(chunks)
97 full_arg = ''.join(chunks)
96 pref = 'cmds='
98 pref = 'cmds='
97 if full_arg.startswith(pref):
99 if full_arg.startswith(pref):
98 # strip the cmds= header defining our batch commands
100 # strip the cmds= header defining our batch commands
99 full_arg = full_arg[len(pref):]
101 full_arg = full_arg[len(pref):]
100 cmds = full_arg.split(';')
102 cmds = full_arg.split(';')
101 return cmds
103 return cmds
102
104
103 @classmethod
105 @classmethod
104 def _get_batch_cmd(cls, environ):
106 def _get_batch_cmd(cls, environ):
105 """
107 """
106 Handle batch command send commands. Those are ';' separated commands
108 Handle batch command send commands. Those are ';' separated commands
107 sent by batch command that server needs to execute. We need to extract
109 sent by batch command that server needs to execute. We need to extract
108 those, and map them to our ACTION_MAPPING to get all push/pull commands
110 those, and map them to our ACTION_MAPPING to get all push/pull commands
109 specified in the batch
111 specified in the batch
110 """
112 """
111 default = 'push'
113 default = 'push'
112 batch_cmds = []
114 batch_cmds = []
113 try:
115 try:
114 cmds = cls._get_xarg_headers(environ)
116 cmds = cls._get_xarg_headers(environ)
115 for pair in cmds:
117 for pair in cmds:
116 parts = pair.split(' ', 1)
118 parts = pair.split(' ', 1)
117 if len(parts) != 2:
119 if len(parts) != 2:
118 continue
120 continue
119 # entry should be in a format `key ARGS`
121 # entry should be in a format `key ARGS`
120 cmd, args = parts
122 cmd, args = parts
121 action = cls._ACTION_MAPPING.get(cmd, default)
123 action = cls._ACTION_MAPPING.get(cmd, default)
122 batch_cmds.append(action)
124 batch_cmds.append(action)
123 except Exception:
125 except Exception:
124 log.exception('Failed to extract batch commands operations')
126 log.exception('Failed to extract batch commands operations')
125
127
126 # in case we failed, (e.g malformed data) assume it's PUSH sub-command
128 # in case we failed, (e.g malformed data) assume it's PUSH sub-command
127 # for safety
129 # for safety
128 return batch_cmds or [default]
130 return batch_cmds or [default]
129
131
130 def _get_action(self, environ):
132 def _get_action(self, environ):
131 """
133 """
132 Maps mercurial request commands into a pull or push command.
134 Maps mercurial request commands into a pull or push command.
133 In case of unknown/unexpected data, it returns 'push' to be safe.
135 In case of unknown/unexpected data, it returns 'push' to be safe.
134
136
135 :param environ:
137 :param environ:
136 """
138 """
137 default = 'push'
139 default = 'push'
138 query = urllib.parse.parse_qs(environ['QUERY_STRING'], keep_blank_values=True)
140 query = urllib.parse.parse_qs(environ['QUERY_STRING'], keep_blank_values=True)
139
141
140 if 'cmd' in query:
142 if 'cmd' in query:
141 cmd = query['cmd'][0]
143 cmd = query['cmd'][0]
142 if cmd == 'batch':
144 if cmd == 'batch':
143 cmds = self._get_batch_cmd(environ)
145 cmds = self._get_batch_cmd(environ)
144 if 'push' in cmds:
146 if 'push' in cmds:
145 return 'push'
147 return 'push'
146 else:
148 else:
147 return 'pull'
149 return 'pull'
148 return self._ACTION_MAPPING.get(cmd, default)
150 return self._ACTION_MAPPING.get(cmd, default)
149
151
150 return default
152 return default
151
153
152 def _create_wsgi_app(self, repo_path, repo_name, config):
154 def _create_wsgi_app(self, repo_path, repo_name, config):
153 return self.scm_app.create_hg_wsgi_app(repo_path, repo_name, config)
155 return self.scm_app.create_hg_wsgi_app(repo_path, repo_name, config)
154
156
155 def _create_config(self, extras, repo_name, scheme='http'):
157 def _create_config(self, extras, repo_name, scheme='http'):
156 config = utils.make_db_config(repo=repo_name)
158 config = utils.make_db_config(repo=repo_name)
157 config.set('rhodecode', 'RC_SCM_DATA', json.dumps(extras))
159 config.set('rhodecode', 'RC_SCM_DATA', json.dumps(extras))
158
160
159 return config.serialize()
161 return config.serialize()
@@ -1,189 +1,193 b''
1
1
2
2
3 # Copyright (C) 2014-2020 RhodeCode GmbH
3 # Copyright (C) 2014-2020 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Implementation of the scm_app interface using raw HTTP communication.
22 Implementation of the scm_app interface using raw HTTP communication.
23 """
23 """
24
24
25 import base64
25 import base64
26 import logging
26 import logging
27 import urllib.parse
27 import urllib.parse
28 import wsgiref.util
28 import wsgiref.util
29
29
30 import msgpack
30 import msgpack
31 import requests
31 import requests
32 import webob.request
32 import webob.request
33
33
34 import rhodecode
34 import rhodecode
35 from rhodecode.lib.middleware.utils import get_path_info
35 from rhodecode.lib.middleware.utils import get_path_info
36
36
37 log = logging.getLogger(__name__)
37 log = logging.getLogger(__name__)
38
38
39
39
40 def create_git_wsgi_app(repo_path, repo_name, config):
40 def create_git_wsgi_app(repo_path, repo_name, config):
41 url = _vcs_streaming_url() + 'git/'
41 url = _vcs_streaming_url() + 'git/'
42 return VcsHttpProxy(url, repo_path, repo_name, config)
42 return VcsHttpProxy(url, repo_path, repo_name, config)
43
43
44
44
45 def create_hg_wsgi_app(repo_path, repo_name, config):
45 def create_hg_wsgi_app(repo_path, repo_name, config):
46 url = _vcs_streaming_url() + 'hg/'
46 url = _vcs_streaming_url() + 'hg/'
47 return VcsHttpProxy(url, repo_path, repo_name, config)
47 return VcsHttpProxy(url, repo_path, repo_name, config)
48
48
49
49
50 def _vcs_streaming_url():
50 def _vcs_streaming_url():
51 template = 'http://{}/stream/'
51 template = 'http://{}/stream/'
52 return template.format(rhodecode.CONFIG['vcs.server'])
52 return template.format(rhodecode.CONFIG['vcs.server'])
53
53
54
54
55 # TODO: johbo: Avoid the global.
55 # TODO: johbo: Avoid the global.
56 session = requests.Session()
56 session = requests.Session()
57 # Requests speedup, avoid reading .netrc and similar
57 # Requests speedup, avoid reading .netrc and similar
58 session.trust_env = False
58 session.trust_env = False
59
59
60 # prevent urllib3 spawning our logs.
60 # prevent urllib3 spawning our logs.
61 logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(
61 logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(
62 logging.WARNING)
62 logging.WARNING)
63
63
64
64
65 class VcsHttpProxy(object):
65 class VcsHttpProxy(object):
66 """
66 """
67 A WSGI application which proxies vcs requests.
67 A WSGI application which proxies vcs requests.
68
68
69 The goal is to shuffle the data around without touching it. The only
69 The goal is to shuffle the data around without touching it. The only
70 exception is the extra data from the config object which we send to the
70 exception is the extra data from the config object which we send to the
71 server as well.
71 server as well.
72 """
72 """
73
73
74 def __init__(self, url, repo_path, repo_name, config):
74 def __init__(self, url, repo_path, repo_name, config):
75 """
75 """
76 :param str url: The URL of the VCSServer to call.
76 :param str url: The URL of the VCSServer to call.
77 """
77 """
78 self._url = url
78 self._url = url
79 self._repo_name = repo_name
79 self._repo_name = repo_name
80 self._repo_path = repo_path
80 self._repo_path = repo_path
81 self._config = config
81 self._config = config
82 self.rc_extras = {}
82 self.rc_extras = {}
83 log.debug(
83 log.debug(
84 "Creating VcsHttpProxy for repo %s, url %s",
84 "Creating VcsHttpProxy for repo %s, url %s",
85 repo_name, url)
85 repo_name, url)
86
86
87 def __call__(self, environ, start_response):
87 def __call__(self, environ, start_response):
88 config = msgpack.packb(self._config)
88 config = self._config
89 request = webob.request.Request(environ)
89 request = webob.request.Request(environ)
90 request_headers = request.headers
90 request_headers = request.headers
91
91
92 request_headers.update({
92 call_context = {
93 # TODO: johbo: Remove this, rely on URL path only
93 # TODO: johbo: Remove this, rely on URL path only
94 'X-RC-Repo-Name': self._repo_name,
94 'repo_name': self._repo_name,
95 'X-RC-Repo-Path': self._repo_path,
95 'repo_path': self._repo_path,
96 'X-RC-Path-Info': environ['PATH_INFO'],
96 'path_info': get_path_info(environ),
97
98 'repo_store': self.rc_extras.get('repo_store'),
99 'server_config_file': self.rc_extras.get('config'),
97
100
98 'X-RC-Repo-Store': self.rc_extras.get('repo_store'),
101 'auth_user': self.rc_extras.get('username'),
99 'X-RC-Server-Config-File': self.rc_extras.get('config'),
102 'auth_user_id': str(self.rc_extras.get('user_id')),
103 'auth_user_ip': self.rc_extras.get('ip'),
100
104
101 'X-RC-Auth-User': self.rc_extras.get('username'),
105 'repo_config': config,
102 'X-RC-Auth-User-Id': str(self.rc_extras.get('user_id')),
106 'locked_status_code': rhodecode.CONFIG.get('lock_ret_code'),
103 'X-RC-Auth-User-Ip': self.rc_extras.get('ip'),
107 }
104
108
109 request_headers.update({
105 # TODO: johbo: Avoid encoding and put this into payload?
110 # TODO: johbo: Avoid encoding and put this into payload?
106 'X-RC-Repo-Config': base64.b64encode(config),
111 'X_RC_VCS_STREAM_CALL_CONTEXT': base64.b64encode(msgpack.packb(call_context))
107 'X-RC-Locked-Status-Code': rhodecode.CONFIG.get('lock_ret_code'),
108 })
112 })
109
113
110 method = environ['REQUEST_METHOD']
114 method = environ['REQUEST_METHOD']
111
115
112 # Preserve the query string
116 # Preserve the query string
113 url = self._url
117 url = self._url
114 url = urllib.parse.urljoin(url, self._repo_name)
118 url = urllib.parse.urljoin(url, self._repo_name)
115 if environ.get('QUERY_STRING'):
119 if environ.get('QUERY_STRING'):
116 url += '?' + environ['QUERY_STRING']
120 url += '?' + environ['QUERY_STRING']
117
121
118 log.debug('http-app: preparing request to: %s', url)
122 log.debug('http-app: preparing request to: %s', url)
119 response = session.request(
123 response = session.request(
120 method,
124 method,
121 url,
125 url,
122 data=_maybe_stream_request(environ),
126 data=_maybe_stream_request(environ),
123 headers=request_headers,
127 headers=request_headers,
124 stream=True)
128 stream=True)
125
129
126 log.debug('http-app: got vcsserver response: %s', response)
130 log.debug('http-app: got vcsserver response: %s', response)
127 if response.status_code >= 500:
131 if response.status_code >= 500:
128 log.error('Exception returned by vcsserver at: %s %s, %s',
132 log.error('Exception returned by vcsserver at: %s %s, %s',
129 url, response.status_code, response.content)
133 url, response.status_code, response.content)
130
134
131 # Preserve the headers of the response, except hop_by_hop ones
135 # Preserve the headers of the response, except hop_by_hop ones
132 response_headers = [
136 response_headers = [
133 (h, v) for h, v in response.headers.items()
137 (h, v) for h, v in response.headers.items()
134 if not wsgiref.util.is_hop_by_hop(h)
138 if not wsgiref.util.is_hop_by_hop(h)
135 ]
139 ]
136
140
137 # Build status argument for start_response callable.
141 # Build status argument for start_response callable.
138 status = '{status_code} {reason_phrase}'.format(
142 status = '{status_code} {reason_phrase}'.format(
139 status_code=response.status_code,
143 status_code=response.status_code,
140 reason_phrase=response.reason)
144 reason_phrase=response.reason)
141
145
142 start_response(status, response_headers)
146 start_response(status, response_headers)
143 return _maybe_stream_response(response)
147 return _maybe_stream_response(response)
144
148
145
149
146 def read_in_chunks(stream_obj, block_size=1024, chunks=-1):
150 def read_in_chunks(stream_obj, block_size=1024, chunks=-1):
147 """
151 """
148 Read Stream in chunks, default chunk size: 1k.
152 Read Stream in chunks, default chunk size: 1k.
149 """
153 """
150 while chunks:
154 while chunks:
151 data = stream_obj.read(block_size)
155 data = stream_obj.read(block_size)
152 if not data:
156 if not data:
153 break
157 break
154 yield data
158 yield data
155 chunks -= 1
159 chunks -= 1
156
160
157
161
158 def _is_request_chunked(environ):
162 def _is_request_chunked(environ):
159 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
163 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
160 return stream
164 return stream
161
165
162
166
163 def _maybe_stream_request(environ):
167 def _maybe_stream_request(environ):
164 path = get_path_info(environ)
168 path = get_path_info(environ)
165 stream = _is_request_chunked(environ)
169 stream = _is_request_chunked(environ)
166 log.debug('handling request `%s` with stream support: %s', path, stream)
170 log.debug('handling request `%s` with stream support: %s', path, stream)
167
171
168 if stream:
172 if stream:
169 # set stream by 256k
173 # set stream by 256k
170 return read_in_chunks(environ['wsgi.input'], block_size=1024 * 256)
174 return read_in_chunks(environ['wsgi.input'], block_size=1024 * 256)
171 else:
175 else:
172 return environ['wsgi.input'].read()
176 return environ['wsgi.input'].read()
173
177
174
178
175 def _maybe_stream_response(response):
179 def _maybe_stream_response(response):
176 """
180 """
177 Try to generate chunks from the response if it is chunked.
181 Try to generate chunks from the response if it is chunked.
178 """
182 """
179 stream = _is_chunked(response)
183 stream = _is_chunked(response)
180 log.debug('returning response with stream: %s', stream)
184 log.debug('returning response with stream: %s', stream)
181 if stream:
185 if stream:
182 # read in 256k Chunks
186 # read in 256k Chunks
183 return response.raw.read_chunked(amt=1024 * 256)
187 return response.raw.read_chunked(amt=1024 * 256)
184 else:
188 else:
185 return [response.content]
189 return [response.content]
186
190
187
191
188 def _is_chunked(response):
192 def _is_chunked(response):
189 return response.headers.get('Transfer-Encoding', '') == 'chunked'
193 return response.headers.get('Transfer-Encoding', '') == 'chunked'
@@ -1,288 +1,290 b''
1
1
2 # Copyright (C) 2010-2020 RhodeCode GmbH
2 # Copyright (C) 2010-2020 RhodeCode GmbH
3 #
3 #
4 # This program is free software: you can redistribute it and/or modify
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License, version 3
5 # it under the terms of the GNU Affero General Public License, version 3
6 # (only), as published by the Free Software Foundation.
6 # (only), as published by the Free Software Foundation.
7 #
7 #
8 # This program is distributed in the hope that it will be useful,
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
11 # GNU General Public License for more details.
12 #
12 #
13 # You should have received a copy of the GNU Affero General Public License
13 # You should have received a copy of the GNU Affero General Public License
14 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 #
15 #
16 # This program is dual-licensed. If you wish to learn more about the
16 # This program is dual-licensed. If you wish to learn more about the
17 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # and proprietary license terms, please see https://rhodecode.com/licenses/
18 # and proprietary license terms, please see https://rhodecode.com/licenses/
19
19
20 import gzip
20 import gzip
21 import shutil
21 import shutil
22 import logging
22 import logging
23 import tempfile
23 import tempfile
24 import urllib.parse
24 import urllib.parse
25
25
26 from webob.exc import HTTPNotFound
26 from webob.exc import HTTPNotFound
27
27
28 import rhodecode
28 import rhodecode
29 from rhodecode.lib.middleware.utils import get_path_info
29 from rhodecode.lib.middleware.utils import get_path_info
30 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
30 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
31 from rhodecode.lib.middleware.simplegit import SimpleGit, GIT_PROTO_PAT
31 from rhodecode.lib.middleware.simplegit import SimpleGit, GIT_PROTO_PAT
32 from rhodecode.lib.middleware.simplehg import SimpleHg
32 from rhodecode.lib.middleware.simplehg import SimpleHg
33 from rhodecode.lib.middleware.simplesvn import SimpleSvn
33 from rhodecode.lib.middleware.simplesvn import SimpleSvn
34 from rhodecode.model.settings import VcsSettingsModel
34 from rhodecode.model.settings import VcsSettingsModel
35
35
36
36
37 log = logging.getLogger(__name__)
37 log = logging.getLogger(__name__)
38
38
39 VCS_TYPE_KEY = '_rc_vcs_type'
39 VCS_TYPE_KEY = '_rc_vcs_type'
40 VCS_TYPE_SKIP = '_rc_vcs_skip'
40 VCS_TYPE_SKIP = '_rc_vcs_skip'
41
41
42
42
43 def is_git(environ):
43 def is_git(environ):
44 """
44 """
45 Returns True if requests should be handled by GIT wsgi middleware
45 Returns True if requests should be handled by GIT wsgi middleware
46 """
46 """
47 path_info = get_path_info(environ)
47 path_info = get_path_info(environ)
48 is_git_path = GIT_PROTO_PAT.match(path_info)
48 is_git_path = GIT_PROTO_PAT.match(path_info)
49 log.debug(
49 log.debug(
50 'request path: `%s` detected as GIT PROTOCOL %s', path_info,
50 'request path: `%s` detected as GIT PROTOCOL %s', path_info,
51 is_git_path is not None)
51 is_git_path is not None)
52
52
53 return is_git_path
53 return is_git_path
54
54
55
55
56 def is_hg(environ):
56 def is_hg(environ):
57 """
57 """
58 Returns True if requests target is mercurial server - header
58 Returns True if requests target is mercurial server - header
59 ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
59 ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
60 """
60 """
61 is_hg_path = False
61 is_hg_path = False
62
62
63 http_accept = environ.get('HTTP_ACCEPT')
63 http_accept = environ.get('HTTP_ACCEPT')
64
64
65 if http_accept and http_accept.startswith('application/mercurial'):
65 if http_accept and http_accept.startswith('application/mercurial'):
66 query = urllib.parse.parse_qs(environ['QUERY_STRING'])
66 query = urllib.parse.parse_qs(environ['QUERY_STRING'])
67 if 'cmd' in query:
67 if 'cmd' in query:
68 is_hg_path = True
68 is_hg_path = True
69
69
70 path_info = get_path_info(environ)
70 path_info = get_path_info(environ)
71 log.debug(
71 log.debug(
72 'request path: `%s` detected as HG PROTOCOL %s', path_info,
72 'request path: `%s` detected as HG PROTOCOL %s', path_info,
73 is_hg_path)
73 is_hg_path)
74
74
75 return is_hg_path
75 return is_hg_path
76
76
77
77
78 def is_svn(environ):
78 def is_svn(environ):
79 """
79 """
80 Returns True if requests target is Subversion server
80 Returns True if requests target is Subversion server
81 """
81 """
82
82
83 http_dav = environ.get('HTTP_DAV', '')
83 http_dav = environ.get('HTTP_DAV', '')
84 magic_path_segment = rhodecode.CONFIG.get(
84 magic_path_segment = rhodecode.CONFIG.get(
85 'rhodecode_subversion_magic_path', '/!svn')
85 'rhodecode_subversion_magic_path', '/!svn')
86 path_info = get_path_info(environ)
86 path_info = get_path_info(environ)
87 is_svn_path = (
87 is_svn_path = (
88 'subversion' in http_dav or
88 'subversion' in http_dav or
89 magic_path_segment in path_info
89 magic_path_segment in path_info
90 or environ['REQUEST_METHOD'] in ['PROPFIND', 'PROPPATCH']
90 or environ['REQUEST_METHOD'] in ['PROPFIND', 'PROPPATCH']
91 )
91 )
92 log.debug(
92 log.debug(
93 'request path: `%s` detected as SVN PROTOCOL %s', path_info,
93 'request path: `%s` detected as SVN PROTOCOL %s', path_info,
94 is_svn_path)
94 is_svn_path)
95
95
96 return is_svn_path
96 return is_svn_path
97
97
98
98
99 class GunzipMiddleware(object):
99 class GunzipMiddleware(object):
100 """
100 """
101 WSGI middleware that unzips gzip-encoded requests before
101 WSGI middleware that unzips gzip-encoded requests before
102 passing on to the underlying application.
102 passing on to the underlying application.
103 """
103 """
104
104
105 def __init__(self, application):
105 def __init__(self, application):
106 self.app = application
106 self.app = application
107
107
108 def __call__(self, environ, start_response):
108 def __call__(self, environ, start_response):
109 accepts_encoding_header = environ.get('HTTP_CONTENT_ENCODING', b'')
109 accepts_encoding_header = environ.get('HTTP_CONTENT_ENCODING', b'')
110
110
111 if b'gzip' in accepts_encoding_header:
111 if b'gzip' in accepts_encoding_header:
112 log.debug('gzip detected, now running gunzip wrapper')
112 log.debug('gzip detected, now running gunzip wrapper')
113 wsgi_input = environ['wsgi.input']
113 wsgi_input = environ['wsgi.input']
114
114
115 if not hasattr(environ['wsgi.input'], 'seek'):
115 if not hasattr(environ['wsgi.input'], 'seek'):
116 # The gzip implementation in the standard library of Python 2.x
116 # The gzip implementation in the standard library of Python 2.x
117 # requires the '.seek()' and '.tell()' methods to be available
117 # requires the '.seek()' and '.tell()' methods to be available
118 # on the input stream. Read the data into a temporary file to
118 # on the input stream. Read the data into a temporary file to
119 # work around this limitation.
119 # work around this limitation.
120
120
121 wsgi_input = tempfile.SpooledTemporaryFile(64 * 1024 * 1024)
121 wsgi_input = tempfile.SpooledTemporaryFile(64 * 1024 * 1024)
122 shutil.copyfileobj(environ['wsgi.input'], wsgi_input)
122 shutil.copyfileobj(environ['wsgi.input'], wsgi_input)
123 wsgi_input.seek(0)
123 wsgi_input.seek(0)
124
124
125 environ['wsgi.input'] = gzip.GzipFile(fileobj=wsgi_input, mode='r')
125 environ['wsgi.input'] = gzip.GzipFile(fileobj=wsgi_input, mode='r')
126 # since we "Ungzipped" the content we say now it's no longer gzip
126 # since we "Ungzipped" the content we say now it's no longer gzip
127 # content encoding
127 # content encoding
128 del environ['HTTP_CONTENT_ENCODING']
128 del environ['HTTP_CONTENT_ENCODING']
129
129
130 # content length has changes ? or i'm not sure
130 # content length has changes ? or i'm not sure
131 if 'CONTENT_LENGTH' in environ:
131 if 'CONTENT_LENGTH' in environ:
132 del environ['CONTENT_LENGTH']
132 del environ['CONTENT_LENGTH']
133 else:
133 else:
134 log.debug('content not gzipped, gzipMiddleware passing '
134 log.debug('content not gzipped, gzipMiddleware passing '
135 'request further')
135 'request further')
136 return self.app(environ, start_response)
136 return self.app(environ, start_response)
137
137
138
138
139 def is_vcs_call(environ):
139 def is_vcs_call(environ):
140 if VCS_TYPE_KEY in environ:
140 if VCS_TYPE_KEY in environ:
141 raw_type = environ[VCS_TYPE_KEY]
141 raw_type = environ[VCS_TYPE_KEY]
142 return raw_type and raw_type != VCS_TYPE_SKIP
142 return raw_type and raw_type != VCS_TYPE_SKIP
143 return False
143 return False
144
144
145
145
146 def get_path_elem(route_path):
147 if not route_path:
148 return None
149
150 cleaned_route_path = route_path.lstrip('/')
151 if cleaned_route_path:
152 cleaned_route_path_elems = cleaned_route_path.split('/')
153 if cleaned_route_path_elems:
154 return cleaned_route_path_elems[0]
155 return None
156
157
158 def detect_vcs_request(environ, backends):
146 def detect_vcs_request(environ, backends):
159 checks = {
147 checks = {
160 'hg': (is_hg, SimpleHg),
148 'hg': (is_hg, SimpleHg),
161 'git': (is_git, SimpleGit),
149 'git': (is_git, SimpleGit),
162 'svn': (is_svn, SimpleSvn),
150 'svn': (is_svn, SimpleSvn),
163 }
151 }
164 handler = None
152 handler = None
165 # List of path views first chunk we don't do any checks
153 # List of path views first chunk we don't do any checks
166 white_list = [
154 white_list = [
155 # favicon often requested by browsers
156 'favicon.ico',
157
167 # e.g /_file_store/download
158 # e.g /_file_store/download
168 '_file_store',
159 '_file_store++',
160
161 # _admin/api is safe too
162 '_admin/api',
163
164 # _admin/gist is safe too
165 '_admin/gists++',
166
167 # _admin/my_account is safe too
168 '_admin/my_account++',
169
169
170 # static files no detection
170 # static files no detection
171 '_static',
171 '_static++',
172
173 # debug-toolbar
174 '_debug_toolbar++',
172
175
173 # skip ops ping, status
176 # skip ops ping, status
174 '_admin/ops/ping',
177 '_admin/ops/ping',
175 '_admin/ops/status',
178 '_admin/ops/status',
176
179
177 # full channelstream connect should be VCS skipped
180 # full channelstream connect should be VCS skipped
178 '_admin/channelstream/connect',
181 '_admin/channelstream/connect',
179 ]
182 ]
180 path_info = get_path_info(environ)
183 path_info = get_path_info(environ)
181 path_url = path_info.lstrip('/')
184 path_url = path_info.lstrip('/')
182
185
183 if path_elem in white_list:
186 for item in white_list:
184 log.debug('path `%s` in whitelist, skipping...', path_info)
187 if item.endswith('++') and path_url.startswith(item[:-2]):
185 return handler
188 log.debug('path `%s` in whitelist (match:%s), skipping...', path_url, item)
186
189 return handler
187 path_url = path_info.lstrip('/')
190 if item == path_url:
188 if path_url in white_list:
191 log.debug('path `%s` in whitelist (match:%s), skipping...', path_url, item)
189 log.debug('full url path `%s` in whitelist, skipping...', path_url)
192 return handler
190 return handler
191
193
192 if VCS_TYPE_KEY in environ:
194 if VCS_TYPE_KEY in environ:
193 raw_type = environ[VCS_TYPE_KEY]
195 raw_type = environ[VCS_TYPE_KEY]
194 if raw_type == VCS_TYPE_SKIP:
196 if raw_type == VCS_TYPE_SKIP:
195 log.debug('got `skip` marker for vcs detection, skipping...')
197 log.debug('got `skip` marker for vcs detection, skipping...')
196 return handler
198 return handler
197
199
198 _check, handler = checks.get(raw_type) or [None, None]
200 _check, handler = checks.get(raw_type) or [None, None]
199 if handler:
201 if handler:
200 log.debug('got handler:%s from environ', handler)
202 log.debug('got handler:%s from environ', handler)
201
203
202 if not handler:
204 if not handler:
203 log.debug('request start: checking if request for `%s` is of VCS type in order: %s', path_elem, backends)
205 log.debug('request start: checking if request for `%s` is of VCS type in order: %s', path_url, backends)
204 for vcs_type in backends:
206 for vcs_type in backends:
205 vcs_check, _handler = checks[vcs_type]
207 vcs_check, _handler = checks[vcs_type]
206 if vcs_check(environ):
208 if vcs_check(environ):
207 log.debug('vcs handler found %s', _handler)
209 log.debug('vcs handler found %s', _handler)
208 handler = _handler
210 handler = _handler
209 break
211 break
210
212
211 return handler
213 return handler
212
214
213
215
214 class VCSMiddleware(object):
216 class VCSMiddleware(object):
215
217
216 def __init__(self, app, registry, config, appenlight_client):
218 def __init__(self, app, registry, config, appenlight_client):
217 self.application = app
219 self.application = app
218 self.registry = registry
220 self.registry = registry
219 self.config = config
221 self.config = config
220 self.appenlight_client = appenlight_client
222 self.appenlight_client = appenlight_client
221 self.use_gzip = True
223 self.use_gzip = True
222 # order in which we check the middlewares, based on vcs.backends config
224 # order in which we check the middlewares, based on vcs.backends config
223 self.check_middlewares = config['vcs.backends']
225 self.check_middlewares = config['vcs.backends']
224
226
225 def vcs_config(self, repo_name=None):
227 def vcs_config(self, repo_name=None):
226 """
228 """
227 returns serialized VcsSettings
229 returns serialized VcsSettings
228 """
230 """
229 try:
231 try:
230 return VcsSettingsModel(
232 return VcsSettingsModel(
231 repo=repo_name).get_ui_settings_as_config_obj()
233 repo=repo_name).get_ui_settings_as_config_obj()
232 except Exception:
234 except Exception:
233 pass
235 pass
234
236
235 def wrap_in_gzip_if_enabled(self, app, config):
237 def wrap_in_gzip_if_enabled(self, app, config):
236 if self.use_gzip:
238 if self.use_gzip:
237 app = GunzipMiddleware(app)
239 app = GunzipMiddleware(app)
238 return app
240 return app
239
241
240 def _get_handler_app(self, environ):
242 def _get_handler_app(self, environ):
241 app = None
243 app = None
242 log.debug('VCSMiddleware: detecting vcs type.')
244 log.debug('VCSMiddleware: detecting vcs type.')
243 handler = detect_vcs_request(environ, self.check_middlewares)
245 handler = detect_vcs_request(environ, self.check_middlewares)
244 if handler:
246 if handler:
245 app = handler(self.config, self.registry)
247 app = handler(self.config, self.registry)
246
248
247 return app
249 return app
248
250
249 def __call__(self, environ, start_response):
251 def __call__(self, environ, start_response):
250 # check if we handle one of interesting protocols, optionally extract
252 # check if we handle one of interesting protocols, optionally extract
251 # specific vcsSettings and allow changes of how things are wrapped
253 # specific vcsSettings and allow changes of how things are wrapped
252 vcs_handler = self._get_handler_app(environ)
254 vcs_handler = self._get_handler_app(environ)
253 if vcs_handler:
255 if vcs_handler:
254 # translate the _REPO_ID into real repo NAME for usage
256 # translate the _REPO_ID into real repo NAME for usage
255 # in middleware
257 # in middleware
256
258
257 path_info = get_path_info(environ)
259 path_info = get_path_info(environ)
258 environ['PATH_INFO'] = vcs_handler._get_by_id(path_info)
260 environ['PATH_INFO'] = vcs_handler._get_by_id(path_info)
259
261
260 # Set acl, url and vcs repo names.
262 # Set acl, url and vcs repo names.
261 vcs_handler.set_repo_names(environ)
263 vcs_handler.set_repo_names(environ)
262
264
263 # register repo config back to the handler
265 # register repo config back to the handler
264 vcs_conf = self.vcs_config(vcs_handler.acl_repo_name)
266 vcs_conf = self.vcs_config(vcs_handler.acl_repo_name)
265 # maybe damaged/non existent settings. We still want to
267 # maybe damaged/non existent settings. We still want to
266 # pass that point to validate on is_valid_and_existing_repo
268 # pass that point to validate on is_valid_and_existing_repo
267 # and return proper HTTP Code back to client
269 # and return proper HTTP Code back to client
268 if vcs_conf:
270 if vcs_conf:
269 vcs_handler.repo_vcs_config = vcs_conf
271 vcs_handler.repo_vcs_config = vcs_conf
270
272
271 # check for type, presence in database and on filesystem
273 # check for type, presence in database and on filesystem
272 if not vcs_handler.is_valid_and_existing_repo(
274 if not vcs_handler.is_valid_and_existing_repo(
273 vcs_handler.acl_repo_name,
275 vcs_handler.acl_repo_name,
274 vcs_handler.base_path,
276 vcs_handler.base_path,
275 vcs_handler.SCM):
277 vcs_handler.SCM):
276 return HTTPNotFound()(environ, start_response)
278 return HTTPNotFound()(environ, start_response)
277
279
278 environ['REPO_NAME'] = vcs_handler.url_repo_name
280 environ['REPO_NAME'] = vcs_handler.url_repo_name
279
281
280 # Wrap handler in middlewares if they are enabled.
282 # Wrap handler in middlewares if they are enabled.
281 vcs_handler = self.wrap_in_gzip_if_enabled(
283 vcs_handler = self.wrap_in_gzip_if_enabled(
282 vcs_handler, self.config)
284 vcs_handler, self.config)
283 vcs_handler, _ = wrap_in_appenlight_if_enabled(
285 vcs_handler, _ = wrap_in_appenlight_if_enabled(
284 vcs_handler, self.config, self.appenlight_client)
286 vcs_handler, self.config, self.appenlight_client)
285
287
286 return vcs_handler(environ, start_response)
288 return vcs_handler(environ, start_response)
287
289
288 return self.application(environ, start_response)
290 return self.application(environ, start_response)
General Comments 0
You need to be logged in to leave comments. Login now