##// END OF EJS Templates
pycurl: skip host verification to speed up curl a bit.
marcink -
r3929:f5cb74c0 default
parent child Browse files
Show More
@@ -1,184 +1,186 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2014-2019 RhodeCode GmbH
3 # Copyright (C) 2014-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Various version Control System version lib (vcs) management abstraction layer
22 Various version Control System version lib (vcs) management abstraction layer
23 for Python. Build with server client architecture.
23 for Python. Build with server client architecture.
24 """
24 """
25 import atexit
25 import atexit
26 import logging
26 import logging
27 import urlparse
27 import urlparse
28 from cStringIO import StringIO
28 from cStringIO import StringIO
29
29
30 import rhodecode
30 import rhodecode
31 from rhodecode.lib.vcs.conf import settings
31 from rhodecode.lib.vcs.conf import settings
32 from rhodecode.lib.vcs.backends import get_vcs_instance, get_backend
32 from rhodecode.lib.vcs.backends import get_vcs_instance, get_backend
33 from rhodecode.lib.vcs.exceptions import (
33 from rhodecode.lib.vcs.exceptions import (
34 VCSError, RepositoryError, CommitError, VCSCommunicationError)
34 VCSError, RepositoryError, CommitError, VCSCommunicationError)
35
35
36 VERSION = (0, 5, 0, 'dev')
36 VERSION = (0, 5, 0, 'dev')
37
37
38 __version__ = '.'.join((str(each) for each in VERSION[:4]))
38 __version__ = '.'.join((str(each) for each in VERSION[:4]))
39
39
40 __all__ = [
40 __all__ = [
41 'get_version', 'get_vcs_instance', 'get_backend',
41 'get_version', 'get_vcs_instance', 'get_backend',
42 'VCSError', 'RepositoryError', 'CommitError', 'VCSCommunicationError'
42 'VCSError', 'RepositoryError', 'CommitError', 'VCSCommunicationError'
43 ]
43 ]
44
44
45 log = logging.getLogger(__name__)
45 log = logging.getLogger(__name__)
46
46
47 # The pycurl library directly accesses C API functions and is not patched by
47 # The pycurl library directly accesses C API functions and is not patched by
48 # gevent. This will potentially lead to deadlocks due to incompatibility to
48 # gevent. This will potentially lead to deadlocks due to incompatibility to
49 # gevent. Therefore we check if gevent is active and import a gevent compatible
49 # gevent. Therefore we check if gevent is active and import a gevent compatible
50 # wrapper in that case.
50 # wrapper in that case.
51 try:
51 try:
52 from gevent import monkey
52 from gevent import monkey
53 if monkey.is_module_patched('__builtin__'):
53 if monkey.is_module_patched('__builtin__'):
54 import geventcurl as pycurl
54 import geventcurl as pycurl
55 log.debug('Using gevent comapatible pycurl: %s', pycurl)
55 log.debug('Using gevent comapatible pycurl: %s', pycurl)
56 else:
56 else:
57 import pycurl
57 import pycurl
58 except ImportError:
58 except ImportError:
59 import pycurl
59 import pycurl
60
60
61
61
62 def get_version():
62 def get_version():
63 """
63 """
64 Returns shorter version (digit parts only) as string.
64 Returns shorter version (digit parts only) as string.
65 """
65 """
66 return '.'.join((str(each) for each in VERSION[:3]))
66 return '.'.join((str(each) for each in VERSION[:3]))
67
67
68
68
69 def connect_http(server_and_port):
69 def connect_http(server_and_port):
70 from rhodecode.lib.vcs import connection, client_http
70 from rhodecode.lib.vcs import connection, client_http
71 from rhodecode.lib.middleware.utils import scm_app
71 from rhodecode.lib.middleware.utils import scm_app
72
72
73 session_factory = client_http.ThreadlocalSessionFactory()
73 session_factory = client_http.ThreadlocalSessionFactory()
74
74
75 connection.Git = client_http.RemoteVCSMaker(
75 connection.Git = client_http.RemoteVCSMaker(
76 server_and_port, '/git', 'git', session_factory)
76 server_and_port, '/git', 'git', session_factory)
77 connection.Hg = client_http.RemoteVCSMaker(
77 connection.Hg = client_http.RemoteVCSMaker(
78 server_and_port, '/hg', 'hg', session_factory)
78 server_and_port, '/hg', 'hg', session_factory)
79 connection.Svn = client_http.RemoteVCSMaker(
79 connection.Svn = client_http.RemoteVCSMaker(
80 server_and_port, '/svn', 'svn', session_factory)
80 server_and_port, '/svn', 'svn', session_factory)
81 connection.Service = client_http.ServiceConnection(
81 connection.Service = client_http.ServiceConnection(
82 server_and_port, '/_service', session_factory)
82 server_and_port, '/_service', session_factory)
83
83
84 scm_app.HG_REMOTE_WSGI = client_http.VcsHttpProxy(
84 scm_app.HG_REMOTE_WSGI = client_http.VcsHttpProxy(
85 server_and_port, '/proxy/hg')
85 server_and_port, '/proxy/hg')
86 scm_app.GIT_REMOTE_WSGI = client_http.VcsHttpProxy(
86 scm_app.GIT_REMOTE_WSGI = client_http.VcsHttpProxy(
87 server_and_port, '/proxy/git')
87 server_and_port, '/proxy/git')
88
88
89 @atexit.register
89 @atexit.register
90 def free_connection_resources():
90 def free_connection_resources():
91 connection.Git = None
91 connection.Git = None
92 connection.Hg = None
92 connection.Hg = None
93 connection.Svn = None
93 connection.Svn = None
94 connection.Service = None
94 connection.Service = None
95
95
96
96
97 def connect_vcs(server_and_port, protocol):
97 def connect_vcs(server_and_port, protocol):
98 """
98 """
99 Initializes the connection to the vcs server.
99 Initializes the connection to the vcs server.
100
100
101 :param server_and_port: str, e.g. "localhost:9900"
101 :param server_and_port: str, e.g. "localhost:9900"
102 :param protocol: str or "http"
102 :param protocol: str or "http"
103 """
103 """
104 if protocol == 'http':
104 if protocol == 'http':
105 connect_http(server_and_port)
105 connect_http(server_and_port)
106 else:
106 else:
107 raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
107 raise Exception('Invalid vcs server protocol "{}"'.format(protocol))
108
108
109
109
110 class CurlSession(object):
110 class CurlSession(object):
111 """
111 """
112 Modeled so that it provides a subset of the requests interface.
112 Modeled so that it provides a subset of the requests interface.
113
113
114 This has been created so that it does only provide a minimal API for our
114 This has been created so that it does only provide a minimal API for our
115 needs. The parts which it provides are based on the API of the library
115 needs. The parts which it provides are based on the API of the library
116 `requests` which allows us to easily benchmark against it.
116 `requests` which allows us to easily benchmark against it.
117
117
118 Please have a look at the class :class:`requests.Session` when you extend
118 Please have a look at the class :class:`requests.Session` when you extend
119 it.
119 it.
120 """
120 """
121
121
122 def __init__(self):
122 def __init__(self):
123 curl = pycurl.Curl()
123 curl = pycurl.Curl()
124 # TODO: johbo: I did test with 7.19 of libcurl. This version has
124 # TODO: johbo: I did test with 7.19 of libcurl. This version has
125 # trouble with 100 - continue being set in the expect header. This
125 # trouble with 100 - continue being set in the expect header. This
126 # can lead to massive performance drops, switching it off here.
126 # can lead to massive performance drops, switching it off here.
127 curl.setopt(curl.HTTPHEADER, ["Expect:"])
127 curl.setopt(curl.HTTPHEADER, ["Expect:"])
128 curl.setopt(curl.TCP_NODELAY, True)
128 curl.setopt(curl.TCP_NODELAY, True)
129 curl.setopt(curl.PROTOCOLS, curl.PROTO_HTTP)
129 curl.setopt(curl.PROTOCOLS, curl.PROTO_HTTP)
130 curl.setopt(curl.USERAGENT, 'RhodeCode HTTP {}'.format(rhodecode.__version__))
130 curl.setopt(curl.USERAGENT, 'RhodeCode HTTP {}'.format(rhodecode.__version__))
131 curl.setopt(curl.SSL_VERIFYPEER, 0)
132 curl.setopt(curl.SSL_VERIFYHOST, 0)
131 self._curl = curl
133 self._curl = curl
132
134
133 def post(self, url, data, allow_redirects=False):
135 def post(self, url, data, allow_redirects=False):
134 response_buffer = StringIO()
136 response_buffer = StringIO()
135
137
136 curl = self._curl
138 curl = self._curl
137 curl.setopt(curl.URL, url)
139 curl.setopt(curl.URL, url)
138 curl.setopt(curl.POST, True)
140 curl.setopt(curl.POST, True)
139 curl.setopt(curl.POSTFIELDS, data)
141 curl.setopt(curl.POSTFIELDS, data)
140 curl.setopt(curl.FOLLOWLOCATION, allow_redirects)
142 curl.setopt(curl.FOLLOWLOCATION, allow_redirects)
141 curl.setopt(curl.WRITEDATA, response_buffer)
143 curl.setopt(curl.WRITEDATA, response_buffer)
142 curl.perform()
144 curl.perform()
143
145
144 status_code = curl.getinfo(pycurl.HTTP_CODE)
146 status_code = curl.getinfo(pycurl.HTTP_CODE)
145
147
146 return CurlResponse(response_buffer, status_code)
148 return CurlResponse(response_buffer, status_code)
147
149
148
150
149 class CurlResponse(object):
151 class CurlResponse(object):
150 """
152 """
151 The response of a request, modeled after the requests API.
153 The response of a request, modeled after the requests API.
152
154
153 This class provides a subset of the response interface known from the
155 This class provides a subset of the response interface known from the
154 library `requests`. It is intentionally kept similar, so that we can use
156 library `requests`. It is intentionally kept similar, so that we can use
155 `requests` as a drop in replacement for benchmarking purposes.
157 `requests` as a drop in replacement for benchmarking purposes.
156 """
158 """
157
159
158 def __init__(self, response_buffer, status_code):
160 def __init__(self, response_buffer, status_code):
159 self._response_buffer = response_buffer
161 self._response_buffer = response_buffer
160 self._status_code = status_code
162 self._status_code = status_code
161
163
162 @property
164 @property
163 def content(self):
165 def content(self):
164 try:
166 try:
165 return self._response_buffer.getvalue()
167 return self._response_buffer.getvalue()
166 finally:
168 finally:
167 self._response_buffer.close()
169 self._response_buffer.close()
168
170
169 @property
171 @property
170 def status_code(self):
172 def status_code(self):
171 return self._status_code
173 return self._status_code
172
174
173 def iter_content(self, chunk_size):
175 def iter_content(self, chunk_size):
174 self._response_buffer.seek(0)
176 self._response_buffer.seek(0)
175 while 1:
177 while 1:
176 chunk = self._response_buffer.read(chunk_size)
178 chunk = self._response_buffer.read(chunk_size)
177 if not chunk:
179 if not chunk:
178 break
180 break
179 yield chunk
181 yield chunk
180
182
181
183
182 def _create_http_rpc_session():
184 def _create_http_rpc_session():
183 session = CurlSession()
185 session = CurlSession()
184 return session
186 return session
General Comments 0
You need to be logged in to leave comments. Login now