##// END OF EJS Templates
fix(pycurl): added missing __all__ imports
super-admin -
r5322:05d560ef default
parent child Browse files
Show More
@@ -1,188 +1,189 b''
1 # Copyright (C) 2014-2023 RhodeCode GmbH
1 # Copyright (C) 2014-2023 RhodeCode GmbH
2 #
2 #
3 # This program is free software: you can redistribute it and/or modify
3 # This program is free software: you can redistribute it and/or modify
4 # it under the terms of the GNU Affero General Public License, version 3
4 # it under the terms of the GNU Affero General Public License, version 3
5 # (only), as published by the Free Software Foundation.
5 # (only), as published by the Free Software Foundation.
6 #
6 #
7 # This program is distributed in the hope that it will be useful,
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
10 # GNU General Public License for more details.
11 #
11 #
12 # You should have received a copy of the GNU Affero General Public License
12 # You should have received a copy of the GNU Affero General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 #
14 #
15 # This program is dual-licensed. If you wish to learn more about the
15 # This program is dual-licensed. If you wish to learn more about the
16 # RhodeCode Enterprise Edition, including its added features, Support services,
16 # RhodeCode Enterprise Edition, including its added features, Support services,
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
17 # and proprietary license terms, please see https://rhodecode.com/licenses/
18
18
19 """
19 """
20 Various version Control System version lib (vcs) management abstraction layer
20 Various version Control System version lib (vcs) management abstraction layer
21 for Python. Build with server client architecture.
21 for Python. Build with server client architecture.
22 """
22 """
23 import io
23 import io
24 import atexit
24 import atexit
25 import logging
25 import logging
26
26
27 import rhodecode
27 import rhodecode
28 from rhodecode.lib.str_utils import safe_bytes
28 from rhodecode.lib.str_utils import safe_bytes
29 from rhodecode.lib.vcs.conf import settings
29 from rhodecode.lib.vcs.conf import settings
30 from rhodecode.lib.vcs.backends import get_vcs_instance, get_backend
30 from rhodecode.lib.vcs.backends import get_vcs_instance, get_backend
31 from rhodecode.lib.vcs.exceptions import (
31 from rhodecode.lib.vcs.exceptions import (
32 VCSError, RepositoryError, CommitError, VCSCommunicationError)
32 VCSError, RepositoryError, CommitError, VCSCommunicationError)
33
33
34 __all__ = [
34 __all__ = [
35 'get_vcs_instance', 'get_backend',
35 'get_vcs_instance', 'get_backend',
36 'VCSError', 'RepositoryError', 'CommitError', 'VCSCommunicationError'
36 'VCSError', 'RepositoryError', 'CommitError', 'VCSCommunicationError',
37 'CurlSession', 'CurlResponse'
37 ]
38 ]
38
39
39 log = logging.getLogger(__name__)
40 log = logging.getLogger(__name__)
40
41
41 # The pycurl library directly accesses C API functions and is not patched by
42 # The pycurl library directly accesses C API functions and is not patched by
42 # gevent. This will potentially lead to deadlocks due to incompatibility to
43 # gevent. This will potentially lead to deadlocks due to incompatibility to
43 # gevent. Therefore we check if gevent is active and import a gevent compatible
44 # gevent. Therefore we check if gevent is active and import a gevent compatible
44 # wrapper in that case.
45 # wrapper in that case.
45 try:
46 try:
46 from gevent import monkey
47 from gevent import monkey
47 if monkey.is_module_patched('__builtin__'):
48 if monkey.is_module_patched('__builtin__'):
48 import geventcurl as pycurl
49 import geventcurl as pycurl
49 log.debug('Using gevent comapatible pycurl: %s', pycurl)
50 log.debug('Using gevent comapatible pycurl: %s', pycurl)
50 else:
51 else:
51 import pycurl
52 import pycurl
52 except ImportError:
53 except ImportError:
53 import pycurl
54 import pycurl
54
55
55
56
56 def connect_http(server_and_port):
57 def connect_http(server_and_port):
57 log.debug('Initialized VCSServer connections to %s.', server_and_port)
58 log.debug('Initialized VCSServer connections to %s.', server_and_port)
58
59
59 from rhodecode.lib.vcs import connection, client_http
60 from rhodecode.lib.vcs import connection, client_http
60 from rhodecode.lib.middleware.utils import scm_app
61 from rhodecode.lib.middleware.utils import scm_app
61
62
62 session_factory = client_http.ThreadlocalSessionFactory()
63 session_factory = client_http.ThreadlocalSessionFactory()
63
64
64 connection.Git = client_http.RemoteVCSMaker(
65 connection.Git = client_http.RemoteVCSMaker(
65 server_and_port, '/git', 'git', session_factory)
66 server_and_port, '/git', 'git', session_factory)
66 connection.Hg = client_http.RemoteVCSMaker(
67 connection.Hg = client_http.RemoteVCSMaker(
67 server_and_port, '/hg', 'hg', session_factory)
68 server_and_port, '/hg', 'hg', session_factory)
68 connection.Svn = client_http.RemoteVCSMaker(
69 connection.Svn = client_http.RemoteVCSMaker(
69 server_and_port, '/svn', 'svn', session_factory)
70 server_and_port, '/svn', 'svn', session_factory)
70 connection.Service = client_http.ServiceConnection(
71 connection.Service = client_http.ServiceConnection(
71 server_and_port, '/_service', session_factory)
72 server_and_port, '/_service', session_factory)
72
73
73 scm_app.HG_REMOTE_WSGI = client_http.VcsHttpProxy(
74 scm_app.HG_REMOTE_WSGI = client_http.VcsHttpProxy(
74 server_and_port, '/proxy/hg')
75 server_and_port, '/proxy/hg')
75 scm_app.GIT_REMOTE_WSGI = client_http.VcsHttpProxy(
76 scm_app.GIT_REMOTE_WSGI = client_http.VcsHttpProxy(
76 server_and_port, '/proxy/git')
77 server_and_port, '/proxy/git')
77
78
78 @atexit.register
79 @atexit.register
79 def free_connection_resources():
80 def free_connection_resources():
80 connection.Git = None
81 connection.Git = None
81 connection.Hg = None
82 connection.Hg = None
82 connection.Svn = None
83 connection.Svn = None
83 connection.Service = None
84 connection.Service = None
84
85
85
86
86 def connect_vcs(server_and_port, protocol):
87 def connect_vcs(server_and_port, protocol):
87 """
88 """
88 Initializes the connection to the vcs server.
89 Initializes the connection to the vcs server.
89
90
90 :param server_and_port: str, e.g. "localhost:9900"
91 :param server_and_port: str, e.g. "localhost:9900"
91 :param protocol: str or "http"
92 :param protocol: str or "http"
92 """
93 """
93 if protocol == 'http':
94 if protocol == 'http':
94 connect_http(server_and_port)
95 connect_http(server_and_port)
95 else:
96 else:
96 raise Exception(f'Invalid vcs server protocol "{protocol}"')
97 raise Exception(f'Invalid vcs server protocol "{protocol}"')
97
98
98
99
99 class CurlSession(object):
100 class CurlSession(object):
100 """
101 """
101 Modeled so that it provides a subset of the requests interface.
102 Modeled so that it provides a subset of the requests interface.
102
103
103 This has been created so that it does only provide a minimal API for our
104 This has been created so that it does only provide a minimal API for our
104 needs. The parts which it provides are based on the API of the library
105 needs. The parts which it provides are based on the API of the library
105 `requests` which allows us to easily benchmark against it.
106 `requests` which allows us to easily benchmark against it.
106
107
107 Please have a look at the class :class:`requests.Session` when you extend
108 Please have a look at the class :class:`requests.Session` when you extend
108 it.
109 it.
109 """
110 """
110 CURL_UA = f'RhodeCode HTTP {rhodecode.__version__}'
111 CURL_UA = f'RhodeCode HTTP {rhodecode.__version__}'
111
112
112 def __init__(self):
113 def __init__(self):
113 curl = pycurl.Curl()
114 curl = pycurl.Curl()
114 # TODO: johbo: I did test with 7.19 of libcurl. This version has
115 # TODO: johbo: I did test with 7.19 of libcurl. This version has
115 # trouble with 100 - continue being set in the expect header. This
116 # trouble with 100 - continue being set in the expect header. This
116 # can lead to massive performance drops, switching it off here.
117 # can lead to massive performance drops, switching it off here.
117
118
118 curl.setopt(curl.TCP_NODELAY, True)
119 curl.setopt(curl.TCP_NODELAY, True)
119 curl.setopt(curl.PROTOCOLS, curl.PROTO_HTTP)
120 curl.setopt(curl.PROTOCOLS, curl.PROTO_HTTP)
120 curl.setopt(curl.USERAGENT, safe_bytes(self.CURL_UA))
121 curl.setopt(curl.USERAGENT, safe_bytes(self.CURL_UA))
121 curl.setopt(curl.SSL_VERIFYPEER, 0)
122 curl.setopt(curl.SSL_VERIFYPEER, 0)
122 curl.setopt(curl.SSL_VERIFYHOST, 0)
123 curl.setopt(curl.SSL_VERIFYHOST, 0)
123 self._curl = curl
124 self._curl = curl
124
125
125 def post(self, url, data, allow_redirects=False, headers=None):
126 def post(self, url, data, allow_redirects=False, headers=None):
126 headers = headers or {}
127 headers = headers or {}
127 # format is ['header_name1: header_value1', 'header_name2: header_value2'])
128 # format is ['header_name1: header_value1', 'header_name2: header_value2'])
128 headers_list = [b"Expect:"] + [safe_bytes('{}: {}'.format(k, v)) for k, v in headers.items()]
129 headers_list = [b"Expect:"] + [safe_bytes('{}: {}'.format(k, v)) for k, v in headers.items()]
129 response_buffer = io.BytesIO()
130 response_buffer = io.BytesIO()
130
131
131 curl = self._curl
132 curl = self._curl
132 curl.setopt(curl.URL, url)
133 curl.setopt(curl.URL, url)
133 curl.setopt(curl.POST, True)
134 curl.setopt(curl.POST, True)
134 curl.setopt(curl.POSTFIELDS, data)
135 curl.setopt(curl.POSTFIELDS, data)
135 curl.setopt(curl.FOLLOWLOCATION, allow_redirects)
136 curl.setopt(curl.FOLLOWLOCATION, allow_redirects)
136 curl.setopt(curl.WRITEDATA, response_buffer)
137 curl.setopt(curl.WRITEDATA, response_buffer)
137 curl.setopt(curl.HTTPHEADER, headers_list)
138 curl.setopt(curl.HTTPHEADER, headers_list)
138 curl.perform()
139 curl.perform()
139
140
140 status_code = curl.getinfo(pycurl.HTTP_CODE)
141 status_code = curl.getinfo(pycurl.HTTP_CODE)
141 content_type = curl.getinfo(pycurl.CONTENT_TYPE)
142 content_type = curl.getinfo(pycurl.CONTENT_TYPE)
142 return CurlResponse(response_buffer, status_code, content_type)
143 return CurlResponse(response_buffer, status_code, content_type)
143
144
144
145
145 class CurlResponse(object):
146 class CurlResponse(object):
146 """
147 """
147 The response of a request, modeled after the requests API.
148 The response of a request, modeled after the requests API.
148
149
149 This class provides a subset of the response interface known from the
150 This class provides a subset of the response interface known from the
150 library `requests`. It is intentionally kept similar, so that we can use
151 library `requests`. It is intentionally kept similar, so that we can use
151 `requests` as a drop in replacement for benchmarking purposes.
152 `requests` as a drop in replacement for benchmarking purposes.
152 """
153 """
153
154
154 def __init__(self, response_buffer, status_code, content_type=''):
155 def __init__(self, response_buffer, status_code, content_type=''):
155 self._response_buffer = response_buffer
156 self._response_buffer = response_buffer
156 self._status_code = status_code
157 self._status_code = status_code
157 self._content_type = content_type
158 self._content_type = content_type
158
159
159 def __repr__(self):
160 def __repr__(self):
160 return f'CurlResponse(code={self._status_code}, content_type={self._content_type})'
161 return f'CurlResponse(code={self._status_code}, content_type={self._content_type})'
161
162
162 @property
163 @property
163 def content(self):
164 def content(self):
164 try:
165 try:
165 return self._response_buffer.getvalue()
166 return self._response_buffer.getvalue()
166 finally:
167 finally:
167 self._response_buffer.close()
168 self._response_buffer.close()
168
169
169 @property
170 @property
170 def status_code(self):
171 def status_code(self):
171 return self._status_code
172 return self._status_code
172
173
173 @property
174 @property
174 def content_type(self):
175 def content_type(self):
175 return self._content_type
176 return self._content_type
176
177
177 def iter_content(self, chunk_size):
178 def iter_content(self, chunk_size):
178 self._response_buffer.seek(0)
179 self._response_buffer.seek(0)
179 while 1:
180 while 1:
180 chunk = self._response_buffer.read(chunk_size)
181 chunk = self._response_buffer.read(chunk_size)
181 if not chunk:
182 if not chunk:
182 break
183 break
183 yield chunk
184 yield chunk
184
185
185
186
186 def _create_http_rpc_session():
187 def _create_http_rpc_session():
187 session = CurlSession()
188 session = CurlSession()
188 return session
189 return session
General Comments 0
You need to be logged in to leave comments. Login now