Show More
__init__.py
248 lines
| 7.9 KiB
| text/x-python
|
PythonLexer
r1 | # -*- coding: utf-8 -*- | |||
r1271 | # Copyright (C) 2014-2017 RhodeCode GmbH | |||
r1 | # | |||
# This program is free software: you can redistribute it and/or modify | ||||
# it under the terms of the GNU Affero General Public License, version 3 | ||||
# (only), as published by the Free Software Foundation. | ||||
# | ||||
# This program is distributed in the hope that it will be useful, | ||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||||
# GNU General Public License for more details. | ||||
# | ||||
# You should have received a copy of the GNU Affero General Public License | ||||
# along with this program. If not, see <http://www.gnu.org/licenses/>. | ||||
# | ||||
# This program is dual-licensed. If you wish to learn more about the | ||||
# RhodeCode Enterprise Edition, including its added features, Support services, | ||||
# and proprietary license terms, please see https://rhodecode.com/licenses/ | ||||
""" | ||||
Various version Control System version lib (vcs) management abstraction layer | ||||
for Python. Build with server client architecture. | ||||
""" | ||||
VERSION = (0, 5, 0, 'dev') | ||||
__version__ = '.'.join((str(each) for each in VERSION[:4])) | ||||
__all__ = [ | ||||
Martin Bornhold
|
r483 | 'get_version', 'get_vcs_instance', 'get_backend', | ||
r1 | 'VCSError', 'RepositoryError', 'CommitError' | |||
] | ||||
import atexit | ||||
import logging | ||||
Martin Bornhold
|
r1007 | import subprocess32 | ||
r1 | import time | |||
import urlparse | ||||
from cStringIO import StringIO | ||||
from rhodecode.lib.vcs.conf import settings | ||||
Martin Bornhold
|
r483 | from rhodecode.lib.vcs.backends import get_vcs_instance, get_backend | ||
r1 | from rhodecode.lib.vcs.exceptions import ( | |||
Martin Bornhold
|
r964 | VCSError, RepositoryError, CommitError, VCSCommunicationError) | ||
r1 | ||||
Martin Bornhold
|
r474 | log = logging.getLogger(__name__) | ||
r1 | ||||
Martin Bornhold
|
r474 | # The pycurl library directly accesses C API functions and is not patched by | ||
# gevent. This will potentially lead to deadlocks due to incompatibility to | ||||
# gevent. Therefore we check if gevent is active and import a gevent compatible | ||||
# wrapper in that case. | ||||
try: | ||||
from gevent import monkey | ||||
if monkey.is_module_patched('__builtin__'): | ||||
import geventcurl as pycurl | ||||
log.debug('Using gevent comapatible pycurl: %s', pycurl) | ||||
else: | ||||
import pycurl | ||||
except ImportError: | ||||
import pycurl | ||||
r1 | ||||
def get_version(): | ||||
""" | ||||
Returns shorter version (digit parts only) as string. | ||||
""" | ||||
return '.'.join((str(each) for each in VERSION[:3])) | ||||
def connect_http(server_and_port): | ||||
from rhodecode.lib.vcs import connection, client_http | ||||
from rhodecode.lib.middleware.utils import scm_app | ||||
Martin Bornhold
|
r244 | session_factory = client_http.ThreadlocalSessionFactory() | ||
r1 | ||||
Martin Bornhold
|
r244 | connection.Git = client_http.RepoMaker( | ||
r1126 | server_and_port, '/git', 'git', session_factory) | |||
Martin Bornhold
|
r244 | connection.Hg = client_http.RepoMaker( | ||
r1126 | server_and_port, '/hg', 'hg', session_factory) | |||
Martin Bornhold
|
r244 | connection.Svn = client_http.RepoMaker( | ||
r1126 | server_and_port, '/svn', 'svn', session_factory) | |||
r1111 | connection.Service = client_http.ServiceConnection( | |||
server_and_port, '/_service', session_factory) | ||||
r1 | ||||
scm_app.HG_REMOTE_WSGI = client_http.VcsHttpProxy( | ||||
server_and_port, '/proxy/hg') | ||||
scm_app.GIT_REMOTE_WSGI = client_http.VcsHttpProxy( | ||||
server_and_port, '/proxy/git') | ||||
@atexit.register | ||||
def free_connection_resources(): | ||||
connection.Git = None | ||||
connection.Hg = None | ||||
connection.Svn = None | ||||
r1111 | connection.Service = None | |||
r1 | ||||
Martin Bornhold
|
r959 | def connect_vcs(server_and_port, protocol): | ||
r1 | """ | |||
Initializes the connection to the vcs server. | ||||
:param server_and_port: str, e.g. "localhost:9900" | ||||
r1409 | :param protocol: str or "http" | |||
r1 | """ | |||
r1409 | if protocol == 'http': | |||
r1 | connect_http(server_and_port) | |||
Martin Bornhold
|
r960 | else: | ||
raise Exception('Invalid vcs server protocol "{}"'.format(protocol)) | ||||
r1 | ||||
# TODO: johbo: This function should be moved into our test suite, there is | ||||
# no reason to support starting the vcsserver in Enterprise itself. | ||||
Martin Bornhold
|
r959 | def start_vcs_server(server_and_port, protocol, log_level=None): | ||
r1 | """ | |||
Starts the vcs server in a subprocess. | ||||
""" | ||||
log.info('Starting VCSServer as a sub process with %s protocol', protocol) | ||||
if protocol == 'http': | ||||
return _start_http_vcs_server(server_and_port, log_level) | ||||
Martin Bornhold
|
r960 | else: | ||
raise Exception('Invalid vcs server protocol "{}"'.format(protocol)) | ||||
r1 | ||||
def _start_http_vcs_server(server_and_port, log_level=None): | ||||
# TODO: mikhail: shutdown if an http server already runs | ||||
host, port = server_and_port.rsplit(":", 1) | ||||
args = [ | ||||
Martin Bornhold
|
r964 | 'pserve', 'rhodecode/tests/vcsserver_http.ini', | ||
r1 | 'http_port=%s' % (port, ), 'http_host=%s' % (host, )] | |||
Martin Bornhold
|
r1007 | proc = subprocess32.Popen(args) | ||
r1 | ||||
def cleanup_server_process(): | ||||
proc.kill() | ||||
atexit.register(cleanup_server_process) | ||||
server = create_vcsserver_proxy(server_and_port, protocol='http') | ||||
_wait_until_vcs_server_is_reachable(server) | ||||
Martin Bornhold
|
r964 | def _wait_until_vcs_server_is_reachable(server, timeout=40): | ||
begin = time.time() | ||||
while (time.time() - begin) < timeout: | ||||
r1 | try: | |||
server.ping() | ||||
Martin Bornhold
|
r964 | return | ||
r1409 | except (VCSCommunicationError, pycurl.error): | |||
Martin Bornhold
|
r964 | log.debug('VCSServer not started yet, retry to connect.') | ||
r1 | time.sleep(0.5) | |||
Martin Bornhold
|
r964 | raise Exception( | ||
'Starting the VCSServer failed or took more than {} ' | ||||
'seconds.'.format(timeout)) | ||||
r1 | ||||
Martin Bornhold
|
r959 | def _try_to_shutdown_running_server(server_and_port, protocol): | ||
server = create_vcsserver_proxy(server_and_port, protocol) | ||||
r1 | try: | |||
server.shutdown() | ||||
r1409 | except pycurl.error: | |||
r1 | return | |||
# TODO: Not sure why this is important, but without it the following start | ||||
# of the server fails. | ||||
Martin Bornhold
|
r959 | server = create_vcsserver_proxy(server_and_port, protocol) | ||
r1 | server.ping() | |||
Martin Bornhold
|
r959 | def create_vcsserver_proxy(server_and_port, protocol): | ||
r1409 | if protocol == 'http': | |||
r1 | return _create_vcsserver_proxy_http(server_and_port) | |||
Martin Bornhold
|
r960 | else: | ||
raise Exception('Invalid vcs server protocol "{}"'.format(protocol)) | ||||
r1 | ||||
def _create_vcsserver_proxy_http(server_and_port): | ||||
from rhodecode.lib.vcs import client_http | ||||
session = _create_http_rpc_session() | ||||
url = urlparse.urljoin('http://%s' % server_and_port, '/server') | ||||
return client_http.RemoteObject(url, session) | ||||
class CurlSession(object): | ||||
""" | ||||
Modeled so that it provides a subset of the requests interface. | ||||
This has been created so that it does only provide a minimal API for our | ||||
needs. The parts which it provides are based on the API of the library | ||||
`requests` which allows us to easily benchmark against it. | ||||
Please have a look at the class :class:`requests.Session` when you extend | ||||
it. | ||||
""" | ||||
def __init__(self): | ||||
curl = pycurl.Curl() | ||||
# TODO: johbo: I did test with 7.19 of libcurl. This version has | ||||
# trouble with 100 - continue being set in the expect header. This | ||||
# can lead to massive performance drops, switching it off here. | ||||
curl.setopt(curl.HTTPHEADER, ["Expect:"]) | ||||
curl.setopt(curl.TCP_NODELAY, True) | ||||
curl.setopt(curl.PROTOCOLS, curl.PROTO_HTTP) | ||||
self._curl = curl | ||||
def post(self, url, data, allow_redirects=False): | ||||
response_buffer = StringIO() | ||||
curl = self._curl | ||||
curl.setopt(curl.URL, url) | ||||
curl.setopt(curl.POST, True) | ||||
curl.setopt(curl.POSTFIELDS, data) | ||||
curl.setopt(curl.FOLLOWLOCATION, allow_redirects) | ||||
curl.setopt(curl.WRITEDATA, response_buffer) | ||||
curl.perform() | ||||
r1410 | status_code = curl.getinfo(pycurl.HTTP_CODE) | |||
return CurlResponse(response_buffer, status_code) | ||||
r1 | ||||
class CurlResponse(object): | ||||
""" | ||||
The response of a request, modeled after the requests API. | ||||
This class provides a subset of the response interface known from the | ||||
library `requests`. It is intentionally kept similar, so that we can use | ||||
`requests` as a drop in replacement for benchmarking purposes. | ||||
""" | ||||
r1410 | def __init__(self, response_buffer, status_code): | |||
r1 | self._response_buffer = response_buffer | |||
r1410 | self._status_code = status_code | |||
r1 | ||||
@property | ||||
def content(self): | ||||
return self._response_buffer.getvalue() | ||||
r1410 | @property | |||
def status_code(self): | ||||
return self._status_code | ||||
r1 | ||||
def _create_http_rpc_session(): | ||||
session = CurlSession() | ||||
return session | ||||