##// END OF EJS Templates
binary envelope: add support for custom response type when using wrapped binary data in envelope
super-admin -
r1107:eadcb91a python3
parent child Browse files
Show More
@@ -1,178 +1,178 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 import os
17 import os
18 import sys
18 import sys
19 import traceback
19 import traceback
20 import logging
20 import logging
21 import urllib.parse
21 import urllib.parse
22
22
23 from vcsserver.lib.rc_cache import region_meta
23 from vcsserver.lib.rc_cache import region_meta
24
24
25 from vcsserver import exceptions
25 from vcsserver import exceptions
26 from vcsserver.exceptions import NoContentException
26 from vcsserver.exceptions import NoContentException
27 from vcsserver.hgcompat import archival
27 from vcsserver.hgcompat import archival
28 from vcsserver.str_utils import safe_bytes
28 from vcsserver.str_utils import safe_bytes
29
29
30 log = logging.getLogger(__name__)
30 log = logging.getLogger(__name__)
31
31
32
32
33 class RepoFactory(object):
33 class RepoFactory(object):
34 """
34 """
35 Utility to create instances of repository
35 Utility to create instances of repository
36
36
37 It provides internal caching of the `repo` object based on
37 It provides internal caching of the `repo` object based on
38 the :term:`call context`.
38 the :term:`call context`.
39 """
39 """
40 repo_type = None
40 repo_type = None
41
41
42 def __init__(self):
42 def __init__(self):
43 self._cache_region = region_meta.dogpile_cache_regions['repo_object']
43 self._cache_region = region_meta.dogpile_cache_regions['repo_object']
44
44
45 def _create_config(self, path, config):
45 def _create_config(self, path, config):
46 config = {}
46 config = {}
47 return config
47 return config
48
48
49 def _create_repo(self, wire, create):
49 def _create_repo(self, wire, create):
50 raise NotImplementedError()
50 raise NotImplementedError()
51
51
52 def repo(self, wire, create=False):
52 def repo(self, wire, create=False):
53 raise NotImplementedError()
53 raise NotImplementedError()
54
54
55
55
56 def obfuscate_qs(query_string):
56 def obfuscate_qs(query_string):
57 if query_string is None:
57 if query_string is None:
58 return None
58 return None
59
59
60 parsed = []
60 parsed = []
61 for k, v in urllib.parse.parse_qsl(query_string, keep_blank_values=True):
61 for k, v in urllib.parse.parse_qsl(query_string, keep_blank_values=True):
62 if k in ['auth_token', 'api_key']:
62 if k in ['auth_token', 'api_key']:
63 v = "*****"
63 v = "*****"
64 parsed.append((k, v))
64 parsed.append((k, v))
65
65
66 return '&'.join('{}{}'.format(
66 return '&'.join('{}{}'.format(
67 k, '={}'.format(v) if v else '') for k, v in parsed)
67 k, '={}'.format(v) if v else '') for k, v in parsed)
68
68
69
69
70 def raise_from_original(new_type, org_exc: Exception):
70 def raise_from_original(new_type, org_exc: Exception):
71 """
71 """
72 Raise a new exception type with original args and traceback.
72 Raise a new exception type with original args and traceback.
73 """
73 """
74
74
75 exc_type, exc_value, exc_traceback = sys.exc_info()
75 exc_type, exc_value, exc_traceback = sys.exc_info()
76 new_exc = new_type(*exc_value.args)
76 new_exc = new_type(*exc_value.args)
77
77
78 # store the original traceback into the new exc
78 # store the original traceback into the new exc
79 new_exc._org_exc_tb = traceback.format_tb(exc_traceback)
79 new_exc._org_exc_tb = traceback.format_tb(exc_traceback)
80
80
81 try:
81 try:
82 raise new_exc.with_traceback(exc_traceback)
82 raise new_exc.with_traceback(exc_traceback)
83 finally:
83 finally:
84 del exc_traceback
84 del exc_traceback
85
85
86
86
87 class ArchiveNode(object):
87 class ArchiveNode(object):
88 def __init__(self, path, mode, is_link, raw_bytes):
88 def __init__(self, path, mode, is_link, raw_bytes):
89 self.path = path
89 self.path = path
90 self.mode = mode
90 self.mode = mode
91 self.is_link = is_link
91 self.is_link = is_link
92 self.raw_bytes = raw_bytes
92 self.raw_bytes = raw_bytes
93
93
94
94
95 def archive_repo(walker, archive_dest_path, kind, mtime, archive_at_path,
95 def archive_repo(walker, archive_dest_path, kind, mtime, archive_at_path,
96 archive_dir_name, commit_id, write_metadata=True, extra_metadata=None):
96 archive_dir_name, commit_id, write_metadata=True, extra_metadata=None):
97 """
97 """
98 walker should be a file walker, for example:
98 walker should be a file walker, for example:
99 def walker():
99 def walker():
100 for file_info in files:
100 for file_info in files:
101 yield ArchiveNode(fn, mode, is_link, ctx[fn].data)
101 yield ArchiveNode(fn, mode, is_link, ctx[fn].data)
102 """
102 """
103 extra_metadata = extra_metadata or {}
103 extra_metadata = extra_metadata or {}
104 archive_dest_path = safe_bytes(archive_dest_path)
104 archive_dest_path = safe_bytes(archive_dest_path)
105
105
106 if kind == "tgz":
106 if kind == "tgz":
107 archiver = archival.tarit(archive_dest_path, mtime, b"gz")
107 archiver = archival.tarit(archive_dest_path, mtime, b"gz")
108 elif kind == "tbz2":
108 elif kind == "tbz2":
109 archiver = archival.tarit(archive_dest_path, mtime, b"bz2")
109 archiver = archival.tarit(archive_dest_path, mtime, b"bz2")
110 elif kind == 'zip':
110 elif kind == 'zip':
111 archiver = archival.zipit(archive_dest_path, mtime)
111 archiver = archival.zipit(archive_dest_path, mtime)
112 else:
112 else:
113 raise exceptions.ArchiveException()(
113 raise exceptions.ArchiveException()(
114 f'Remote does not support: "{kind}" archive type.')
114 f'Remote does not support: "{kind}" archive type.')
115
115
116 for f in walker(commit_id, archive_at_path):
116 for f in walker(commit_id, archive_at_path):
117 f_path = os.path.join(safe_bytes(archive_dir_name), safe_bytes(f.path).lstrip(b'/'))
117 f_path = os.path.join(safe_bytes(archive_dir_name), safe_bytes(f.path).lstrip(b'/'))
118 try:
118 try:
119 archiver.addfile(f_path, f.mode, f.is_link, f.raw_bytes())
119 archiver.addfile(f_path, f.mode, f.is_link, f.raw_bytes())
120 except NoContentException:
120 except NoContentException:
121 # NOTE(marcink): this is a special case for SVN so we can create "empty"
121 # NOTE(marcink): this is a special case for SVN so we can create "empty"
122 # directories which arent supported by archiver
122 # directories which arent supported by archiver
123 archiver.addfile(os.path.join(f_path, b'.dir'), f.mode, f.is_link, b'')
123 archiver.addfile(os.path.join(f_path, b'.dir'), f.mode, f.is_link, b'')
124
124
125 if write_metadata:
125 if write_metadata:
126 metadata = dict([
126 metadata = dict([
127 ('commit_id', commit_id),
127 ('commit_id', commit_id),
128 ('mtime', mtime),
128 ('mtime', mtime),
129 ])
129 ])
130 metadata.update(extra_metadata)
130 metadata.update(extra_metadata)
131
131
132 meta = [safe_bytes(f"{f_name}:{value}") for f_name, value in metadata.items()]
132 meta = [safe_bytes(f"{f_name}:{value}") for f_name, value in metadata.items()]
133 f_path = os.path.join(safe_bytes(archive_dir_name), b'.archival.txt')
133 f_path = os.path.join(safe_bytes(archive_dir_name), b'.archival.txt')
134 archiver.addfile(f_path, 0o644, False, b'\n'.join(meta))
134 archiver.addfile(f_path, 0o644, False, b'\n'.join(meta))
135
135
136 return archiver.done()
136 return archiver.done()
137
137
138
138
139 class BinaryEnvelope(object):
139 class BinaryEnvelope(object):
140 def __init__(self, value, bin_type=True):
140 def __init__(self, value: bytes, bin_type=True):
141 self.value = value
141 self.value = value
142 self.bin_type = bin_type
142 self.bin_type = bin_type
143
143
144 def __len__(self):
144 def __len__(self):
145 return len(self.value)
145 return len(self.value)
146
146
147 def __getitem__(self, index):
147 def __getitem__(self, index):
148 return self.value[index]
148 return self.value[index]
149
149
150 def __iter__(self):
150 def __iter__(self):
151 return iter(self.value)
151 return iter(self.value)
152
152
153 def __str__(self):
153 def __str__(self):
154 return str(self.value)
154 return str(self.value)
155
155
156 def __repr__(self):
156 def __repr__(self):
157 return repr(self.value)
157 return repr(self.value)
158
158
159 def __eq__(self, other):
159 def __eq__(self, other):
160 if isinstance(other, BinaryEnvelope):
160 if isinstance(other, BinaryEnvelope):
161 return self.value == other.value
161 return self.value == other.value
162 return False
162 return False
163
163
164 def __ne__(self, other):
164 def __ne__(self, other):
165 return not self.__eq__(other)
165 return not self.__eq__(other)
166
166
167 def __add__(self, other):
167 def __add__(self, other):
168 if isinstance(other, BinaryEnvelope):
168 if isinstance(other, BinaryEnvelope):
169 return BinaryEnvelope(self.value + other.value)
169 return BinaryEnvelope(self.value + other.value)
170 raise TypeError(f"unsupported operand type(s) for +: 'BinaryEnvelope' and '{type(other)}'")
170 raise TypeError(f"unsupported operand type(s) for +: 'BinaryEnvelope' and '{type(other)}'")
171
171
172 def __radd__(self, other):
172 def __radd__(self, other):
173 if isinstance(other, BinaryEnvelope):
173 if isinstance(other, BinaryEnvelope):
174 return BinaryEnvelope(other.value + self.value)
174 return BinaryEnvelope(other.value + self.value)
175 raise TypeError(f"unsupported operand type(s) for +: '{type(other)}' and 'BinaryEnvelope'")
175 raise TypeError(f"unsupported operand type(s) for +: '{type(other)}' and 'BinaryEnvelope'")
176
176
177
177
178
178
@@ -1,87 +1,88 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 """
18 """
19 Mercurial libs compatibility
19 Mercurial libs compatibility
20 """
20 """
21
21
22 import mercurial
22 import mercurial
23 from mercurial import demandimport
23 from mercurial import demandimport
24
24
25 # patch demandimport, due to bug in mercurial when it always triggers
25 # patch demandimport, due to bug in mercurial when it always triggers
26 # demandimport.enable()
26 # demandimport.enable()
27 from vcsserver.str_utils import safe_bytes
27 from vcsserver.str_utils import safe_bytes
28
28
29 demandimport.enable = lambda *args, **kwargs: 1
29 demandimport.enable = lambda *args, **kwargs: 1
30
30
31 from mercurial import ui
31 from mercurial import ui
32 from mercurial import patch
32 from mercurial import patch
33 from mercurial import config
33 from mercurial import config
34 from mercurial import extensions
34 from mercurial import extensions
35 from mercurial import scmutil
35 from mercurial import scmutil
36 from mercurial import archival
36 from mercurial import archival
37 from mercurial import discovery
37 from mercurial import discovery
38 from mercurial import unionrepo
38 from mercurial import unionrepo
39 from mercurial import localrepo
39 from mercurial import localrepo
40 from mercurial import merge as hg_merge
40 from mercurial import merge as hg_merge
41 from mercurial import subrepo
41 from mercurial import subrepo
42 from mercurial import subrepoutil
42 from mercurial import subrepoutil
43 from mercurial import tags as hg_tag
43 from mercurial import tags as hg_tag
44 from mercurial import util as hgutil
44 from mercurial import util as hgutil
45 from mercurial.commands import clone, pull
45 from mercurial.commands import clone, pull
46 from mercurial.node import nullid
46 from mercurial.node import nullid
47 from mercurial.context import memctx, memfilectx
47 from mercurial.context import memctx, memfilectx
48 from mercurial.error import (
48 from mercurial.error import (
49 LookupError, RepoError, RepoLookupError, Abort, InterventionRequired,
49 LookupError, RepoError, RepoLookupError, Abort, InterventionRequired,
50 RequirementError, ProgrammingError)
50 RequirementError, ProgrammingError)
51 from mercurial.hgweb import hgweb_mod
51 from mercurial.hgweb import hgweb_mod
52 from mercurial.localrepo import instance
52 from mercurial.localrepo import instance
53 from mercurial.match import match, alwaysmatcher, patternmatcher
53 from mercurial.match import match, alwaysmatcher, patternmatcher
54 from mercurial.mdiff import diffopts
54 from mercurial.mdiff import diffopts
55 from mercurial.node import bin, hex
55 from mercurial.node import bin, hex
56 from mercurial.encoding import tolocal
56 from mercurial.encoding import tolocal
57 from mercurial.discovery import findcommonoutgoing
57 from mercurial.discovery import findcommonoutgoing
58 from mercurial.hg import peer
58 from mercurial.hg import peer
59 from mercurial.httppeer import makepeer
59 from mercurial.httppeer import makepeer
60 from mercurial.utils.urlutil import url as hg_url
60 from mercurial.utils.urlutil import url as hg_url
61 from mercurial.scmutil import revrange, revsymbol
61 from mercurial.scmutil import revrange, revsymbol
62 from mercurial.node import nullrev
62 from mercurial.node import nullrev
63 from mercurial import exchange
63 from mercurial import exchange
64 from hgext import largefiles
64 from hgext import largefiles
65
65
66 # those authnadlers are patched for python 2.6.5 bug an
66 # those authnadlers are patched for python 2.6.5 bug an
67 # infinit looping when given invalid resources
67 # infinit looping when given invalid resources
68 from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
68 from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
69
69
70 # hg strip is in core now
70 # hg strip is in core now
71 from mercurial import strip as hgext_strip
71 from mercurial import strip as hgext_strip
72
72
73
73
74 def get_ctx(repo, ref):
74 def get_ctx(repo, ref):
75 if not isinstance(ref, int):
75 ref = safe_bytes(ref)
76 ref = safe_bytes(ref)
76 try:
77 try:
77 ctx = repo[ref]
78 ctx = repo[ref]
78 except (ProgrammingError, TypeError):
79 except (ProgrammingError, TypeError):
79 # we're unable to find the rev using a regular lookup, we fallback
80 # we're unable to find the rev using a regular lookup, we fallback
80 # to slower, but backward compat revsymbol usage
81 # to slower, but backward compat revsymbol usage
81 ctx = revsymbol(repo, ref)
82 ctx = revsymbol(repo, ref)
82 except (LookupError, RepoLookupError):
83 except (LookupError, RepoLookupError):
83 # Similar case as above but only for refs that are not numeric
84 # Similar case as above but only for refs that are not numeric
84 if isinstance(ref, int):
85 if isinstance(ref, int):
85 raise
86 raise
86 ctx = revsymbol(repo, ref)
87 ctx = revsymbol(repo, ref)
87 return ctx
88 return ctx
@@ -1,770 +1,773 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import io
18 import io
19 import os
19 import os
20 import sys
20 import sys
21 import base64
21 import base64
22 import locale
22 import locale
23 import logging
23 import logging
24 import uuid
24 import uuid
25 import time
25 import time
26 import wsgiref.util
26 import wsgiref.util
27 import traceback
27 import traceback
28 import tempfile
28 import tempfile
29 import psutil
29 import psutil
30
30
31 from itertools import chain
31 from itertools import chain
32
32
33 import msgpack
33 import msgpack
34 import configparser
34 import configparser
35
35
36 from pyramid.config import Configurator
36 from pyramid.config import Configurator
37 from pyramid.wsgi import wsgiapp
37 from pyramid.wsgi import wsgiapp
38 from pyramid.response import Response
38 from pyramid.response import Response
39
39
40 from vcsserver.base import BinaryEnvelope
40 from vcsserver.base import BinaryEnvelope
41 from vcsserver.lib.rc_json import json
41 from vcsserver.lib.rc_json import json
42 from vcsserver.config.settings_maker import SettingsMaker
42 from vcsserver.config.settings_maker import SettingsMaker
43 from vcsserver.str_utils import safe_int, safe_bytes, safe_str
43 from vcsserver.str_utils import safe_int, safe_bytes, safe_str
44 from vcsserver.lib.statsd_client import StatsdClient
44 from vcsserver.lib.statsd_client import StatsdClient
45
45
46 log = logging.getLogger(__name__)
46 log = logging.getLogger(__name__)
47
47
48 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
48 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
49 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
49 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
50
50
51 try:
51 try:
52 locale.setlocale(locale.LC_ALL, '')
52 locale.setlocale(locale.LC_ALL, '')
53 except locale.Error as e:
53 except locale.Error as e:
54 log.error(
54 log.error(
55 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
55 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
56 os.environ['LC_ALL'] = 'C'
56 os.environ['LC_ALL'] = 'C'
57
57
58
58
59 import vcsserver
59 import vcsserver
60 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
60 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
61 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
61 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
62 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
62 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
63 from vcsserver.echo_stub.echo_app import EchoApp
63 from vcsserver.echo_stub.echo_app import EchoApp
64 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
64 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
65 from vcsserver.lib.exc_tracking import store_exception
65 from vcsserver.lib.exc_tracking import store_exception
66 from vcsserver.server import VcsServer
66 from vcsserver.server import VcsServer
67
67
68 strict_vcs = True
68 strict_vcs = True
69
69
70 git_import_err = None
70 git_import_err = None
71 try:
71 try:
72 from vcsserver.remote.git import GitFactory, GitRemote
72 from vcsserver.remote.git import GitFactory, GitRemote
73 except ImportError as e:
73 except ImportError as e:
74 GitFactory = None
74 GitFactory = None
75 GitRemote = None
75 GitRemote = None
76 git_import_err = e
76 git_import_err = e
77 if strict_vcs:
77 if strict_vcs:
78 raise
78 raise
79
79
80
80
81 hg_import_err = None
81 hg_import_err = None
82 try:
82 try:
83 from vcsserver.remote.hg import MercurialFactory, HgRemote
83 from vcsserver.remote.hg import MercurialFactory, HgRemote
84 except ImportError as e:
84 except ImportError as e:
85 MercurialFactory = None
85 MercurialFactory = None
86 HgRemote = None
86 HgRemote = None
87 hg_import_err = e
87 hg_import_err = e
88 if strict_vcs:
88 if strict_vcs:
89 raise
89 raise
90
90
91
91
92 svn_import_err = None
92 svn_import_err = None
93 try:
93 try:
94 from vcsserver.remote.svn import SubversionFactory, SvnRemote
94 from vcsserver.remote.svn import SubversionFactory, SvnRemote
95 except ImportError as e:
95 except ImportError as e:
96 SubversionFactory = None
96 SubversionFactory = None
97 SvnRemote = None
97 SvnRemote = None
98 svn_import_err = e
98 svn_import_err = e
99 if strict_vcs:
99 if strict_vcs:
100 raise
100 raise
101
101
102
102
103 def _is_request_chunked(environ):
103 def _is_request_chunked(environ):
104 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
104 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
105 return stream
105 return stream
106
106
107
107
108 def log_max_fd():
108 def log_max_fd():
109 try:
109 try:
110 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
110 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
111 log.info('Max file descriptors value: %s', maxfd)
111 log.info('Max file descriptors value: %s', maxfd)
112 except Exception:
112 except Exception:
113 pass
113 pass
114
114
115
115
116 class VCS(object):
116 class VCS(object):
117 def __init__(self, locale_conf=None, cache_config=None):
117 def __init__(self, locale_conf=None, cache_config=None):
118 self.locale = locale_conf
118 self.locale = locale_conf
119 self.cache_config = cache_config
119 self.cache_config = cache_config
120 self._configure_locale()
120 self._configure_locale()
121
121
122 log_max_fd()
122 log_max_fd()
123
123
124 if GitFactory and GitRemote:
124 if GitFactory and GitRemote:
125 git_factory = GitFactory()
125 git_factory = GitFactory()
126 self._git_remote = GitRemote(git_factory)
126 self._git_remote = GitRemote(git_factory)
127 else:
127 else:
128 log.error("Git client import failed: %s", git_import_err)
128 log.error("Git client import failed: %s", git_import_err)
129
129
130 if MercurialFactory and HgRemote:
130 if MercurialFactory and HgRemote:
131 hg_factory = MercurialFactory()
131 hg_factory = MercurialFactory()
132 self._hg_remote = HgRemote(hg_factory)
132 self._hg_remote = HgRemote(hg_factory)
133 else:
133 else:
134 log.error("Mercurial client import failed: %s", hg_import_err)
134 log.error("Mercurial client import failed: %s", hg_import_err)
135
135
136 if SubversionFactory and SvnRemote:
136 if SubversionFactory and SvnRemote:
137 svn_factory = SubversionFactory()
137 svn_factory = SubversionFactory()
138
138
139 # hg factory is used for svn url validation
139 # hg factory is used for svn url validation
140 hg_factory = MercurialFactory()
140 hg_factory = MercurialFactory()
141 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
141 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
142 else:
142 else:
143 log.error("Subversion client import failed: %s", svn_import_err)
143 log.error("Subversion client import failed: %s", svn_import_err)
144
144
145 self._vcsserver = VcsServer()
145 self._vcsserver = VcsServer()
146
146
147 def _configure_locale(self):
147 def _configure_locale(self):
148 if self.locale:
148 if self.locale:
149 log.info('Settings locale: `LC_ALL` to %s', self.locale)
149 log.info('Settings locale: `LC_ALL` to %s', self.locale)
150 else:
150 else:
151 log.info('Configuring locale subsystem based on environment variables')
151 log.info('Configuring locale subsystem based on environment variables')
152 try:
152 try:
153 # If self.locale is the empty string, then the locale
153 # If self.locale is the empty string, then the locale
154 # module will use the environment variables. See the
154 # module will use the environment variables. See the
155 # documentation of the package `locale`.
155 # documentation of the package `locale`.
156 locale.setlocale(locale.LC_ALL, self.locale)
156 locale.setlocale(locale.LC_ALL, self.locale)
157
157
158 language_code, encoding = locale.getlocale()
158 language_code, encoding = locale.getlocale()
159 log.info(
159 log.info(
160 'Locale set to language code "%s" with encoding "%s".',
160 'Locale set to language code "%s" with encoding "%s".',
161 language_code, encoding)
161 language_code, encoding)
162 except locale.Error:
162 except locale.Error:
163 log.exception('Cannot set locale, not configuring the locale system')
163 log.exception('Cannot set locale, not configuring the locale system')
164
164
165
165
166 class WsgiProxy(object):
166 class WsgiProxy(object):
167 def __init__(self, wsgi):
167 def __init__(self, wsgi):
168 self.wsgi = wsgi
168 self.wsgi = wsgi
169
169
170 def __call__(self, environ, start_response):
170 def __call__(self, environ, start_response):
171 input_data = environ['wsgi.input'].read()
171 input_data = environ['wsgi.input'].read()
172 input_data = msgpack.unpackb(input_data)
172 input_data = msgpack.unpackb(input_data)
173
173
174 error = None
174 error = None
175 try:
175 try:
176 data, status, headers = self.wsgi.handle(
176 data, status, headers = self.wsgi.handle(
177 input_data['environment'], input_data['input_data'],
177 input_data['environment'], input_data['input_data'],
178 *input_data['args'], **input_data['kwargs'])
178 *input_data['args'], **input_data['kwargs'])
179 except Exception as e:
179 except Exception as e:
180 data, status, headers = [], None, None
180 data, status, headers = [], None, None
181 error = {
181 error = {
182 'message': str(e),
182 'message': str(e),
183 '_vcs_kind': getattr(e, '_vcs_kind', None)
183 '_vcs_kind': getattr(e, '_vcs_kind', None)
184 }
184 }
185
185
186 start_response(200, {})
186 start_response(200, {})
187 return self._iterator(error, status, headers, data)
187 return self._iterator(error, status, headers, data)
188
188
189 def _iterator(self, error, status, headers, data):
189 def _iterator(self, error, status, headers, data):
190 initial_data = [
190 initial_data = [
191 error,
191 error,
192 status,
192 status,
193 headers,
193 headers,
194 ]
194 ]
195
195
196 for d in chain(initial_data, data):
196 for d in chain(initial_data, data):
197 yield msgpack.packb(d)
197 yield msgpack.packb(d)
198
198
199
199
200 def not_found(request):
200 def not_found(request):
201 return {'status': '404 NOT FOUND'}
201 return {'status': '404 NOT FOUND'}
202
202
203
203
204 class VCSViewPredicate(object):
204 class VCSViewPredicate(object):
205 def __init__(self, val, config):
205 def __init__(self, val, config):
206 self.remotes = val
206 self.remotes = val
207
207
208 def text(self):
208 def text(self):
209 return 'vcs view method = %s' % (list(self.remotes.keys()),)
209 return 'vcs view method = %s' % (list(self.remotes.keys()),)
210
210
211 phash = text
211 phash = text
212
212
213 def __call__(self, context, request):
213 def __call__(self, context, request):
214 """
214 """
215 View predicate that returns true if given backend is supported by
215 View predicate that returns true if given backend is supported by
216 defined remotes.
216 defined remotes.
217 """
217 """
218 backend = request.matchdict.get('backend')
218 backend = request.matchdict.get('backend')
219 return backend in self.remotes
219 return backend in self.remotes
220
220
221
221
222 class HTTPApplication(object):
222 class HTTPApplication(object):
223 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
223 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
224
224
225 remote_wsgi = remote_wsgi
225 remote_wsgi = remote_wsgi
226 _use_echo_app = False
226 _use_echo_app = False
227
227
228 def __init__(self, settings=None, global_config=None):
228 def __init__(self, settings=None, global_config=None):
229
229
230 self.config = Configurator(settings=settings)
230 self.config = Configurator(settings=settings)
231 # Init our statsd at very start
231 # Init our statsd at very start
232 self.config.registry.statsd = StatsdClient.statsd
232 self.config.registry.statsd = StatsdClient.statsd
233 self.config.registry.vcs_call_context = {}
233 self.config.registry.vcs_call_context = {}
234
234
235 self.global_config = global_config
235 self.global_config = global_config
236 self.config.include('vcsserver.lib.rc_cache')
236 self.config.include('vcsserver.lib.rc_cache')
237
237
238 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
238 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
239 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
239 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
240 self._remotes = {
240 self._remotes = {
241 'hg': vcs._hg_remote,
241 'hg': vcs._hg_remote,
242 'git': vcs._git_remote,
242 'git': vcs._git_remote,
243 'svn': vcs._svn_remote,
243 'svn': vcs._svn_remote,
244 'server': vcs._vcsserver,
244 'server': vcs._vcsserver,
245 }
245 }
246 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
246 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
247 self._use_echo_app = True
247 self._use_echo_app = True
248 log.warning("Using EchoApp for VCS operations.")
248 log.warning("Using EchoApp for VCS operations.")
249 self.remote_wsgi = remote_wsgi_stub
249 self.remote_wsgi = remote_wsgi_stub
250
250
251 self._configure_settings(global_config, settings)
251 self._configure_settings(global_config, settings)
252
252
253 self._configure()
253 self._configure()
254
254
255 def _configure_settings(self, global_config, app_settings):
255 def _configure_settings(self, global_config, app_settings):
256 """
256 """
257 Configure the settings module.
257 Configure the settings module.
258 """
258 """
259 settings_merged = global_config.copy()
259 settings_merged = global_config.copy()
260 settings_merged.update(app_settings)
260 settings_merged.update(app_settings)
261
261
262 git_path = app_settings.get('git_path', None)
262 git_path = app_settings.get('git_path', None)
263 if git_path:
263 if git_path:
264 settings.GIT_EXECUTABLE = git_path
264 settings.GIT_EXECUTABLE = git_path
265 binary_dir = app_settings.get('core.binary_dir', None)
265 binary_dir = app_settings.get('core.binary_dir', None)
266 if binary_dir:
266 if binary_dir:
267 settings.BINARY_DIR = binary_dir
267 settings.BINARY_DIR = binary_dir
268
268
269 # Store the settings to make them available to other modules.
269 # Store the settings to make them available to other modules.
270 vcsserver.PYRAMID_SETTINGS = settings_merged
270 vcsserver.PYRAMID_SETTINGS = settings_merged
271 vcsserver.CONFIG = settings_merged
271 vcsserver.CONFIG = settings_merged
272
272
273 def _configure(self):
273 def _configure(self):
274 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
274 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
275
275
276 self.config.add_route('service', '/_service')
276 self.config.add_route('service', '/_service')
277 self.config.add_route('status', '/status')
277 self.config.add_route('status', '/status')
278 self.config.add_route('hg_proxy', '/proxy/hg')
278 self.config.add_route('hg_proxy', '/proxy/hg')
279 self.config.add_route('git_proxy', '/proxy/git')
279 self.config.add_route('git_proxy', '/proxy/git')
280
280
281 # rpc methods
281 # rpc methods
282 self.config.add_route('vcs', '/{backend}')
282 self.config.add_route('vcs', '/{backend}')
283
283
284 # streaming rpc remote methods
284 # streaming rpc remote methods
285 self.config.add_route('vcs_stream', '/{backend}/stream')
285 self.config.add_route('vcs_stream', '/{backend}/stream')
286
286
287 # vcs operations clone/push as streaming
287 # vcs operations clone/push as streaming
288 self.config.add_route('stream_git', '/stream/git/*repo_name')
288 self.config.add_route('stream_git', '/stream/git/*repo_name')
289 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
289 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
290
290
291 self.config.add_view(self.status_view, route_name='status', renderer='json')
291 self.config.add_view(self.status_view, route_name='status', renderer='json')
292 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
292 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
293
293
294 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
294 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
295 self.config.add_view(self.git_proxy(), route_name='git_proxy')
295 self.config.add_view(self.git_proxy(), route_name='git_proxy')
296 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
296 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
297 vcs_view=self._remotes)
297 vcs_view=self._remotes)
298 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
298 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
299 vcs_view=self._remotes)
299 vcs_view=self._remotes)
300
300
301 self.config.add_view(self.hg_stream(), route_name='stream_hg')
301 self.config.add_view(self.hg_stream(), route_name='stream_hg')
302 self.config.add_view(self.git_stream(), route_name='stream_git')
302 self.config.add_view(self.git_stream(), route_name='stream_git')
303
303
304 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
304 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
305
305
306 self.config.add_notfound_view(not_found, renderer='json')
306 self.config.add_notfound_view(not_found, renderer='json')
307
307
308 self.config.add_view(self.handle_vcs_exception, context=Exception)
308 self.config.add_view(self.handle_vcs_exception, context=Exception)
309
309
310 self.config.add_tween(
310 self.config.add_tween(
311 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
311 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
312 )
312 )
313 self.config.add_request_method(
313 self.config.add_request_method(
314 'vcsserver.lib.request_counter.get_request_counter',
314 'vcsserver.lib.request_counter.get_request_counter',
315 'request_count')
315 'request_count')
316
316
317 def wsgi_app(self):
317 def wsgi_app(self):
318 return self.config.make_wsgi_app()
318 return self.config.make_wsgi_app()
319
319
320 def _vcs_view_params(self, request):
320 def _vcs_view_params(self, request):
321 remote = self._remotes[request.matchdict['backend']]
321 remote = self._remotes[request.matchdict['backend']]
322 payload = msgpack.unpackb(request.body, use_list=True)
322 payload = msgpack.unpackb(request.body, use_list=True)
323
323
324 method = payload.get('method')
324 method = payload.get('method')
325 params = payload['params']
325 params = payload['params']
326 wire = params.get('wire')
326 wire = params.get('wire')
327 args = params.get('args')
327 args = params.get('args')
328 kwargs = params.get('kwargs')
328 kwargs = params.get('kwargs')
329 context_uid = None
329 context_uid = None
330
330
331 request.registry.vcs_call_context = {
331 request.registry.vcs_call_context = {
332 'method': method,
332 'method': method,
333 'repo_name': payload.get('_repo_name')
333 'repo_name': payload.get('_repo_name')
334 }
334 }
335
335
336 if wire:
336 if wire:
337 try:
337 try:
338 wire['context'] = context_uid = uuid.UUID(wire['context'])
338 wire['context'] = context_uid = uuid.UUID(wire['context'])
339 except KeyError:
339 except KeyError:
340 pass
340 pass
341 args.insert(0, wire)
341 args.insert(0, wire)
342 repo_state_uid = wire.get('repo_state_uid') if wire else None
342 repo_state_uid = wire.get('repo_state_uid') if wire else None
343
343
344 # NOTE(marcink): trading complexity for slight performance
344 # NOTE(marcink): trading complexity for slight performance
345 if log.isEnabledFor(logging.DEBUG):
345 if log.isEnabledFor(logging.DEBUG):
346 # also we SKIP printing out any of those methods args since they maybe excessive
346 # also we SKIP printing out any of those methods args since they maybe excessive
347 just_args_methods = {
347 just_args_methods = {
348 'commitctx': ('content', 'removed', 'updated')
348 'commitctx': ('content', 'removed', 'updated')
349 }
349 }
350 if method in just_args_methods:
350 if method in just_args_methods:
351 skip_args = just_args_methods[method]
351 skip_args = just_args_methods[method]
352 call_args = ''
352 call_args = ''
353 call_kwargs = {}
353 call_kwargs = {}
354 for k in kwargs:
354 for k in kwargs:
355 if k in skip_args:
355 if k in skip_args:
356 # replace our skip key with dummy
356 # replace our skip key with dummy
357 call_kwargs[k] = f'RemovedParam({k})'
357 call_kwargs[k] = f'RemovedParam({k})'
358 else:
358 else:
359 call_kwargs[k] = kwargs[k]
359 call_kwargs[k] = kwargs[k]
360 else:
360 else:
361 call_args = args[1:]
361 call_args = args[1:]
362 call_kwargs = kwargs
362 call_kwargs = kwargs
363
363
364 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
364 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
365 method, call_args, call_kwargs, context_uid, repo_state_uid)
365 method, call_args, call_kwargs, context_uid, repo_state_uid)
366
366
367 statsd = request.registry.statsd
367 statsd = request.registry.statsd
368 if statsd:
368 if statsd:
369 statsd.incr(
369 statsd.incr(
370 'vcsserver_method_total', tags=[
370 'vcsserver_method_total', tags=[
371 "method:{}".format(method),
371 "method:{}".format(method),
372 ])
372 ])
373 return payload, remote, method, args, kwargs
373 return payload, remote, method, args, kwargs
374
374
375 def vcs_view(self, request):
375 def vcs_view(self, request):
376
376
377 payload, remote, method, args, kwargs = self._vcs_view_params(request)
377 payload, remote, method, args, kwargs = self._vcs_view_params(request)
378 payload_id = payload.get('id')
378 payload_id = payload.get('id')
379
379
380 try:
380 try:
381 resp = getattr(remote, method)(*args, **kwargs)
381 resp = getattr(remote, method)(*args, **kwargs)
382 except Exception as e:
382 except Exception as e:
383 exc_info = list(sys.exc_info())
383 exc_info = list(sys.exc_info())
384 exc_type, exc_value, exc_traceback = exc_info
384 exc_type, exc_value, exc_traceback = exc_info
385
385
386 org_exc = getattr(e, '_org_exc', None)
386 org_exc = getattr(e, '_org_exc', None)
387 org_exc_name = None
387 org_exc_name = None
388 org_exc_tb = ''
388 org_exc_tb = ''
389 if org_exc:
389 if org_exc:
390 org_exc_name = org_exc.__class__.__name__
390 org_exc_name = org_exc.__class__.__name__
391 org_exc_tb = getattr(e, '_org_exc_tb', '')
391 org_exc_tb = getattr(e, '_org_exc_tb', '')
392 # replace our "faked" exception with our org
392 # replace our "faked" exception with our org
393 exc_info[0] = org_exc.__class__
393 exc_info[0] = org_exc.__class__
394 exc_info[1] = org_exc
394 exc_info[1] = org_exc
395
395
396 should_store_exc = True
396 should_store_exc = True
397 if org_exc:
397 if org_exc:
398 def get_exc_fqn(_exc_obj):
398 def get_exc_fqn(_exc_obj):
399 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
399 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
400 return module_name + '.' + org_exc_name
400 return module_name + '.' + org_exc_name
401
401
402 exc_fqn = get_exc_fqn(org_exc)
402 exc_fqn = get_exc_fqn(org_exc)
403
403
404 if exc_fqn in ['mercurial.error.RepoLookupError',
404 if exc_fqn in ['mercurial.error.RepoLookupError',
405 'vcsserver.exceptions.RefNotFoundException']:
405 'vcsserver.exceptions.RefNotFoundException']:
406 should_store_exc = False
406 should_store_exc = False
407
407
408 if should_store_exc:
408 if should_store_exc:
409 store_exception(id(exc_info), exc_info, request_path=request.path)
409 store_exception(id(exc_info), exc_info, request_path=request.path)
410
410
411 tb_info = ''.join(
411 tb_info = ''.join(
412 traceback.format_exception(exc_type, exc_value, exc_traceback))
412 traceback.format_exception(exc_type, exc_value, exc_traceback))
413
413
414 type_ = e.__class__.__name__
414 type_ = e.__class__.__name__
415 if type_ not in self.ALLOWED_EXCEPTIONS:
415 if type_ not in self.ALLOWED_EXCEPTIONS:
416 type_ = None
416 type_ = None
417
417
418 resp = {
418 resp = {
419 'id': payload_id,
419 'id': payload_id,
420 'error': {
420 'error': {
421 'message': str(e),
421 'message': str(e),
422 'traceback': tb_info,
422 'traceback': tb_info,
423 'org_exc': org_exc_name,
423 'org_exc': org_exc_name,
424 'org_exc_tb': org_exc_tb,
424 'org_exc_tb': org_exc_tb,
425 'type': type_
425 'type': type_
426 }
426 }
427 }
427 }
428
428
429 try:
429 try:
430 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
430 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
431 except AttributeError:
431 except AttributeError:
432 pass
432 pass
433 else:
433 else:
434 resp = {
434 resp = {
435 'id': payload_id,
435 'id': payload_id,
436 'result': resp
436 'result': resp
437 }
437 }
438 log.debug('Serving data for method %s', method)
438 log.debug('Serving data for method %s', method)
439 return resp
439 return resp
440
440
441 def vcs_stream_view(self, request):
441 def vcs_stream_view(self, request):
442 payload, remote, method, args, kwargs = self._vcs_view_params(request)
442 payload, remote, method, args, kwargs = self._vcs_view_params(request)
443 # this method has a stream: marker we remove it here
443 # this method has a stream: marker we remove it here
444 method = method.split('stream:')[-1]
444 method = method.split('stream:')[-1]
445 chunk_size = safe_int(payload.get('chunk_size')) or 4096
445 chunk_size = safe_int(payload.get('chunk_size')) or 4096
446
446
447 try:
447 try:
448 resp = getattr(remote, method)(*args, **kwargs)
448 resp = getattr(remote, method)(*args, **kwargs)
449 except Exception as e:
449 except Exception as e:
450 raise
450 raise
451
451
452 def get_chunked_data(method_resp):
452 def get_chunked_data(method_resp):
453 stream = io.BytesIO(method_resp)
453 stream = io.BytesIO(method_resp)
454 while 1:
454 while 1:
455 chunk = stream.read(chunk_size)
455 chunk = stream.read(chunk_size)
456 if not chunk:
456 if not chunk:
457 break
457 break
458 yield chunk
458 yield chunk
459
459
460 response = Response(app_iter=get_chunked_data(resp))
460 response = Response(app_iter=get_chunked_data(resp))
461 response.content_type = 'application/octet-stream'
461 response.content_type = 'application/octet-stream'
462
462
463 return response
463 return response
464
464
465 def status_view(self, request):
465 def status_view(self, request):
466 import vcsserver
466 import vcsserver
467 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
467 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
468 'pid': os.getpid()}
468 'pid': os.getpid()}
469
469
470 def service_view(self, request):
470 def service_view(self, request):
471 import vcsserver
471 import vcsserver
472
472
473 payload = msgpack.unpackb(request.body, use_list=True)
473 payload = msgpack.unpackb(request.body, use_list=True)
474 server_config, app_config = {}, {}
474 server_config, app_config = {}, {}
475
475
476 try:
476 try:
477 path = self.global_config['__file__']
477 path = self.global_config['__file__']
478 config = configparser.RawConfigParser()
478 config = configparser.RawConfigParser()
479
479
480 config.read(path)
480 config.read(path)
481
481
482 if config.has_section('server:main'):
482 if config.has_section('server:main'):
483 server_config = dict(config.items('server:main'))
483 server_config = dict(config.items('server:main'))
484 if config.has_section('app:main'):
484 if config.has_section('app:main'):
485 app_config = dict(config.items('app:main'))
485 app_config = dict(config.items('app:main'))
486
486
487 except Exception:
487 except Exception:
488 log.exception('Failed to read .ini file for display')
488 log.exception('Failed to read .ini file for display')
489
489
490 environ = list(os.environ.items())
490 environ = list(os.environ.items())
491
491
492 resp = {
492 resp = {
493 'id': payload.get('id'),
493 'id': payload.get('id'),
494 'result': dict(
494 'result': dict(
495 version=vcsserver.__version__,
495 version=vcsserver.__version__,
496 config=server_config,
496 config=server_config,
497 app_config=app_config,
497 app_config=app_config,
498 environ=environ,
498 environ=environ,
499 payload=payload,
499 payload=payload,
500 )
500 )
501 }
501 }
502 return resp
502 return resp
503
503
504 def _msgpack_renderer_factory(self, info):
504 def _msgpack_renderer_factory(self, info):
505
505
506 def _render(value, system):
506 def _render(value, system):
507 bin_type = False
507 bin_type = False
508 res = value.get('result')
508 res = value.get('result')
509 if res and isinstance(res, BinaryEnvelope):
509 if res and isinstance(res, BinaryEnvelope):
510 log.debug('Result is wrapped in BinaryEnvelope type')
510 value['result'] = res.value
511 value['result'] = res.value
511 bin_type = res.bin_type
512 bin_type = res.bin_type
512
513
513 request = system.get('request')
514 request = system.get('request')
514 if request is not None:
515 if request is not None:
515 response = request.response
516 response = request.response
516 ct = response.content_type
517 ct = response.content_type
517 if ct == response.default_content_type:
518 if ct == response.default_content_type:
518 response.content_type = 'application/x-msgpack'
519 response.content_type = 'application/x-msgpack'
520 if bin_type:
521 response.content_type = 'application/x-msgpack-bin'
519
522
520 return msgpack.packb(value, use_bin_type=bin_type)
523 return msgpack.packb(value, use_bin_type=bin_type)
521 return _render
524 return _render
522
525
523 def set_env_from_config(self, environ, config):
526 def set_env_from_config(self, environ, config):
524 dict_conf = {}
527 dict_conf = {}
525 try:
528 try:
526 for elem in config:
529 for elem in config:
527 if elem[0] == 'rhodecode':
530 if elem[0] == 'rhodecode':
528 dict_conf = json.loads(elem[2])
531 dict_conf = json.loads(elem[2])
529 break
532 break
530 except Exception:
533 except Exception:
531 log.exception('Failed to fetch SCM CONFIG')
534 log.exception('Failed to fetch SCM CONFIG')
532 return
535 return
533
536
534 username = dict_conf.get('username')
537 username = dict_conf.get('username')
535 if username:
538 if username:
536 environ['REMOTE_USER'] = username
539 environ['REMOTE_USER'] = username
537 # mercurial specific, some extension api rely on this
540 # mercurial specific, some extension api rely on this
538 environ['HGUSER'] = username
541 environ['HGUSER'] = username
539
542
540 ip = dict_conf.get('ip')
543 ip = dict_conf.get('ip')
541 if ip:
544 if ip:
542 environ['REMOTE_HOST'] = ip
545 environ['REMOTE_HOST'] = ip
543
546
544 if _is_request_chunked(environ):
547 if _is_request_chunked(environ):
545 # set the compatibility flag for webob
548 # set the compatibility flag for webob
546 environ['wsgi.input_terminated'] = True
549 environ['wsgi.input_terminated'] = True
547
550
548 def hg_proxy(self):
551 def hg_proxy(self):
549 @wsgiapp
552 @wsgiapp
550 def _hg_proxy(environ, start_response):
553 def _hg_proxy(environ, start_response):
551 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
554 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
552 return app(environ, start_response)
555 return app(environ, start_response)
553 return _hg_proxy
556 return _hg_proxy
554
557
555 def git_proxy(self):
558 def git_proxy(self):
556 @wsgiapp
559 @wsgiapp
557 def _git_proxy(environ, start_response):
560 def _git_proxy(environ, start_response):
558 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
561 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
559 return app(environ, start_response)
562 return app(environ, start_response)
560 return _git_proxy
563 return _git_proxy
561
564
562 def hg_stream(self):
565 def hg_stream(self):
563 if self._use_echo_app:
566 if self._use_echo_app:
564 @wsgiapp
567 @wsgiapp
565 def _hg_stream(environ, start_response):
568 def _hg_stream(environ, start_response):
566 app = EchoApp('fake_path', 'fake_name', None)
569 app = EchoApp('fake_path', 'fake_name', None)
567 return app(environ, start_response)
570 return app(environ, start_response)
568 return _hg_stream
571 return _hg_stream
569 else:
572 else:
570 @wsgiapp
573 @wsgiapp
571 def _hg_stream(environ, start_response):
574 def _hg_stream(environ, start_response):
572 log.debug('http-app: handling hg stream')
575 log.debug('http-app: handling hg stream')
573
576
574 packed_cc = base64.b64decode(environ['HTTP_X_RC_VCS_STREAM_CALL_CONTEXT'])
577 packed_cc = base64.b64decode(environ['HTTP_X_RC_VCS_STREAM_CALL_CONTEXT'])
575 call_context = msgpack.unpackb(packed_cc)
578 call_context = msgpack.unpackb(packed_cc)
576
579
577 repo_path = call_context['repo_path']
580 repo_path = call_context['repo_path']
578 repo_name = call_context['repo_name']
581 repo_name = call_context['repo_name']
579 config = call_context['repo_config']
582 config = call_context['repo_config']
580
583
581 app = scm_app.create_hg_wsgi_app(
584 app = scm_app.create_hg_wsgi_app(
582 repo_path, repo_name, config)
585 repo_path, repo_name, config)
583
586
584 # Consistent path information for hgweb
587 # Consistent path information for hgweb
585 environ['PATH_INFO'] = call_context['path_info']
588 environ['PATH_INFO'] = call_context['path_info']
586 environ['REPO_NAME'] = repo_name
589 environ['REPO_NAME'] = repo_name
587 self.set_env_from_config(environ, config)
590 self.set_env_from_config(environ, config)
588
591
589 log.debug('http-app: starting app handler '
592 log.debug('http-app: starting app handler '
590 'with %s and process request', app)
593 'with %s and process request', app)
591 return app(environ, ResponseFilter(start_response))
594 return app(environ, ResponseFilter(start_response))
592 return _hg_stream
595 return _hg_stream
593
596
594 def git_stream(self):
597 def git_stream(self):
595 if self._use_echo_app:
598 if self._use_echo_app:
596 @wsgiapp
599 @wsgiapp
597 def _git_stream(environ, start_response):
600 def _git_stream(environ, start_response):
598 app = EchoApp('fake_path', 'fake_name', None)
601 app = EchoApp('fake_path', 'fake_name', None)
599 return app(environ, start_response)
602 return app(environ, start_response)
600 return _git_stream
603 return _git_stream
601 else:
604 else:
602 @wsgiapp
605 @wsgiapp
603 def _git_stream(environ, start_response):
606 def _git_stream(environ, start_response):
604 log.debug('http-app: handling git stream')
607 log.debug('http-app: handling git stream')
605
608
606 packed_cc = base64.b64decode(environ['HTTP_X_RC_VCS_STREAM_CALL_CONTEXT'])
609 packed_cc = base64.b64decode(environ['HTTP_X_RC_VCS_STREAM_CALL_CONTEXT'])
607 call_context = msgpack.unpackb(packed_cc)
610 call_context = msgpack.unpackb(packed_cc)
608
611
609 repo_path = call_context['repo_path']
612 repo_path = call_context['repo_path']
610 repo_name = call_context['repo_name']
613 repo_name = call_context['repo_name']
611 config = call_context['repo_config']
614 config = call_context['repo_config']
612
615
613 environ['PATH_INFO'] = call_context['path_info']
616 environ['PATH_INFO'] = call_context['path_info']
614 self.set_env_from_config(environ, config)
617 self.set_env_from_config(environ, config)
615
618
616 content_type = environ.get('CONTENT_TYPE', '')
619 content_type = environ.get('CONTENT_TYPE', '')
617
620
618 path = environ['PATH_INFO']
621 path = environ['PATH_INFO']
619 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
622 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
620 log.debug(
623 log.debug(
621 'LFS: Detecting if request `%s` is LFS server path based '
624 'LFS: Detecting if request `%s` is LFS server path based '
622 'on content type:`%s`, is_lfs:%s',
625 'on content type:`%s`, is_lfs:%s',
623 path, content_type, is_lfs_request)
626 path, content_type, is_lfs_request)
624
627
625 if not is_lfs_request:
628 if not is_lfs_request:
626 # fallback detection by path
629 # fallback detection by path
627 if GIT_LFS_PROTO_PAT.match(path):
630 if GIT_LFS_PROTO_PAT.match(path):
628 is_lfs_request = True
631 is_lfs_request = True
629 log.debug(
632 log.debug(
630 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
633 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
631 path, is_lfs_request)
634 path, is_lfs_request)
632
635
633 if is_lfs_request:
636 if is_lfs_request:
634 app = scm_app.create_git_lfs_wsgi_app(
637 app = scm_app.create_git_lfs_wsgi_app(
635 repo_path, repo_name, config)
638 repo_path, repo_name, config)
636 else:
639 else:
637 app = scm_app.create_git_wsgi_app(
640 app = scm_app.create_git_wsgi_app(
638 repo_path, repo_name, config)
641 repo_path, repo_name, config)
639
642
640 log.debug('http-app: starting app handler '
643 log.debug('http-app: starting app handler '
641 'with %s and process request', app)
644 'with %s and process request', app)
642
645
643 return app(environ, start_response)
646 return app(environ, start_response)
644
647
645 return _git_stream
648 return _git_stream
646
649
647 def handle_vcs_exception(self, exception, request):
650 def handle_vcs_exception(self, exception, request):
648 _vcs_kind = getattr(exception, '_vcs_kind', '')
651 _vcs_kind = getattr(exception, '_vcs_kind', '')
649 if _vcs_kind == 'repo_locked':
652 if _vcs_kind == 'repo_locked':
650 # Get custom repo-locked status code if present.
653 # Get custom repo-locked status code if present.
651 status_code = request.headers.get('X-RC-Locked-Status-Code')
654 status_code = request.headers.get('X-RC-Locked-Status-Code')
652 return HTTPRepoLocked(
655 return HTTPRepoLocked(
653 title=exception.message, status_code=status_code)
656 title=exception.message, status_code=status_code)
654
657
655 elif _vcs_kind == 'repo_branch_protected':
658 elif _vcs_kind == 'repo_branch_protected':
656 # Get custom repo-branch-protected status code if present.
659 # Get custom repo-branch-protected status code if present.
657 return HTTPRepoBranchProtected(title=exception.message)
660 return HTTPRepoBranchProtected(title=exception.message)
658
661
659 exc_info = request.exc_info
662 exc_info = request.exc_info
660 store_exception(id(exc_info), exc_info)
663 store_exception(id(exc_info), exc_info)
661
664
662 traceback_info = 'unavailable'
665 traceback_info = 'unavailable'
663 if request.exc_info:
666 if request.exc_info:
664 exc_type, exc_value, exc_tb = request.exc_info
667 exc_type, exc_value, exc_tb = request.exc_info
665 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
668 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
666
669
667 log.error(
670 log.error(
668 'error occurred handling this request for path: %s, \n tb: %s',
671 'error occurred handling this request for path: %s, \n tb: %s',
669 request.path, traceback_info)
672 request.path, traceback_info)
670
673
671 statsd = request.registry.statsd
674 statsd = request.registry.statsd
672 if statsd:
675 if statsd:
673 exc_type = "{}.{}".format(exception.__class__.__module__, exception.__class__.__name__)
676 exc_type = "{}.{}".format(exception.__class__.__module__, exception.__class__.__name__)
674 statsd.incr('vcsserver_exception_total',
677 statsd.incr('vcsserver_exception_total',
675 tags=["type:{}".format(exc_type)])
678 tags=["type:{}".format(exc_type)])
676 raise exception
679 raise exception
677
680
678
681
679 class ResponseFilter(object):
682 class ResponseFilter(object):
680
683
681 def __init__(self, start_response):
684 def __init__(self, start_response):
682 self._start_response = start_response
685 self._start_response = start_response
683
686
684 def __call__(self, status, response_headers, exc_info=None):
687 def __call__(self, status, response_headers, exc_info=None):
685 headers = tuple(
688 headers = tuple(
686 (h, v) for h, v in response_headers
689 (h, v) for h, v in response_headers
687 if not wsgiref.util.is_hop_by_hop(h))
690 if not wsgiref.util.is_hop_by_hop(h))
688 return self._start_response(status, headers, exc_info)
691 return self._start_response(status, headers, exc_info)
689
692
690
693
691 def sanitize_settings_and_apply_defaults(global_config, settings):
694 def sanitize_settings_and_apply_defaults(global_config, settings):
692 global_settings_maker = SettingsMaker(global_config)
695 global_settings_maker = SettingsMaker(global_config)
693 settings_maker = SettingsMaker(settings)
696 settings_maker = SettingsMaker(settings)
694
697
695 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
698 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
696
699
697 logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
700 logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
698 settings_maker.enable_logging(logging_conf)
701 settings_maker.enable_logging(logging_conf)
699
702
700 # Default includes, possible to change as a user
703 # Default includes, possible to change as a user
701 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
704 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
702 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
705 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
703
706
704 settings_maker.make_setting('__file__', global_config.get('__file__'))
707 settings_maker.make_setting('__file__', global_config.get('__file__'))
705
708
706 settings_maker.make_setting('pyramid.default_locale_name', 'en')
709 settings_maker.make_setting('pyramid.default_locale_name', 'en')
707 settings_maker.make_setting('locale', 'en_US.UTF-8')
710 settings_maker.make_setting('locale', 'en_US.UTF-8')
708
711
709 settings_maker.make_setting('core.binary_dir', '')
712 settings_maker.make_setting('core.binary_dir', '')
710
713
711 temp_store = tempfile.gettempdir()
714 temp_store = tempfile.gettempdir()
712 default_cache_dir = os.path.join(temp_store, 'rc_cache')
715 default_cache_dir = os.path.join(temp_store, 'rc_cache')
713 # save default, cache dir, and use it for all backends later.
716 # save default, cache dir, and use it for all backends later.
714 default_cache_dir = settings_maker.make_setting(
717 default_cache_dir = settings_maker.make_setting(
715 'cache_dir',
718 'cache_dir',
716 default=default_cache_dir, default_when_empty=True,
719 default=default_cache_dir, default_when_empty=True,
717 parser='dir:ensured')
720 parser='dir:ensured')
718
721
719 # exception store cache
722 # exception store cache
720 settings_maker.make_setting(
723 settings_maker.make_setting(
721 'exception_tracker.store_path',
724 'exception_tracker.store_path',
722 default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
725 default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
723 parser='dir:ensured'
726 parser='dir:ensured'
724 )
727 )
725
728
726 # repo_object cache defaults
729 # repo_object cache defaults
727 settings_maker.make_setting(
730 settings_maker.make_setting(
728 'rc_cache.repo_object.backend',
731 'rc_cache.repo_object.backend',
729 default='dogpile.cache.rc.file_namespace',
732 default='dogpile.cache.rc.file_namespace',
730 parser='string')
733 parser='string')
731 settings_maker.make_setting(
734 settings_maker.make_setting(
732 'rc_cache.repo_object.expiration_time',
735 'rc_cache.repo_object.expiration_time',
733 default=30 * 24 * 60 * 60, # 30days
736 default=30 * 24 * 60 * 60, # 30days
734 parser='int')
737 parser='int')
735 settings_maker.make_setting(
738 settings_maker.make_setting(
736 'rc_cache.repo_object.arguments.filename',
739 'rc_cache.repo_object.arguments.filename',
737 default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
740 default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
738 parser='string')
741 parser='string')
739
742
740 # statsd
743 # statsd
741 settings_maker.make_setting('statsd.enabled', False, parser='bool')
744 settings_maker.make_setting('statsd.enabled', False, parser='bool')
742 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
745 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
743 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
746 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
744 settings_maker.make_setting('statsd.statsd_prefix', '')
747 settings_maker.make_setting('statsd.statsd_prefix', '')
745 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
748 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
746
749
747 settings_maker.env_expand()
750 settings_maker.env_expand()
748
751
749
752
750 def main(global_config, **settings):
753 def main(global_config, **settings):
751 start_time = time.time()
754 start_time = time.time()
752 log.info('Pyramid app config starting')
755 log.info('Pyramid app config starting')
753
756
754 if MercurialFactory:
757 if MercurialFactory:
755 hgpatches.patch_largefiles_capabilities()
758 hgpatches.patch_largefiles_capabilities()
756 hgpatches.patch_subrepo_type_mapping()
759 hgpatches.patch_subrepo_type_mapping()
757
760
758 # Fill in and sanitize the defaults & do ENV expansion
761 # Fill in and sanitize the defaults & do ENV expansion
759 sanitize_settings_and_apply_defaults(global_config, settings)
762 sanitize_settings_and_apply_defaults(global_config, settings)
760
763
761 # init and bootstrap StatsdClient
764 # init and bootstrap StatsdClient
762 StatsdClient.setup(settings)
765 StatsdClient.setup(settings)
763
766
764 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
767 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
765 total_time = time.time() - start_time
768 total_time = time.time() - start_time
766 log.info('Pyramid app `%s` created and configured in %.2fs',
769 log.info('Pyramid app `%s` created and configured in %.2fs',
767 getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
770 getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
768 return pyramid_app
771 return pyramid_app
769
772
770
773
General Comments 0
You need to be logged in to leave comments. Login now