##// END OF EJS Templates
fix(svn-hooks): fixed problem with svn subprocess execution fixes RCCE-62
super-admin -
r1220:99aaec7b v5.0.0 stable
parent child Browse files
Show More
@@ -1,220 +1,230 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18
18 import re
19 import re
19 import os
20 import os
20 import sys
21 import sys
21 import datetime
22 import datetime
22 import logging
23 import logging
23 import pkg_resources
24 import pkg_resources
24
25
25 import vcsserver
26 import vcsserver
27 import vcsserver.settings
26 from vcsserver.str_utils import safe_bytes
28 from vcsserver.str_utils import safe_bytes
27
29
28 log = logging.getLogger(__name__)
30 log = logging.getLogger(__name__)
29
31
30 HOOKS_DIR_MODE = 0o755
32 HOOKS_DIR_MODE = 0o755
31 HOOKS_FILE_MODE = 0o755
33 HOOKS_FILE_MODE = 0o755
32
34
33
35
34 def set_permissions_if_needed(path_to_check, perms: oct):
36 def set_permissions_if_needed(path_to_check, perms: oct):
35 # Get current permissions
37 # Get current permissions
36 current_permissions = os.stat(path_to_check).st_mode & 0o777 # Extract permission bits
38 current_permissions = os.stat(path_to_check).st_mode & 0o777 # Extract permission bits
37
39
38 # Check if current permissions are lower than required
40 # Check if current permissions are lower than required
39 if current_permissions < int(perms):
41 if current_permissions < int(perms):
40 # Change the permissions if they are lower than required
42 # Change the permissions if they are lower than required
41 os.chmod(path_to_check, perms)
43 os.chmod(path_to_check, perms)
42
44
43
45
44 def get_git_hooks_path(repo_path, bare):
46 def get_git_hooks_path(repo_path, bare):
45 hooks_path = os.path.join(repo_path, 'hooks')
47 hooks_path = os.path.join(repo_path, 'hooks')
46 if not bare:
48 if not bare:
47 hooks_path = os.path.join(repo_path, '.git', 'hooks')
49 hooks_path = os.path.join(repo_path, '.git', 'hooks')
48
50
49 return hooks_path
51 return hooks_path
50
52
51
53
52 def install_git_hooks(repo_path, bare, executable=None, force_create=False):
54 def install_git_hooks(repo_path, bare, executable=None, force_create=False):
53 """
55 """
54 Creates a RhodeCode hook inside a git repository
56 Creates a RhodeCode hook inside a git repository
55
57
56 :param repo_path: path to repository
58 :param repo_path: path to repository
57 :param bare: defines if repository is considered a bare git repo
59 :param bare: defines if repository is considered a bare git repo
58 :param executable: binary executable to put in the hooks
60 :param executable: binary executable to put in the hooks
59 :param force_create: Creates even if the same name hook exists
61 :param force_create: Creates even if the same name hook exists
60 """
62 """
61 executable = executable or sys.executable
63 executable = executable or sys.executable
62 hooks_path = get_git_hooks_path(repo_path, bare)
64 hooks_path = get_git_hooks_path(repo_path, bare)
63
65
64 # we always call it to ensure dir exists and it has a proper mode
66 # we always call it to ensure dir exists and it has a proper mode
65 if not os.path.exists(hooks_path):
67 if not os.path.exists(hooks_path):
66 # If it doesn't exist, create a new directory with the specified mode
68 # If it doesn't exist, create a new directory with the specified mode
67 os.makedirs(hooks_path, mode=HOOKS_DIR_MODE, exist_ok=True)
69 os.makedirs(hooks_path, mode=HOOKS_DIR_MODE, exist_ok=True)
68 # If it exists, change the directory's mode to the specified mode
70 # If it exists, change the directory's mode to the specified mode
69 set_permissions_if_needed(hooks_path, perms=HOOKS_DIR_MODE)
71 set_permissions_if_needed(hooks_path, perms=HOOKS_DIR_MODE)
70
72
71 tmpl_post = pkg_resources.resource_string(
73 tmpl_post = pkg_resources.resource_string(
72 'vcsserver', '/'.join(
74 'vcsserver', '/'.join(
73 ('hook_utils', 'hook_templates', 'git_post_receive.py.tmpl')))
75 ('hook_utils', 'hook_templates', 'git_post_receive.py.tmpl')))
74 tmpl_pre = pkg_resources.resource_string(
76 tmpl_pre = pkg_resources.resource_string(
75 'vcsserver', '/'.join(
77 'vcsserver', '/'.join(
76 ('hook_utils', 'hook_templates', 'git_pre_receive.py.tmpl')))
78 ('hook_utils', 'hook_templates', 'git_pre_receive.py.tmpl')))
77
79
78 path = '' # not used for now
80 path = '' # not used for now
79 timestamp = datetime.datetime.utcnow().isoformat()
81 timestamp = datetime.datetime.utcnow().isoformat()
80
82
81 for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
83 for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
82 log.debug('Installing git hook in repo %s', repo_path)
84 log.debug('Installing git hook in repo %s', repo_path)
83 _hook_file = os.path.join(hooks_path, f'{h_type}-receive')
85 _hook_file = os.path.join(hooks_path, f'{h_type}-receive')
84 _rhodecode_hook = check_rhodecode_hook(_hook_file)
86 _rhodecode_hook = check_rhodecode_hook(_hook_file)
85
87
86 if _rhodecode_hook or force_create:
88 if _rhodecode_hook or force_create:
87 log.debug('writing git %s hook file at %s !', h_type, _hook_file)
89 log.debug('writing git %s hook file at %s !', h_type, _hook_file)
88 try:
90 try:
89 with open(_hook_file, 'wb') as f:
91 with open(_hook_file, 'wb') as f:
90 template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version()))
92 template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version()))
91 template = template.replace(b'_DATE_', safe_bytes(timestamp))
93 template = template.replace(b'_DATE_', safe_bytes(timestamp))
92 template = template.replace(b'_ENV_', safe_bytes(executable))
94 template = template.replace(b'_ENV_', safe_bytes(executable))
93 template = template.replace(b'_PATH_', safe_bytes(path))
95 template = template.replace(b'_PATH_', safe_bytes(path))
94 f.write(template)
96 f.write(template)
95 set_permissions_if_needed(_hook_file, perms=HOOKS_FILE_MODE)
97 set_permissions_if_needed(_hook_file, perms=HOOKS_FILE_MODE)
96 except OSError:
98 except OSError:
97 log.exception('error writing hook file %s', _hook_file)
99 log.exception('error writing hook file %s', _hook_file)
98 else:
100 else:
99 log.debug('skipping writing hook file')
101 log.debug('skipping writing hook file')
100
102
101 return True
103 return True
102
104
103
105
104 def get_svn_hooks_path(repo_path):
106 def get_svn_hooks_path(repo_path):
105 hooks_path = os.path.join(repo_path, 'hooks')
107 hooks_path = os.path.join(repo_path, 'hooks')
106
108
107 return hooks_path
109 return hooks_path
108
110
109
111
110 def install_svn_hooks(repo_path, executable=None, force_create=False):
112 def install_svn_hooks(repo_path, executable=None, force_create=False):
111 """
113 """
112 Creates RhodeCode hooks inside a svn repository
114 Creates RhodeCode hooks inside a svn repository
113
115
114 :param repo_path: path to repository
116 :param repo_path: path to repository
115 :param executable: binary executable to put in the hooks
117 :param executable: binary executable to put in the hooks
116 :param force_create: Create even if same name hook exists
118 :param force_create: Create even if same name hook exists
117 """
119 """
118 executable = executable or sys.executable
120 executable = executable or sys.executable
119 hooks_path = get_svn_hooks_path(repo_path)
121 hooks_path = get_svn_hooks_path(repo_path)
120 if not os.path.isdir(hooks_path):
122 if not os.path.isdir(hooks_path):
121 os.makedirs(hooks_path, mode=0o777, exist_ok=True)
123 os.makedirs(hooks_path, mode=0o777, exist_ok=True)
122
124
123 tmpl_post = pkg_resources.resource_string(
125 tmpl_post = pkg_resources.resource_string(
124 'vcsserver', '/'.join(
126 'vcsserver', '/'.join(
125 ('hook_utils', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
127 ('hook_utils', 'hook_templates', 'svn_post_commit_hook.py.tmpl')))
126 tmpl_pre = pkg_resources.resource_string(
128 tmpl_pre = pkg_resources.resource_string(
127 'vcsserver', '/'.join(
129 'vcsserver', '/'.join(
128 ('hook_utils', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
130 ('hook_utils', 'hook_templates', 'svn_pre_commit_hook.py.tmpl')))
129
131
130 path = '' # not used for now
132 path = '' # not used for now
131 timestamp = datetime.datetime.utcnow().isoformat()
133 timestamp = datetime.datetime.utcnow().isoformat()
132
134
133 for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
135 for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]:
134 log.debug('Installing svn hook in repo %s', repo_path)
136 log.debug('Installing svn hook in repo %s', repo_path)
135 _hook_file = os.path.join(hooks_path, f'{h_type}-commit')
137 _hook_file = os.path.join(hooks_path, f'{h_type}-commit')
136 _rhodecode_hook = check_rhodecode_hook(_hook_file)
138 _rhodecode_hook = check_rhodecode_hook(_hook_file)
137
139
138 if _rhodecode_hook or force_create:
140 if _rhodecode_hook or force_create:
139 log.debug('writing svn %s hook file at %s !', h_type, _hook_file)
141 log.debug('writing svn %s hook file at %s !', h_type, _hook_file)
140
142
143 env_expand = str([
144 ('RC_CORE_BINARY_DIR', vcsserver.settings.BINARY_DIR),
145 ('RC_GIT_EXECUTABLE', vcsserver.settings.GIT_EXECUTABLE),
146 ('RC_SVN_EXECUTABLE', vcsserver.settings.SVN_EXECUTABLE),
147 ('RC_SVNLOOK_EXECUTABLE', vcsserver.settings.SVNLOOK_EXECUTABLE),
148
149 ])
141 try:
150 try:
142 with open(_hook_file, 'wb') as f:
151 with open(_hook_file, 'wb') as f:
143 template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version()))
152 template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version()))
144 template = template.replace(b'_DATE_', safe_bytes(timestamp))
153 template = template.replace(b'_DATE_', safe_bytes(timestamp))
154 template = template.replace(b'_OS_EXPAND_', safe_bytes(env_expand))
145 template = template.replace(b'_ENV_', safe_bytes(executable))
155 template = template.replace(b'_ENV_', safe_bytes(executable))
146 template = template.replace(b'_PATH_', safe_bytes(path))
156 template = template.replace(b'_PATH_', safe_bytes(path))
147
157
148 f.write(template)
158 f.write(template)
149 os.chmod(_hook_file, 0o755)
159 os.chmod(_hook_file, 0o755)
150 except OSError:
160 except OSError:
151 log.exception('error writing hook file %s', _hook_file)
161 log.exception('error writing hook file %s', _hook_file)
152 else:
162 else:
153 log.debug('skipping writing hook file')
163 log.debug('skipping writing hook file')
154
164
155 return True
165 return True
156
166
157
167
158 def get_version_from_hook(hook_path):
168 def get_version_from_hook(hook_path):
159 version = b''
169 version = b''
160 hook_content = read_hook_content(hook_path)
170 hook_content = read_hook_content(hook_path)
161 matches = re.search(rb'RC_HOOK_VER\s*=\s*(.*)', hook_content)
171 matches = re.search(rb'RC_HOOK_VER\s*=\s*(.*)', hook_content)
162 if matches:
172 if matches:
163 try:
173 try:
164 version = matches.groups()[0]
174 version = matches.groups()[0]
165 log.debug('got version %s from hooks.', version)
175 log.debug('got version %s from hooks.', version)
166 except Exception:
176 except Exception:
167 log.exception("Exception while reading the hook version.")
177 log.exception("Exception while reading the hook version.")
168 return version.replace(b"'", b"")
178 return version.replace(b"'", b"")
169
179
170
180
171 def check_rhodecode_hook(hook_path):
181 def check_rhodecode_hook(hook_path):
172 """
182 """
173 Check if the hook was created by RhodeCode
183 Check if the hook was created by RhodeCode
174 """
184 """
175 if not os.path.exists(hook_path):
185 if not os.path.exists(hook_path):
176 return True
186 return True
177
187
178 log.debug('hook exists, checking if it is from RhodeCode')
188 log.debug('hook exists, checking if it is from RhodeCode')
179
189
180 version = get_version_from_hook(hook_path)
190 version = get_version_from_hook(hook_path)
181 if version:
191 if version:
182 return True
192 return True
183
193
184 return False
194 return False
185
195
186
196
187 def read_hook_content(hook_path) -> bytes:
197 def read_hook_content(hook_path) -> bytes:
188 content = b''
198 content = b''
189 if os.path.isfile(hook_path):
199 if os.path.isfile(hook_path):
190 with open(hook_path, 'rb') as f:
200 with open(hook_path, 'rb') as f:
191 content = f.read()
201 content = f.read()
192 return content
202 return content
193
203
194
204
195 def get_git_pre_hook_version(repo_path, bare):
205 def get_git_pre_hook_version(repo_path, bare):
196 hooks_path = get_git_hooks_path(repo_path, bare)
206 hooks_path = get_git_hooks_path(repo_path, bare)
197 _hook_file = os.path.join(hooks_path, 'pre-receive')
207 _hook_file = os.path.join(hooks_path, 'pre-receive')
198 version = get_version_from_hook(_hook_file)
208 version = get_version_from_hook(_hook_file)
199 return version
209 return version
200
210
201
211
202 def get_git_post_hook_version(repo_path, bare):
212 def get_git_post_hook_version(repo_path, bare):
203 hooks_path = get_git_hooks_path(repo_path, bare)
213 hooks_path = get_git_hooks_path(repo_path, bare)
204 _hook_file = os.path.join(hooks_path, 'post-receive')
214 _hook_file = os.path.join(hooks_path, 'post-receive')
205 version = get_version_from_hook(_hook_file)
215 version = get_version_from_hook(_hook_file)
206 return version
216 return version
207
217
208
218
209 def get_svn_pre_hook_version(repo_path):
219 def get_svn_pre_hook_version(repo_path):
210 hooks_path = get_svn_hooks_path(repo_path)
220 hooks_path = get_svn_hooks_path(repo_path)
211 _hook_file = os.path.join(hooks_path, 'pre-commit')
221 _hook_file = os.path.join(hooks_path, 'pre-commit')
212 version = get_version_from_hook(_hook_file)
222 version = get_version_from_hook(_hook_file)
213 return version
223 return version
214
224
215
225
216 def get_svn_post_hook_version(repo_path):
226 def get_svn_post_hook_version(repo_path):
217 hooks_path = get_svn_hooks_path(repo_path)
227 hooks_path = get_svn_hooks_path(repo_path)
218 _hook_file = os.path.join(hooks_path, 'post-commit')
228 _hook_file = os.path.join(hooks_path, 'post-commit')
219 version = get_version_from_hook(_hook_file)
229 version = get_version_from_hook(_hook_file)
220 return version
230 return version
@@ -1,50 +1,54 b''
1 #!_ENV_
1 #!_ENV_
2
2
3 import os
3 import os
4 import sys
4 import sys
5 path_adjust = [_PATH_]
5 path_adjust = [_PATH_]
6
6
7 if path_adjust:
7 if path_adjust:
8 sys.path = path_adjust
8 sys.path = path_adjust
9
9
10 try:
10 try:
11 from vcsserver import hooks
11 from vcsserver import hooks
12 except ImportError:
12 except ImportError:
13 if os.environ.get('RC_DEBUG_SVN_HOOK'):
13 if os.environ.get('RC_DEBUG_SVN_HOOK'):
14 import traceback
14 import traceback
15 print(traceback.format_exc())
15 print(traceback.format_exc())
16 hooks = None
16 hooks = None
17
17
18
18
19 # TIMESTAMP: _DATE_
19 # TIMESTAMP: _DATE_
20 RC_HOOK_VER = '_TMPL_'
20 RC_HOOK_VER = '_TMPL_'
21
21
22
22
23 # special trick to pass in some information from rc to hooks
24 # mod_dav strips ALL env vars and we can't even access things like PATH
25 for env_k, env_v in _OS_EXPAND_:
26 os.environ[env_k] = env_v
27
23 def main():
28 def main():
24 if hooks is None:
29 if hooks is None:
25 # exit with success if we cannot import vcsserver.hooks !!
30 # exit with success if we cannot import vcsserver.hooks !!
26 # this allows simply push to this repo even without rhodecode
31 # this allows simply push to this repo even without rhodecode
27 sys.exit(0)
32 sys.exit(0)
28
33
29 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
34 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
30 sys.exit(0)
35 sys.exit(0)
31 repo_path = os.getcwd()
36 repo_path = os.getcwd()
32 push_data = sys.argv[1:]
37 push_data = sys.argv[1:]
33
38
34 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
39 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
35
40
36 try:
41 try:
37 result = hooks.svn_post_commit(repo_path, push_data, os.environ)
42 result = hooks.svn_post_commit(repo_path, push_data, os.environ)
38 sys.exit(result)
43 sys.exit(result)
39 except Exception as error:
44 except Exception as error:
40 # TODO: johbo: Improve handling of this special case
45 # TODO: johbo: Improve handling of this special case
41 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
46 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
42 raise
47 raise
43 print(f'ERROR: {error}')
48 print(f'ERROR: {error}')
44 sys.exit(1)
49 sys.exit(1)
45 sys.exit(0)
50 sys.exit(0)
46
51
47
52
48
49 if __name__ == '__main__':
53 if __name__ == '__main__':
50 main()
54 main()
@@ -1,52 +1,58 b''
1 #!_ENV_
1 #!_ENV_
2
2
3 import os
3 import os
4 import sys
4 import sys
5 path_adjust = [_PATH_]
5 path_adjust = [_PATH_]
6
6
7 if path_adjust:
7 if path_adjust:
8 sys.path = path_adjust
8 sys.path = path_adjust
9
9
10 try:
10 try:
11 from vcsserver import hooks
11 from vcsserver import hooks
12 except ImportError:
12 except ImportError:
13 if os.environ.get('RC_DEBUG_SVN_HOOK'):
13 if os.environ.get('RC_DEBUG_SVN_HOOK'):
14 import traceback
14 import traceback
15 print(traceback.format_exc())
15 print(traceback.format_exc())
16 hooks = None
16 hooks = None
17
17
18
18
19 # TIMESTAMP: _DATE_
19 # TIMESTAMP: _DATE_
20 RC_HOOK_VER = '_TMPL_'
20 RC_HOOK_VER = '_TMPL_'
21
21
22
22
23 # special trick to pass in some information from rc to hooks
24 # mod_dav strips ALL env vars and we can't even access things like PATH
25 for env_k, env_v in _OS_EXPAND_:
26 os.environ[env_k] = env_v
27
23 def main():
28 def main():
24 if os.environ.get('SSH_READ_ONLY') == '1':
29 if os.environ.get('SSH_READ_ONLY') == '1':
25 sys.stderr.write('Only read-only access is allowed')
30 sys.stderr.write('Only read-only access is allowed')
26 sys.exit(1)
31 sys.exit(1)
27
32
28 if hooks is None:
33 if hooks is None:
29 # exit with success if we cannot import vcsserver.hooks !!
34 # exit with success if we cannot import vcsserver.hooks !!
30 # this allows simply push to this repo even without rhodecode
35 # this allows simply push to this repo even without rhodecode
31 sys.exit(0)
36 sys.exit(0)
37
32 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
38 if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
33 sys.exit(0)
39 sys.exit(0)
34 repo_path = os.getcwd()
40 repo_path = os.getcwd()
35 push_data = sys.argv[1:]
41 push_data = sys.argv[1:]
36
42
37 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
43 os.environ['RC_HOOK_VER'] = RC_HOOK_VER
38
44
39 try:
45 try:
40 result = hooks.svn_pre_commit(repo_path, push_data, os.environ)
46 result = hooks.svn_pre_commit(repo_path, push_data, os.environ)
41 sys.exit(result)
47 sys.exit(result)
42 except Exception as error:
48 except Exception as error:
43 # TODO: johbo: Improve handling of this special case
49 # TODO: johbo: Improve handling of this special case
44 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
50 if not getattr(error, '_vcs_kind', None) == 'repo_locked':
45 raise
51 raise
46 print(f'ERROR: {error}')
52 print(f'ERROR: {error}')
47 sys.exit(1)
53 sys.exit(1)
48 sys.exit(0)
54 sys.exit(0)
49
55
50
56
51 if __name__ == '__main__':
57 if __name__ == '__main__':
52 main()
58 main()
@@ -1,795 +1,818 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import io
18 import io
19 import os
19 import os
20 import sys
20 import sys
21 import logging
21 import logging
22 import collections
22 import collections
23 import base64
23 import base64
24 import msgpack
24 import msgpack
25 import dataclasses
25 import dataclasses
26 import pygit2
26 import pygit2
27
27
28 import http.client
28 import http.client
29 from celery import Celery
29 from celery import Celery
30
30
31 import mercurial.scmutil
31 import mercurial.scmutil
32 import mercurial.node
32 import mercurial.node
33
33
34 import vcsserver.settings
34 from vcsserver.lib.rc_json import json
35 from vcsserver.lib.rc_json import json
35 from vcsserver import exceptions, subprocessio, settings
36 from vcsserver import exceptions, subprocessio, settings
36 from vcsserver.str_utils import ascii_str, safe_str
37 from vcsserver.str_utils import ascii_str, safe_str
37 from vcsserver.remote.git_remote import Repository
38 from vcsserver.remote.git_remote import Repository
38
39
39 celery_app = Celery('__vcsserver__')
40 celery_app = Celery('__vcsserver__')
40 log = logging.getLogger(__name__)
41 log = logging.getLogger(__name__)
41
42
42
43
43 class HooksHttpClient:
44 class HooksHttpClient:
44 proto = 'msgpack.v1'
45 proto = 'msgpack.v1'
45 connection = None
46 connection = None
46
47
47 def __init__(self, hooks_uri):
48 def __init__(self, hooks_uri):
48 self.hooks_uri = hooks_uri
49 self.hooks_uri = hooks_uri
49
50
50 def __repr__(self):
51 def __repr__(self):
51 return f'{self.__class__}(hook_uri={self.hooks_uri}, proto={self.proto})'
52 return f'{self.__class__}(hook_uri={self.hooks_uri}, proto={self.proto})'
52
53
53 def __call__(self, method, extras):
54 def __call__(self, method, extras):
54 connection = http.client.HTTPConnection(self.hooks_uri)
55 connection = http.client.HTTPConnection(self.hooks_uri)
55 # binary msgpack body
56 # binary msgpack body
56 headers, body = self._serialize(method, extras)
57 headers, body = self._serialize(method, extras)
57 log.debug('Doing a new hooks call using HTTPConnection to %s', self.hooks_uri)
58 log.debug('Doing a new hooks call using HTTPConnection to %s', self.hooks_uri)
58
59
59 try:
60 try:
60 try:
61 try:
61 connection.request('POST', '/', body, headers)
62 connection.request('POST', '/', body, headers)
62 except Exception as error:
63 except Exception as error:
63 log.error('Hooks calling Connection failed on %s, org error: %s', connection.__dict__, error)
64 log.error('Hooks calling Connection failed on %s, org error: %s', connection.__dict__, error)
64 raise
65 raise
65
66
66 response = connection.getresponse()
67 response = connection.getresponse()
67 try:
68 try:
68 return msgpack.load(response)
69 return msgpack.load(response)
69 except Exception:
70 except Exception:
70 response_data = response.read()
71 response_data = response.read()
71 log.exception('Failed to decode hook response json data. '
72 log.exception('Failed to decode hook response json data. '
72 'response_code:%s, raw_data:%s',
73 'response_code:%s, raw_data:%s',
73 response.status, response_data)
74 response.status, response_data)
74 raise
75 raise
75 finally:
76 finally:
76 connection.close()
77 connection.close()
77
78
78 @classmethod
79 @classmethod
79 def _serialize(cls, hook_name, extras):
80 def _serialize(cls, hook_name, extras):
80 data = {
81 data = {
81 'method': hook_name,
82 'method': hook_name,
82 'extras': extras
83 'extras': extras
83 }
84 }
84 headers = {
85 headers = {
85 "rc-hooks-protocol": cls.proto,
86 "rc-hooks-protocol": cls.proto,
86 "Connection": "keep-alive"
87 "Connection": "keep-alive"
87 }
88 }
88 return headers, msgpack.packb(data)
89 return headers, msgpack.packb(data)
89
90
90
91
91 class HooksCeleryClient:
92 class HooksCeleryClient:
92 TASK_TIMEOUT = 60 # time in seconds
93 TASK_TIMEOUT = 60 # time in seconds
93
94
94 def __init__(self, queue, backend):
95 def __init__(self, queue, backend):
95 celery_app.config_from_object({
96 celery_app.config_from_object({
96 'broker_url': queue, 'result_backend': backend,
97 'broker_url': queue, 'result_backend': backend,
97 'broker_connection_retry_on_startup': True,
98 'broker_connection_retry_on_startup': True,
98 'task_serializer': 'msgpack',
99 'task_serializer': 'msgpack',
99 'accept_content': ['json', 'msgpack'],
100 'accept_content': ['json', 'msgpack'],
100 'result_serializer': 'msgpack',
101 'result_serializer': 'msgpack',
101 'result_accept_content': ['json', 'msgpack']
102 'result_accept_content': ['json', 'msgpack']
102 })
103 })
103 self.celery_app = celery_app
104 self.celery_app = celery_app
104
105
105 def __call__(self, method, extras):
106 def __call__(self, method, extras):
106 inquired_task = self.celery_app.signature(
107 inquired_task = self.celery_app.signature(
107 f'rhodecode.lib.celerylib.tasks.{method}'
108 f'rhodecode.lib.celerylib.tasks.{method}'
108 )
109 )
109 return inquired_task.delay(extras).get(timeout=self.TASK_TIMEOUT)
110 return inquired_task.delay(extras).get(timeout=self.TASK_TIMEOUT)
110
111
111
112
112 class HooksShadowRepoClient:
113 class HooksShadowRepoClient:
113
114
114 def __call__(self, hook_name, extras):
115 def __call__(self, hook_name, extras):
115 return {'output': '', 'status': 0}
116 return {'output': '', 'status': 0}
116
117
117
118
118 class RemoteMessageWriter:
119 class RemoteMessageWriter:
119 """Writer base class."""
120 """Writer base class."""
120 def write(self, message):
121 def write(self, message):
121 raise NotImplementedError()
122 raise NotImplementedError()
122
123
123
124
124 class HgMessageWriter(RemoteMessageWriter):
125 class HgMessageWriter(RemoteMessageWriter):
125 """Writer that knows how to send messages to mercurial clients."""
126 """Writer that knows how to send messages to mercurial clients."""
126
127
127 def __init__(self, ui):
128 def __init__(self, ui):
128 self.ui = ui
129 self.ui = ui
129
130
130 def write(self, message: str):
131 def write(self, message: str):
131 # TODO: Check why the quiet flag is set by default.
132 # TODO: Check why the quiet flag is set by default.
132 old = self.ui.quiet
133 old = self.ui.quiet
133 self.ui.quiet = False
134 self.ui.quiet = False
134 self.ui.status(message.encode('utf-8'))
135 self.ui.status(message.encode('utf-8'))
135 self.ui.quiet = old
136 self.ui.quiet = old
136
137
137
138
138 class GitMessageWriter(RemoteMessageWriter):
139 class GitMessageWriter(RemoteMessageWriter):
139 """Writer that knows how to send messages to git clients."""
140 """Writer that knows how to send messages to git clients."""
140
141
141 def __init__(self, stdout=None):
142 def __init__(self, stdout=None):
142 self.stdout = stdout or sys.stdout
143 self.stdout = stdout or sys.stdout
143
144
144 def write(self, message: str):
145 def write(self, message: str):
145 self.stdout.write(message)
146 self.stdout.write(message)
146
147
147
148
148 class SvnMessageWriter(RemoteMessageWriter):
149 class SvnMessageWriter(RemoteMessageWriter):
149 """Writer that knows how to send messages to svn clients."""
150 """Writer that knows how to send messages to svn clients."""
150
151
151 def __init__(self, stderr=None):
152 def __init__(self, stderr=None):
152 # SVN needs data sent to stderr for back-to-client messaging
153 # SVN needs data sent to stderr for back-to-client messaging
153 self.stderr = stderr or sys.stderr
154 self.stderr = stderr or sys.stderr
154
155
155 def write(self, message):
156 def write(self, message):
156 self.stderr.write(message.encode('utf-8'))
157 self.stderr.write(message)
157
158
158
159
159 def _handle_exception(result):
160 def _handle_exception(result):
160 exception_class = result.get('exception')
161 exception_class = result.get('exception')
161 exception_traceback = result.get('exception_traceback')
162 exception_traceback = result.get('exception_traceback')
162 log.debug('Handling hook-call exception: %s', exception_class)
163 log.debug('Handling hook-call exception: %s', exception_class)
163
164
164 if exception_traceback:
165 if exception_traceback:
165 log.error('Got traceback from remote call:%s', exception_traceback)
166 log.error('Got traceback from remote call:%s', exception_traceback)
166
167
167 if exception_class == 'HTTPLockedRC':
168 if exception_class == 'HTTPLockedRC':
168 raise exceptions.RepositoryLockedException()(*result['exception_args'])
169 raise exceptions.RepositoryLockedException()(*result['exception_args'])
169 elif exception_class == 'HTTPBranchProtected':
170 elif exception_class == 'HTTPBranchProtected':
170 raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
171 raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
171 elif exception_class == 'RepositoryError':
172 elif exception_class == 'RepositoryError':
172 raise exceptions.VcsException()(*result['exception_args'])
173 raise exceptions.VcsException()(*result['exception_args'])
173 elif exception_class:
174 elif exception_class:
174 raise Exception(
175 raise Exception(
175 f"""Got remote exception "{exception_class}" with args "{result['exception_args']}" """
176 f"""Got remote exception "{exception_class}" with args "{result['exception_args']}" """
176 )
177 )
177
178
178
179
179 def _get_hooks_client(extras):
180 def _get_hooks_client(extras):
180 hooks_uri = extras.get('hooks_uri')
181 hooks_uri = extras.get('hooks_uri')
181 task_queue = extras.get('task_queue')
182 task_queue = extras.get('task_queue')
182 task_backend = extras.get('task_backend')
183 task_backend = extras.get('task_backend')
183 is_shadow_repo = extras.get('is_shadow_repo')
184 is_shadow_repo = extras.get('is_shadow_repo')
184
185
185 if hooks_uri:
186 if hooks_uri:
186 return HooksHttpClient(hooks_uri)
187 return HooksHttpClient(hooks_uri)
187 elif task_queue and task_backend:
188 elif task_queue and task_backend:
188 return HooksCeleryClient(task_queue, task_backend)
189 return HooksCeleryClient(task_queue, task_backend)
189 elif is_shadow_repo:
190 elif is_shadow_repo:
190 return HooksShadowRepoClient()
191 return HooksShadowRepoClient()
191 else:
192 else:
192 raise Exception("Hooks client not found!")
193 raise Exception("Hooks client not found!")
193
194
194
195
195 def _call_hook(hook_name, extras, writer):
196 def _call_hook(hook_name, extras, writer):
196 hooks_client = _get_hooks_client(extras)
197 hooks_client = _get_hooks_client(extras)
197 log.debug('Hooks, using client:%s', hooks_client)
198 log.debug('Hooks, using client:%s', hooks_client)
198 result = hooks_client(hook_name, extras)
199 result = hooks_client(hook_name, extras)
199 log.debug('Hooks got result: %s', result)
200 log.debug('Hooks got result: %s', result)
200 _handle_exception(result)
201 _handle_exception(result)
201 writer.write(result['output'])
202 writer.write(result['output'])
202
203
203 return result['status']
204 return result['status']
204
205
205
206
206 def _extras_from_ui(ui):
207 def _extras_from_ui(ui):
207 hook_data = ui.config(b'rhodecode', b'RC_SCM_DATA')
208 hook_data = ui.config(b'rhodecode', b'RC_SCM_DATA')
208 if not hook_data:
209 if not hook_data:
209 # maybe it's inside environ ?
210 # maybe it's inside environ ?
210 env_hook_data = os.environ.get('RC_SCM_DATA')
211 env_hook_data = os.environ.get('RC_SCM_DATA')
211 if env_hook_data:
212 if env_hook_data:
212 hook_data = env_hook_data
213 hook_data = env_hook_data
213
214
214 extras = {}
215 extras = {}
215 if hook_data:
216 if hook_data:
216 extras = json.loads(hook_data)
217 extras = json.loads(hook_data)
217 return extras
218 return extras
218
219
219
220
220 def _rev_range_hash(repo, node, check_heads=False):
221 def _rev_range_hash(repo, node, check_heads=False):
221 from vcsserver.hgcompat import get_ctx
222 from vcsserver.hgcompat import get_ctx
222
223
223 commits = []
224 commits = []
224 revs = []
225 revs = []
225 start = get_ctx(repo, node).rev()
226 start = get_ctx(repo, node).rev()
226 end = len(repo)
227 end = len(repo)
227 for rev in range(start, end):
228 for rev in range(start, end):
228 revs.append(rev)
229 revs.append(rev)
229 ctx = get_ctx(repo, rev)
230 ctx = get_ctx(repo, rev)
230 commit_id = ascii_str(mercurial.node.hex(ctx.node()))
231 commit_id = ascii_str(mercurial.node.hex(ctx.node()))
231 branch = safe_str(ctx.branch())
232 branch = safe_str(ctx.branch())
232 commits.append((commit_id, branch))
233 commits.append((commit_id, branch))
233
234
234 parent_heads = []
235 parent_heads = []
235 if check_heads:
236 if check_heads:
236 parent_heads = _check_heads(repo, start, end, revs)
237 parent_heads = _check_heads(repo, start, end, revs)
237 return commits, parent_heads
238 return commits, parent_heads
238
239
239
240
240 def _check_heads(repo, start, end, commits):
241 def _check_heads(repo, start, end, commits):
241 from vcsserver.hgcompat import get_ctx
242 from vcsserver.hgcompat import get_ctx
242 changelog = repo.changelog
243 changelog = repo.changelog
243 parents = set()
244 parents = set()
244
245
245 for new_rev in commits:
246 for new_rev in commits:
246 for p in changelog.parentrevs(new_rev):
247 for p in changelog.parentrevs(new_rev):
247 if p == mercurial.node.nullrev:
248 if p == mercurial.node.nullrev:
248 continue
249 continue
249 if p < start:
250 if p < start:
250 parents.add(p)
251 parents.add(p)
251
252
252 for p in parents:
253 for p in parents:
253 branch = get_ctx(repo, p).branch()
254 branch = get_ctx(repo, p).branch()
254 # The heads descending from that parent, on the same branch
255 # The heads descending from that parent, on the same branch
255 parent_heads = {p}
256 parent_heads = {p}
256 reachable = {p}
257 reachable = {p}
257 for x in range(p + 1, end):
258 for x in range(p + 1, end):
258 if get_ctx(repo, x).branch() != branch:
259 if get_ctx(repo, x).branch() != branch:
259 continue
260 continue
260 for pp in changelog.parentrevs(x):
261 for pp in changelog.parentrevs(x):
261 if pp in reachable:
262 if pp in reachable:
262 reachable.add(x)
263 reachable.add(x)
263 parent_heads.discard(pp)
264 parent_heads.discard(pp)
264 parent_heads.add(x)
265 parent_heads.add(x)
265 # More than one head? Suggest merging
266 # More than one head? Suggest merging
266 if len(parent_heads) > 1:
267 if len(parent_heads) > 1:
267 return list(parent_heads)
268 return list(parent_heads)
268
269
269 return []
270 return []
270
271
271
272
272 def _get_git_env():
273 def _get_git_env():
273 env = {}
274 env = {}
274 for k, v in os.environ.items():
275 for k, v in os.environ.items():
275 if k.startswith('GIT'):
276 if k.startswith('GIT'):
276 env[k] = v
277 env[k] = v
277
278
278 # serialized version
279 # serialized version
279 return [(k, v) for k, v in env.items()]
280 return [(k, v) for k, v in env.items()]
280
281
281
282
282 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
283 def _get_hg_env(old_rev, new_rev, txnid, repo_path):
283 env = {}
284 env = {}
284 for k, v in os.environ.items():
285 for k, v in os.environ.items():
285 if k.startswith('HG'):
286 if k.startswith('HG'):
286 env[k] = v
287 env[k] = v
287
288
288 env['HG_NODE'] = old_rev
289 env['HG_NODE'] = old_rev
289 env['HG_NODE_LAST'] = new_rev
290 env['HG_NODE_LAST'] = new_rev
290 env['HG_TXNID'] = txnid
291 env['HG_TXNID'] = txnid
291 env['HG_PENDING'] = repo_path
292 env['HG_PENDING'] = repo_path
292
293
293 return [(k, v) for k, v in env.items()]
294 return [(k, v) for k, v in env.items()]
294
295
295
296
297 def _fix_hooks_executables():
298 """
299 This is a trick to set proper settings.EXECUTABLE paths for certain execution patterns
300 especially for subversion where hooks strip entire env, and calling just 'svn' command will most likely fail
301 because svn is not on PATH
302 """
303 vcsserver.settings.BINARY_DIR = (
304 os.environ.get('RC_BINARY_DIR') or vcsserver.settings.BINARY_DIR)
305 vcsserver.settings.GIT_EXECUTABLE = (
306 os.environ.get('RC_GIT_EXECUTABLE') or vcsserver.settings.GIT_EXECUTABLE)
307 vcsserver.settings.SVN_EXECUTABLE = (
308 os.environ.get('RC_SVN_EXECUTABLE') or vcsserver.settings.SVN_EXECUTABLE)
309 vcsserver.settings.SVNLOOK_EXECUTABLE = (
310 os.environ.get('RC_SVNLOOK_EXECUTABLE') or vcsserver.settings.SVNLOOK_EXECUTABLE)
311
312
296 def repo_size(ui, repo, **kwargs):
313 def repo_size(ui, repo, **kwargs):
297 extras = _extras_from_ui(ui)
314 extras = _extras_from_ui(ui)
298 return _call_hook('repo_size', extras, HgMessageWriter(ui))
315 return _call_hook('repo_size', extras, HgMessageWriter(ui))
299
316
300
317
301 def pre_pull(ui, repo, **kwargs):
318 def pre_pull(ui, repo, **kwargs):
302 extras = _extras_from_ui(ui)
319 extras = _extras_from_ui(ui)
303 return _call_hook('pre_pull', extras, HgMessageWriter(ui))
320 return _call_hook('pre_pull', extras, HgMessageWriter(ui))
304
321
305
322
306 def pre_pull_ssh(ui, repo, **kwargs):
323 def pre_pull_ssh(ui, repo, **kwargs):
307 extras = _extras_from_ui(ui)
324 extras = _extras_from_ui(ui)
308 if extras and extras.get('SSH'):
325 if extras and extras.get('SSH'):
309 return pre_pull(ui, repo, **kwargs)
326 return pre_pull(ui, repo, **kwargs)
310 return 0
327 return 0
311
328
312
329
313 def post_pull(ui, repo, **kwargs):
330 def post_pull(ui, repo, **kwargs):
314 extras = _extras_from_ui(ui)
331 extras = _extras_from_ui(ui)
315 return _call_hook('post_pull', extras, HgMessageWriter(ui))
332 return _call_hook('post_pull', extras, HgMessageWriter(ui))
316
333
317
334
318 def post_pull_ssh(ui, repo, **kwargs):
335 def post_pull_ssh(ui, repo, **kwargs):
319 extras = _extras_from_ui(ui)
336 extras = _extras_from_ui(ui)
320 if extras and extras.get('SSH'):
337 if extras and extras.get('SSH'):
321 return post_pull(ui, repo, **kwargs)
338 return post_pull(ui, repo, **kwargs)
322 return 0
339 return 0
323
340
324
341
325 def pre_push(ui, repo, node=None, **kwargs):
342 def pre_push(ui, repo, node=None, **kwargs):
326 """
343 """
327 Mercurial pre_push hook
344 Mercurial pre_push hook
328 """
345 """
329 extras = _extras_from_ui(ui)
346 extras = _extras_from_ui(ui)
330 detect_force_push = extras.get('detect_force_push')
347 detect_force_push = extras.get('detect_force_push')
331
348
332 rev_data = []
349 rev_data = []
333 hook_type: str = safe_str(kwargs.get('hooktype'))
350 hook_type: str = safe_str(kwargs.get('hooktype'))
334
351
335 if node and hook_type == 'pretxnchangegroup':
352 if node and hook_type == 'pretxnchangegroup':
336 branches = collections.defaultdict(list)
353 branches = collections.defaultdict(list)
337 commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
354 commits, _heads = _rev_range_hash(repo, node, check_heads=detect_force_push)
338 for commit_id, branch in commits:
355 for commit_id, branch in commits:
339 branches[branch].append(commit_id)
356 branches[branch].append(commit_id)
340
357
341 for branch, commits in branches.items():
358 for branch, commits in branches.items():
342 old_rev = ascii_str(kwargs.get('node_last')) or commits[0]
359 old_rev = ascii_str(kwargs.get('node_last')) or commits[0]
343 rev_data.append({
360 rev_data.append({
344 'total_commits': len(commits),
361 'total_commits': len(commits),
345 'old_rev': old_rev,
362 'old_rev': old_rev,
346 'new_rev': commits[-1],
363 'new_rev': commits[-1],
347 'ref': '',
364 'ref': '',
348 'type': 'branch',
365 'type': 'branch',
349 'name': branch,
366 'name': branch,
350 })
367 })
351
368
352 for push_ref in rev_data:
369 for push_ref in rev_data:
353 push_ref['multiple_heads'] = _heads
370 push_ref['multiple_heads'] = _heads
354
371
355 repo_path = os.path.join(
372 repo_path = os.path.join(
356 extras.get('repo_store', ''), extras.get('repository', ''))
373 extras.get('repo_store', ''), extras.get('repository', ''))
357 push_ref['hg_env'] = _get_hg_env(
374 push_ref['hg_env'] = _get_hg_env(
358 old_rev=push_ref['old_rev'],
375 old_rev=push_ref['old_rev'],
359 new_rev=push_ref['new_rev'], txnid=ascii_str(kwargs.get('txnid')),
376 new_rev=push_ref['new_rev'], txnid=ascii_str(kwargs.get('txnid')),
360 repo_path=repo_path)
377 repo_path=repo_path)
361
378
362 extras['hook_type'] = hook_type or 'pre_push'
379 extras['hook_type'] = hook_type or 'pre_push'
363 extras['commit_ids'] = rev_data
380 extras['commit_ids'] = rev_data
364
381
365 return _call_hook('pre_push', extras, HgMessageWriter(ui))
382 return _call_hook('pre_push', extras, HgMessageWriter(ui))
366
383
367
384
368 def pre_push_ssh(ui, repo, node=None, **kwargs):
385 def pre_push_ssh(ui, repo, node=None, **kwargs):
369 extras = _extras_from_ui(ui)
386 extras = _extras_from_ui(ui)
370 if extras.get('SSH'):
387 if extras.get('SSH'):
371 return pre_push(ui, repo, node, **kwargs)
388 return pre_push(ui, repo, node, **kwargs)
372
389
373 return 0
390 return 0
374
391
375
392
376 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
393 def pre_push_ssh_auth(ui, repo, node=None, **kwargs):
377 """
394 """
378 Mercurial pre_push hook for SSH
395 Mercurial pre_push hook for SSH
379 """
396 """
380 extras = _extras_from_ui(ui)
397 extras = _extras_from_ui(ui)
381 if extras.get('SSH'):
398 if extras.get('SSH'):
382 permission = extras['SSH_PERMISSIONS']
399 permission = extras['SSH_PERMISSIONS']
383
400
384 if 'repository.write' == permission or 'repository.admin' == permission:
401 if 'repository.write' == permission or 'repository.admin' == permission:
385 return 0
402 return 0
386
403
387 # non-zero ret code
404 # non-zero ret code
388 return 1
405 return 1
389
406
390 return 0
407 return 0
391
408
392
409
393 def post_push(ui, repo, node, **kwargs):
410 def post_push(ui, repo, node, **kwargs):
394 """
411 """
395 Mercurial post_push hook
412 Mercurial post_push hook
396 """
413 """
397 extras = _extras_from_ui(ui)
414 extras = _extras_from_ui(ui)
398
415
399 commit_ids = []
416 commit_ids = []
400 branches = []
417 branches = []
401 bookmarks = []
418 bookmarks = []
402 tags = []
419 tags = []
403 hook_type: str = safe_str(kwargs.get('hooktype'))
420 hook_type: str = safe_str(kwargs.get('hooktype'))
404
421
405 commits, _heads = _rev_range_hash(repo, node)
422 commits, _heads = _rev_range_hash(repo, node)
406 for commit_id, branch in commits:
423 for commit_id, branch in commits:
407 commit_ids.append(commit_id)
424 commit_ids.append(commit_id)
408 if branch not in branches:
425 if branch not in branches:
409 branches.append(branch)
426 branches.append(branch)
410
427
411 if hasattr(ui, '_rc_pushkey_bookmarks'):
428 if hasattr(ui, '_rc_pushkey_bookmarks'):
412 bookmarks = ui._rc_pushkey_bookmarks
429 bookmarks = ui._rc_pushkey_bookmarks
413
430
414 extras['hook_type'] = hook_type or 'post_push'
431 extras['hook_type'] = hook_type or 'post_push'
415 extras['commit_ids'] = commit_ids
432 extras['commit_ids'] = commit_ids
416
433
417 extras['new_refs'] = {
434 extras['new_refs'] = {
418 'branches': branches,
435 'branches': branches,
419 'bookmarks': bookmarks,
436 'bookmarks': bookmarks,
420 'tags': tags
437 'tags': tags
421 }
438 }
422
439
423 return _call_hook('post_push', extras, HgMessageWriter(ui))
440 return _call_hook('post_push', extras, HgMessageWriter(ui))
424
441
425
442
426 def post_push_ssh(ui, repo, node, **kwargs):
443 def post_push_ssh(ui, repo, node, **kwargs):
427 """
444 """
428 Mercurial post_push hook for SSH
445 Mercurial post_push hook for SSH
429 """
446 """
430 if _extras_from_ui(ui).get('SSH'):
447 if _extras_from_ui(ui).get('SSH'):
431 return post_push(ui, repo, node, **kwargs)
448 return post_push(ui, repo, node, **kwargs)
432 return 0
449 return 0
433
450
434
451
435 def key_push(ui, repo, **kwargs):
452 def key_push(ui, repo, **kwargs):
436 from vcsserver.hgcompat import get_ctx
453 from vcsserver.hgcompat import get_ctx
437
454
438 if kwargs['new'] != b'0' and kwargs['namespace'] == b'bookmarks':
455 if kwargs['new'] != b'0' and kwargs['namespace'] == b'bookmarks':
439 # store new bookmarks in our UI object propagated later to post_push
456 # store new bookmarks in our UI object propagated later to post_push
440 ui._rc_pushkey_bookmarks = get_ctx(repo, kwargs['key']).bookmarks()
457 ui._rc_pushkey_bookmarks = get_ctx(repo, kwargs['key']).bookmarks()
441 return
458 return
442
459
443
460
444 # backward compat
461 # backward compat
445 log_pull_action = post_pull
462 log_pull_action = post_pull
446
463
447 # backward compat
464 # backward compat
448 log_push_action = post_push
465 log_push_action = post_push
449
466
450
467
451 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
468 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
452 """
469 """
453 Old hook name: keep here for backward compatibility.
470 Old hook name: keep here for backward compatibility.
454
471
455 This is only required when the installed git hooks are not upgraded.
472 This is only required when the installed git hooks are not upgraded.
456 """
473 """
457 pass
474 pass
458
475
459
476
460 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
477 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
461 """
478 """
462 Old hook name: keep here for backward compatibility.
479 Old hook name: keep here for backward compatibility.
463
480
464 This is only required when the installed git hooks are not upgraded.
481 This is only required when the installed git hooks are not upgraded.
465 """
482 """
466 pass
483 pass
467
484
468
485
469 @dataclasses.dataclass
486 @dataclasses.dataclass
470 class HookResponse:
487 class HookResponse:
471 status: int
488 status: int
472 output: str
489 output: str
473
490
474
491
475 def git_pre_pull(extras) -> HookResponse:
492 def git_pre_pull(extras) -> HookResponse:
476 """
493 """
477 Pre pull hook.
494 Pre pull hook.
478
495
479 :param extras: dictionary containing the keys defined in simplevcs
496 :param extras: dictionary containing the keys defined in simplevcs
480 :type extras: dict
497 :type extras: dict
481
498
482 :return: status code of the hook. 0 for success.
499 :return: status code of the hook. 0 for success.
483 :rtype: int
500 :rtype: int
484 """
501 """
485
502
486 if 'pull' not in extras['hooks']:
503 if 'pull' not in extras['hooks']:
487 return HookResponse(0, '')
504 return HookResponse(0, '')
488
505
489 stdout = io.StringIO()
506 stdout = io.StringIO()
490 try:
507 try:
491 status_code = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
508 status_code = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
492
509
493 except Exception as error:
510 except Exception as error:
494 log.exception('Failed to call pre_pull hook')
511 log.exception('Failed to call pre_pull hook')
495 status_code = 128
512 status_code = 128
496 stdout.write(f'ERROR: {error}\n')
513 stdout.write(f'ERROR: {error}\n')
497
514
498 return HookResponse(status_code, stdout.getvalue())
515 return HookResponse(status_code, stdout.getvalue())
499
516
500
517
501 def git_post_pull(extras) -> HookResponse:
518 def git_post_pull(extras) -> HookResponse:
502 """
519 """
503 Post pull hook.
520 Post pull hook.
504
521
505 :param extras: dictionary containing the keys defined in simplevcs
522 :param extras: dictionary containing the keys defined in simplevcs
506 :type extras: dict
523 :type extras: dict
507
524
508 :return: status code of the hook. 0 for success.
525 :return: status code of the hook. 0 for success.
509 :rtype: int
526 :rtype: int
510 """
527 """
511 if 'pull' not in extras['hooks']:
528 if 'pull' not in extras['hooks']:
512 return HookResponse(0, '')
529 return HookResponse(0, '')
513
530
514 stdout = io.StringIO()
531 stdout = io.StringIO()
515 try:
532 try:
516 status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
533 status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
517 except Exception as error:
534 except Exception as error:
518 status = 128
535 status = 128
519 stdout.write(f'ERROR: {error}\n')
536 stdout.write(f'ERROR: {error}\n')
520
537
521 return HookResponse(status, stdout.getvalue())
538 return HookResponse(status, stdout.getvalue())
522
539
523
540
524 def _parse_git_ref_lines(revision_lines):
541 def _parse_git_ref_lines(revision_lines):
525 rev_data = []
542 rev_data = []
526 for revision_line in revision_lines or []:
543 for revision_line in revision_lines or []:
527 old_rev, new_rev, ref = revision_line.strip().split(' ')
544 old_rev, new_rev, ref = revision_line.strip().split(' ')
528 ref_data = ref.split('/', 2)
545 ref_data = ref.split('/', 2)
529 if ref_data[1] in ('tags', 'heads'):
546 if ref_data[1] in ('tags', 'heads'):
530 rev_data.append({
547 rev_data.append({
531 # NOTE(marcink):
548 # NOTE(marcink):
532 # we're unable to tell total_commits for git at this point
549 # we're unable to tell total_commits for git at this point
533 # but we set the variable for consistency with GIT
550 # but we set the variable for consistency with GIT
534 'total_commits': -1,
551 'total_commits': -1,
535 'old_rev': old_rev,
552 'old_rev': old_rev,
536 'new_rev': new_rev,
553 'new_rev': new_rev,
537 'ref': ref,
554 'ref': ref,
538 'type': ref_data[1],
555 'type': ref_data[1],
539 'name': ref_data[2],
556 'name': ref_data[2],
540 })
557 })
541 return rev_data
558 return rev_data
542
559
543
560
544 def git_pre_receive(unused_repo_path, revision_lines, env) -> int:
561 def git_pre_receive(unused_repo_path, revision_lines, env) -> int:
545 """
562 """
546 Pre push hook.
563 Pre push hook.
547
564
548 :return: status code of the hook. 0 for success.
565 :return: status code of the hook. 0 for success.
549 """
566 """
550 extras = json.loads(env['RC_SCM_DATA'])
567 extras = json.loads(env['RC_SCM_DATA'])
551 rev_data = _parse_git_ref_lines(revision_lines)
568 rev_data = _parse_git_ref_lines(revision_lines)
552 if 'push' not in extras['hooks']:
569 if 'push' not in extras['hooks']:
553 return 0
570 return 0
554 empty_commit_id = '0' * 40
571 empty_commit_id = '0' * 40
555
572
556 detect_force_push = extras.get('detect_force_push')
573 detect_force_push = extras.get('detect_force_push')
557
574 _fix_hooks_executables()
558 for push_ref in rev_data:
575 for push_ref in rev_data:
559 # store our git-env which holds the temp store
576 # store our git-env which holds the temp store
560 push_ref['git_env'] = _get_git_env()
577 push_ref['git_env'] = _get_git_env()
561 push_ref['pruned_sha'] = ''
578 push_ref['pruned_sha'] = ''
562 if not detect_force_push:
579 if not detect_force_push:
563 # don't check for forced-push when we don't need to
580 # don't check for forced-push when we don't need to
564 continue
581 continue
565
582
566 type_ = push_ref['type']
583 type_ = push_ref['type']
567 new_branch = push_ref['old_rev'] == empty_commit_id
584 new_branch = push_ref['old_rev'] == empty_commit_id
568 delete_branch = push_ref['new_rev'] == empty_commit_id
585 delete_branch = push_ref['new_rev'] == empty_commit_id
569 if type_ == 'heads' and not (new_branch or delete_branch):
586 if type_ == 'heads' and not (new_branch or delete_branch):
570 old_rev = push_ref['old_rev']
587 old_rev = push_ref['old_rev']
571 new_rev = push_ref['new_rev']
588 new_rev = push_ref['new_rev']
572 cmd = [settings.GIT_EXECUTABLE, 'rev-list', old_rev, f'^{new_rev}']
589 cmd = [settings.GIT_EXECUTABLE, 'rev-list', old_rev, f'^{new_rev}']
573 stdout, stderr = subprocessio.run_command(
590 stdout, stderr = subprocessio.run_command(
574 cmd, env=os.environ.copy())
591 cmd, env=os.environ.copy())
575 # means we're having some non-reachable objects, this forced push was used
592 # means we're having some non-reachable objects, this forced push was used
576 if stdout:
593 if stdout:
577 push_ref['pruned_sha'] = stdout.splitlines()
594 push_ref['pruned_sha'] = stdout.splitlines()
578
595
579 extras['hook_type'] = 'pre_receive'
596 extras['hook_type'] = 'pre_receive'
580 extras['commit_ids'] = rev_data
597 extras['commit_ids'] = rev_data
581
598
582 stdout = sys.stdout
599 stdout = sys.stdout
583 status_code = _call_hook('pre_push', extras, GitMessageWriter(stdout))
600 status_code = _call_hook('pre_push', extras, GitMessageWriter(stdout))
584
601
585 return status_code
602 return status_code
586
603
587
604
588 def git_post_receive(unused_repo_path, revision_lines, env) -> int:
605 def git_post_receive(unused_repo_path, revision_lines, env) -> int:
589 """
606 """
590 Post push hook.
607 Post push hook.
591
608
592 :return: status code of the hook. 0 for success.
609 :return: status code of the hook. 0 for success.
593 """
610 """
594 extras = json.loads(env['RC_SCM_DATA'])
611 extras = json.loads(env['RC_SCM_DATA'])
595 if 'push' not in extras['hooks']:
612 if 'push' not in extras['hooks']:
596 return 0
613 return 0
614 _fix_hooks_executables()
597
615
598 rev_data = _parse_git_ref_lines(revision_lines)
616 rev_data = _parse_git_ref_lines(revision_lines)
599
617
600 git_revs = []
618 git_revs = []
601
619
602 # N.B.(skreft): it is ok to just call git, as git before calling a
620 # N.B.(skreft): it is ok to just call git, as git before calling a
603 # subcommand sets the PATH environment variable so that it point to the
621 # subcommand sets the PATH environment variable so that it point to the
604 # correct version of the git executable.
622 # correct version of the git executable.
605 empty_commit_id = '0' * 40
623 empty_commit_id = '0' * 40
606 branches = []
624 branches = []
607 tags = []
625 tags = []
608 for push_ref in rev_data:
626 for push_ref in rev_data:
609 type_ = push_ref['type']
627 type_ = push_ref['type']
610
628
611 if type_ == 'heads':
629 if type_ == 'heads':
612 # starting new branch case
630 # starting new branch case
613 if push_ref['old_rev'] == empty_commit_id:
631 if push_ref['old_rev'] == empty_commit_id:
614 push_ref_name = push_ref['name']
632 push_ref_name = push_ref['name']
615
633
616 if push_ref_name not in branches:
634 if push_ref_name not in branches:
617 branches.append(push_ref_name)
635 branches.append(push_ref_name)
618
636
619 need_head_set = ''
637 need_head_set = ''
620 with Repository(os.getcwd()) as repo:
638 with Repository(os.getcwd()) as repo:
621 try:
639 try:
622 repo.head
640 repo.head
623 except pygit2.GitError:
641 except pygit2.GitError:
624 need_head_set = f'refs/heads/{push_ref_name}'
642 need_head_set = f'refs/heads/{push_ref_name}'
625
643
626 if need_head_set:
644 if need_head_set:
627 repo.set_head(need_head_set)
645 repo.set_head(need_head_set)
628 print(f"Setting default branch to {push_ref_name}")
646 print(f"Setting default branch to {push_ref_name}")
629
647
630 cmd = [settings.GIT_EXECUTABLE, 'for-each-ref', '--format=%(refname)', 'refs/heads/*']
648 cmd = [settings.GIT_EXECUTABLE, 'for-each-ref', '--format=%(refname)', 'refs/heads/*']
631 stdout, stderr = subprocessio.run_command(
649 stdout, stderr = subprocessio.run_command(
632 cmd, env=os.environ.copy())
650 cmd, env=os.environ.copy())
633 heads = safe_str(stdout)
651 heads = safe_str(stdout)
634 heads = heads.replace(push_ref['ref'], '')
652 heads = heads.replace(push_ref['ref'], '')
635 heads = ' '.join(head for head
653 heads = ' '.join(head for head
636 in heads.splitlines() if head) or '.'
654 in heads.splitlines() if head) or '.'
637 cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse',
655 cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse',
638 '--pretty=format:%H', '--', push_ref['new_rev'],
656 '--pretty=format:%H', '--', push_ref['new_rev'],
639 '--not', heads]
657 '--not', heads]
640 stdout, stderr = subprocessio.run_command(
658 stdout, stderr = subprocessio.run_command(
641 cmd, env=os.environ.copy())
659 cmd, env=os.environ.copy())
642 git_revs.extend(list(map(ascii_str, stdout.splitlines())))
660 git_revs.extend(list(map(ascii_str, stdout.splitlines())))
643
661
644 # delete branch case
662 # delete branch case
645 elif push_ref['new_rev'] == empty_commit_id:
663 elif push_ref['new_rev'] == empty_commit_id:
646 git_revs.append(f'delete_branch=>{push_ref["name"]}')
664 git_revs.append(f'delete_branch=>{push_ref["name"]}')
647 else:
665 else:
648 if push_ref['name'] not in branches:
666 if push_ref['name'] not in branches:
649 branches.append(push_ref['name'])
667 branches.append(push_ref['name'])
650
668
651 cmd = [settings.GIT_EXECUTABLE, 'log',
669 cmd = [settings.GIT_EXECUTABLE, 'log',
652 f'{push_ref["old_rev"]}..{push_ref["new_rev"]}',
670 f'{push_ref["old_rev"]}..{push_ref["new_rev"]}',
653 '--reverse', '--pretty=format:%H']
671 '--reverse', '--pretty=format:%H']
654 stdout, stderr = subprocessio.run_command(
672 stdout, stderr = subprocessio.run_command(
655 cmd, env=os.environ.copy())
673 cmd, env=os.environ.copy())
656 # we get bytes from stdout, we need str to be consistent
674 # we get bytes from stdout, we need str to be consistent
657 log_revs = list(map(ascii_str, stdout.splitlines()))
675 log_revs = list(map(ascii_str, stdout.splitlines()))
658 git_revs.extend(log_revs)
676 git_revs.extend(log_revs)
659
677
660 # Pure pygit2 impl. but still 2-3x slower :/
678 # Pure pygit2 impl. but still 2-3x slower :/
661 # results = []
679 # results = []
662 #
680 #
663 # with Repository(os.getcwd()) as repo:
681 # with Repository(os.getcwd()) as repo:
664 # repo_new_rev = repo[push_ref['new_rev']]
682 # repo_new_rev = repo[push_ref['new_rev']]
665 # repo_old_rev = repo[push_ref['old_rev']]
683 # repo_old_rev = repo[push_ref['old_rev']]
666 # walker = repo.walk(repo_new_rev.id, pygit2.GIT_SORT_TOPOLOGICAL)
684 # walker = repo.walk(repo_new_rev.id, pygit2.GIT_SORT_TOPOLOGICAL)
667 #
685 #
668 # for commit in walker:
686 # for commit in walker:
669 # if commit.id == repo_old_rev.id:
687 # if commit.id == repo_old_rev.id:
670 # break
688 # break
671 # results.append(commit.id.hex)
689 # results.append(commit.id.hex)
672 # # reverse the order, can't use GIT_SORT_REVERSE
690 # # reverse the order, can't use GIT_SORT_REVERSE
673 # log_revs = results[::-1]
691 # log_revs = results[::-1]
674
692
675 elif type_ == 'tags':
693 elif type_ == 'tags':
676 if push_ref['name'] not in tags:
694 if push_ref['name'] not in tags:
677 tags.append(push_ref['name'])
695 tags.append(push_ref['name'])
678 git_revs.append(f'tag=>{push_ref["name"]}')
696 git_revs.append(f'tag=>{push_ref["name"]}')
679
697
680 extras['hook_type'] = 'post_receive'
698 extras['hook_type'] = 'post_receive'
681 extras['commit_ids'] = git_revs
699 extras['commit_ids'] = git_revs
682 extras['new_refs'] = {
700 extras['new_refs'] = {
683 'branches': branches,
701 'branches': branches,
684 'bookmarks': [],
702 'bookmarks': [],
685 'tags': tags,
703 'tags': tags,
686 }
704 }
687
705
688 stdout = sys.stdout
706 stdout = sys.stdout
689
707
690 if 'repo_size' in extras['hooks']:
708 if 'repo_size' in extras['hooks']:
691 try:
709 try:
692 _call_hook('repo_size', extras, GitMessageWriter(stdout))
710 _call_hook('repo_size', extras, GitMessageWriter(stdout))
693 except Exception:
711 except Exception:
694 pass
712 pass
695
713
696 status_code = _call_hook('post_push', extras, GitMessageWriter(stdout))
714 status_code = _call_hook('post_push', extras, GitMessageWriter(stdout))
697 return status_code
715 return status_code
698
716
699
717
700 def _get_extras_from_txn_id(path, txn_id):
718 def _get_extras_from_txn_id(path, txn_id):
701 extras = {}
719 extras = {}
702 try:
720 try:
703 cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
721 cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
704 '-t', txn_id,
722 '-t', txn_id,
705 '--revprop', path, 'rc-scm-extras']
723 '--revprop', path, 'rc-scm-extras']
706 stdout, stderr = subprocessio.run_command(
724 stdout, stderr = subprocessio.run_command(
707 cmd, env=os.environ.copy())
725 cmd, env=os.environ.copy())
708 extras = json.loads(base64.urlsafe_b64decode(stdout))
726 extras = json.loads(base64.urlsafe_b64decode(stdout))
709 except Exception:
727 except Exception:
710 log.exception('Failed to extract extras info from txn_id')
728 log.exception('Failed to extract extras info from txn_id')
711
729
712 return extras
730 return extras
713
731
714
732
715 def _get_extras_from_commit_id(commit_id, path):
733 def _get_extras_from_commit_id(commit_id, path):
716 extras = {}
734 extras = {}
717 try:
735 try:
718 cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
736 cmd = [settings.SVNLOOK_EXECUTABLE, 'pget',
719 '-r', commit_id,
737 '-r', commit_id,
720 '--revprop', path, 'rc-scm-extras']
738 '--revprop', path, 'rc-scm-extras']
721 stdout, stderr = subprocessio.run_command(
739 stdout, stderr = subprocessio.run_command(
722 cmd, env=os.environ.copy())
740 cmd, env=os.environ.copy())
723 extras = json.loads(base64.urlsafe_b64decode(stdout))
741 extras = json.loads(base64.urlsafe_b64decode(stdout))
724 except Exception:
742 except Exception:
725 log.exception('Failed to extract extras info from commit_id')
743 log.exception('Failed to extract extras info from commit_id')
726
744
727 return extras
745 return extras
728
746
729
747
730 def svn_pre_commit(repo_path, commit_data, env):
748 def svn_pre_commit(repo_path, commit_data, env):
731 path, txn_id = commit_data
749 path, txn_id = commit_data
732 branches = []
750 branches = []
733 tags = []
751 tags = []
734
752
753 _fix_hooks_executables()
735 if env.get('RC_SCM_DATA'):
754 if env.get('RC_SCM_DATA'):
736 extras = json.loads(env['RC_SCM_DATA'])
755 extras = json.loads(env['RC_SCM_DATA'])
737 else:
756 else:
738 # fallback method to read from TXN-ID stored data
757 # fallback method to read from TXN-ID stored data
739 extras = _get_extras_from_txn_id(path, txn_id)
758 extras = _get_extras_from_txn_id(path, txn_id)
740 if not extras:
759 if not extras:
741 return 0
760 return 0
742
761
743 extras['hook_type'] = 'pre_commit'
762 extras['hook_type'] = 'pre_commit'
744 extras['commit_ids'] = [txn_id]
763 extras['commit_ids'] = [txn_id]
745 extras['txn_id'] = txn_id
764 extras['txn_id'] = txn_id
746 extras['new_refs'] = {
765 extras['new_refs'] = {
747 'total_commits': 1,
766 'total_commits': 1,
748 'branches': branches,
767 'branches': branches,
749 'bookmarks': [],
768 'bookmarks': [],
750 'tags': tags,
769 'tags': tags,
751 }
770 }
752
771
753 return _call_hook('pre_push', extras, SvnMessageWriter())
772 return _call_hook('pre_push', extras, SvnMessageWriter())
754
773
755
774
756 def svn_post_commit(repo_path, commit_data, env):
775 def svn_post_commit(repo_path, commit_data, env):
757 """
776 """
758 commit_data is path, rev, txn_id
777 commit_data is path, rev, txn_id
759 """
778 """
779
760 if len(commit_data) == 3:
780 if len(commit_data) == 3:
761 path, commit_id, txn_id = commit_data
781 path, commit_id, txn_id = commit_data
762 elif len(commit_data) == 2:
782 elif len(commit_data) == 2:
763 log.error('Failed to extract txn_id from commit_data using legacy method. '
783 log.error('Failed to extract txn_id from commit_data using legacy method. '
764 'Some functionality might be limited')
784 'Some functionality might be limited')
765 path, commit_id = commit_data
785 path, commit_id = commit_data
766 txn_id = None
786 txn_id = None
787 else:
788 return 0
767
789
768 branches = []
790 branches = []
769 tags = []
791 tags = []
770
792
793 _fix_hooks_executables()
771 if env.get('RC_SCM_DATA'):
794 if env.get('RC_SCM_DATA'):
772 extras = json.loads(env['RC_SCM_DATA'])
795 extras = json.loads(env['RC_SCM_DATA'])
773 else:
796 else:
774 # fallback method to read from TXN-ID stored data
797 # fallback method to read from TXN-ID stored data
775 extras = _get_extras_from_commit_id(commit_id, path)
798 extras = _get_extras_from_commit_id(commit_id, path)
776 if not extras:
799 if not extras:
777 return 0
800 return 0
778
801
779 extras['hook_type'] = 'post_commit'
802 extras['hook_type'] = 'post_commit'
780 extras['commit_ids'] = [commit_id]
803 extras['commit_ids'] = [commit_id]
781 extras['txn_id'] = txn_id
804 extras['txn_id'] = txn_id
782 extras['new_refs'] = {
805 extras['new_refs'] = {
783 'branches': branches,
806 'branches': branches,
784 'bookmarks': [],
807 'bookmarks': [],
785 'tags': tags,
808 'tags': tags,
786 'total_commits': 1,
809 'total_commits': 1,
787 }
810 }
788
811
789 if 'repo_size' in extras['hooks']:
812 if 'repo_size' in extras['hooks']:
790 try:
813 try:
791 _call_hook('repo_size', extras, SvnMessageWriter())
814 _call_hook('repo_size', extras, SvnMessageWriter())
792 except Exception:
815 except Exception:
793 pass
816 pass
794
817
795 return _call_hook('post_push', extras, SvnMessageWriter())
818 return _call_hook('post_push', extras, SvnMessageWriter())
@@ -1,775 +1,777 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import io
18 import io
19 import os
19 import os
20 import platform
20 import platform
21 import sys
21 import sys
22 import locale
22 import locale
23 import logging
23 import logging
24 import uuid
24 import uuid
25 import time
25 import time
26 import wsgiref.util
26 import wsgiref.util
27 import tempfile
27 import tempfile
28 import psutil
28 import psutil
29
29
30 from itertools import chain
30 from itertools import chain
31
31
32 import msgpack
32 import msgpack
33 import configparser
33 import configparser
34
34
35 from pyramid.config import Configurator
35 from pyramid.config import Configurator
36 from pyramid.wsgi import wsgiapp
36 from pyramid.wsgi import wsgiapp
37 from pyramid.response import Response
37 from pyramid.response import Response
38
38
39 from vcsserver.base import BytesEnvelope, BinaryEnvelope
39 from vcsserver.base import BytesEnvelope, BinaryEnvelope
40 from vcsserver.lib.rc_json import json
40 from vcsserver.lib.rc_json import json
41 from vcsserver.config.settings_maker import SettingsMaker
41 from vcsserver.config.settings_maker import SettingsMaker
42 from vcsserver.str_utils import safe_int
42 from vcsserver.str_utils import safe_int
43 from vcsserver.lib.statsd_client import StatsdClient
43 from vcsserver.lib.statsd_client import StatsdClient
44 from vcsserver.tweens.request_wrapper import get_headers_call_context
44 from vcsserver.tweens.request_wrapper import get_headers_call_context
45
45
46 import vcsserver
46 import vcsserver
47 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
47 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
48 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
48 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
49 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
49 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
50 from vcsserver.echo_stub.echo_app import EchoApp
50 from vcsserver.echo_stub.echo_app import EchoApp
51 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
51 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
52 from vcsserver.lib.exc_tracking import store_exception, format_exc
52 from vcsserver.lib.exc_tracking import store_exception, format_exc
53 from vcsserver.server import VcsServer
53 from vcsserver.server import VcsServer
54
54
55 strict_vcs = True
55 strict_vcs = True
56
56
57 git_import_err = None
57 git_import_err = None
58 try:
58 try:
59 from vcsserver.remote.git_remote import GitFactory, GitRemote
59 from vcsserver.remote.git_remote import GitFactory, GitRemote
60 except ImportError as e:
60 except ImportError as e:
61 GitFactory = None
61 GitFactory = None
62 GitRemote = None
62 GitRemote = None
63 git_import_err = e
63 git_import_err = e
64 if strict_vcs:
64 if strict_vcs:
65 raise
65 raise
66
66
67
67
68 hg_import_err = None
68 hg_import_err = None
69 try:
69 try:
70 from vcsserver.remote.hg_remote import MercurialFactory, HgRemote
70 from vcsserver.remote.hg_remote import MercurialFactory, HgRemote
71 except ImportError as e:
71 except ImportError as e:
72 MercurialFactory = None
72 MercurialFactory = None
73 HgRemote = None
73 HgRemote = None
74 hg_import_err = e
74 hg_import_err = e
75 if strict_vcs:
75 if strict_vcs:
76 raise
76 raise
77
77
78
78
79 svn_import_err = None
79 svn_import_err = None
80 try:
80 try:
81 from vcsserver.remote.svn_remote import SubversionFactory, SvnRemote
81 from vcsserver.remote.svn_remote import SubversionFactory, SvnRemote
82 except ImportError as e:
82 except ImportError as e:
83 SubversionFactory = None
83 SubversionFactory = None
84 SvnRemote = None
84 SvnRemote = None
85 svn_import_err = e
85 svn_import_err = e
86 if strict_vcs:
86 if strict_vcs:
87 raise
87 raise
88
88
89 log = logging.getLogger(__name__)
89 log = logging.getLogger(__name__)
90
90
91 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
91 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
92 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
92 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
93
93
94 try:
94 try:
95 locale.setlocale(locale.LC_ALL, '')
95 locale.setlocale(locale.LC_ALL, '')
96 except locale.Error as e:
96 except locale.Error as e:
97 log.error(
97 log.error(
98 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
98 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
99 os.environ['LC_ALL'] = 'C'
99 os.environ['LC_ALL'] = 'C'
100
100
101
101
102 def _is_request_chunked(environ):
102 def _is_request_chunked(environ):
103 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
103 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
104 return stream
104 return stream
105
105
106
106
107 def log_max_fd():
107 def log_max_fd():
108 try:
108 try:
109 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
109 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
110 log.info('Max file descriptors value: %s', maxfd)
110 log.info('Max file descriptors value: %s', maxfd)
111 except Exception:
111 except Exception:
112 pass
112 pass
113
113
114
114
115 class VCS:
115 class VCS:
116 def __init__(self, locale_conf=None, cache_config=None):
116 def __init__(self, locale_conf=None, cache_config=None):
117 self.locale = locale_conf
117 self.locale = locale_conf
118 self.cache_config = cache_config
118 self.cache_config = cache_config
119 self._configure_locale()
119 self._configure_locale()
120
120
121 log_max_fd()
121 log_max_fd()
122
122
123 if GitFactory and GitRemote:
123 if GitFactory and GitRemote:
124 git_factory = GitFactory()
124 git_factory = GitFactory()
125 self._git_remote = GitRemote(git_factory)
125 self._git_remote = GitRemote(git_factory)
126 else:
126 else:
127 log.error("Git client import failed: %s", git_import_err)
127 log.error("Git client import failed: %s", git_import_err)
128
128
129 if MercurialFactory and HgRemote:
129 if MercurialFactory and HgRemote:
130 hg_factory = MercurialFactory()
130 hg_factory = MercurialFactory()
131 self._hg_remote = HgRemote(hg_factory)
131 self._hg_remote = HgRemote(hg_factory)
132 else:
132 else:
133 log.error("Mercurial client import failed: %s", hg_import_err)
133 log.error("Mercurial client import failed: %s", hg_import_err)
134
134
135 if SubversionFactory and SvnRemote:
135 if SubversionFactory and SvnRemote:
136 svn_factory = SubversionFactory()
136 svn_factory = SubversionFactory()
137
137
138 # hg factory is used for svn url validation
138 # hg factory is used for svn url validation
139 hg_factory = MercurialFactory()
139 hg_factory = MercurialFactory()
140 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
140 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
141 else:
141 else:
142 log.error("Subversion client import failed: %s", svn_import_err)
142 log.error("Subversion client import failed: %s", svn_import_err)
143
143
144 self._vcsserver = VcsServer()
144 self._vcsserver = VcsServer()
145
145
146 def _configure_locale(self):
146 def _configure_locale(self):
147 if self.locale:
147 if self.locale:
148 log.info('Settings locale: `LC_ALL` to %s', self.locale)
148 log.info('Settings locale: `LC_ALL` to %s', self.locale)
149 else:
149 else:
150 log.info('Configuring locale subsystem based on environment variables')
150 log.info('Configuring locale subsystem based on environment variables')
151 try:
151 try:
152 # If self.locale is the empty string, then the locale
152 # If self.locale is the empty string, then the locale
153 # module will use the environment variables. See the
153 # module will use the environment variables. See the
154 # documentation of the package `locale`.
154 # documentation of the package `locale`.
155 locale.setlocale(locale.LC_ALL, self.locale)
155 locale.setlocale(locale.LC_ALL, self.locale)
156
156
157 language_code, encoding = locale.getlocale()
157 language_code, encoding = locale.getlocale()
158 log.info(
158 log.info(
159 'Locale set to language code "%s" with encoding "%s".',
159 'Locale set to language code "%s" with encoding "%s".',
160 language_code, encoding)
160 language_code, encoding)
161 except locale.Error:
161 except locale.Error:
162 log.exception('Cannot set locale, not configuring the locale system')
162 log.exception('Cannot set locale, not configuring the locale system')
163
163
164
164
165 class WsgiProxy:
165 class WsgiProxy:
166 def __init__(self, wsgi):
166 def __init__(self, wsgi):
167 self.wsgi = wsgi
167 self.wsgi = wsgi
168
168
169 def __call__(self, environ, start_response):
169 def __call__(self, environ, start_response):
170 input_data = environ['wsgi.input'].read()
170 input_data = environ['wsgi.input'].read()
171 input_data = msgpack.unpackb(input_data)
171 input_data = msgpack.unpackb(input_data)
172
172
173 error = None
173 error = None
174 try:
174 try:
175 data, status, headers = self.wsgi.handle(
175 data, status, headers = self.wsgi.handle(
176 input_data['environment'], input_data['input_data'],
176 input_data['environment'], input_data['input_data'],
177 *input_data['args'], **input_data['kwargs'])
177 *input_data['args'], **input_data['kwargs'])
178 except Exception as e:
178 except Exception as e:
179 data, status, headers = [], None, None
179 data, status, headers = [], None, None
180 error = {
180 error = {
181 'message': str(e),
181 'message': str(e),
182 '_vcs_kind': getattr(e, '_vcs_kind', None)
182 '_vcs_kind': getattr(e, '_vcs_kind', None)
183 }
183 }
184
184
185 start_response(200, {})
185 start_response(200, {})
186 return self._iterator(error, status, headers, data)
186 return self._iterator(error, status, headers, data)
187
187
188 def _iterator(self, error, status, headers, data):
188 def _iterator(self, error, status, headers, data):
189 initial_data = [
189 initial_data = [
190 error,
190 error,
191 status,
191 status,
192 headers,
192 headers,
193 ]
193 ]
194
194
195 for d in chain(initial_data, data):
195 for d in chain(initial_data, data):
196 yield msgpack.packb(d)
196 yield msgpack.packb(d)
197
197
198
198
199 def not_found(request):
199 def not_found(request):
200 return {'status': '404 NOT FOUND'}
200 return {'status': '404 NOT FOUND'}
201
201
202
202
203 class VCSViewPredicate:
203 class VCSViewPredicate:
204 def __init__(self, val, config):
204 def __init__(self, val, config):
205 self.remotes = val
205 self.remotes = val
206
206
207 def text(self):
207 def text(self):
208 return f'vcs view method = {list(self.remotes.keys())}'
208 return f'vcs view method = {list(self.remotes.keys())}'
209
209
210 phash = text
210 phash = text
211
211
212 def __call__(self, context, request):
212 def __call__(self, context, request):
213 """
213 """
214 View predicate that returns true if given backend is supported by
214 View predicate that returns true if given backend is supported by
215 defined remotes.
215 defined remotes.
216 """
216 """
217 backend = request.matchdict.get('backend')
217 backend = request.matchdict.get('backend')
218 return backend in self.remotes
218 return backend in self.remotes
219
219
220
220
221 class HTTPApplication:
221 class HTTPApplication:
222 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
222 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
223
223
224 remote_wsgi = remote_wsgi
224 remote_wsgi = remote_wsgi
225 _use_echo_app = False
225 _use_echo_app = False
226
226
227 def __init__(self, settings=None, global_config=None):
227 def __init__(self, settings=None, global_config=None):
228
228
229 self.config = Configurator(settings=settings)
229 self.config = Configurator(settings=settings)
230 # Init our statsd at very start
230 # Init our statsd at very start
231 self.config.registry.statsd = StatsdClient.statsd
231 self.config.registry.statsd = StatsdClient.statsd
232 self.config.registry.vcs_call_context = {}
232 self.config.registry.vcs_call_context = {}
233
233
234 self.global_config = global_config
234 self.global_config = global_config
235 self.config.include('vcsserver.lib.rc_cache')
235 self.config.include('vcsserver.lib.rc_cache')
236 self.config.include('vcsserver.lib.rc_cache.archive_cache')
236 self.config.include('vcsserver.lib.rc_cache.archive_cache')
237
237
238 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
238 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
239 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
239 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
240 self._remotes = {
240 self._remotes = {
241 'hg': vcs._hg_remote,
241 'hg': vcs._hg_remote,
242 'git': vcs._git_remote,
242 'git': vcs._git_remote,
243 'svn': vcs._svn_remote,
243 'svn': vcs._svn_remote,
244 'server': vcs._vcsserver,
244 'server': vcs._vcsserver,
245 }
245 }
246 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
246 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
247 self._use_echo_app = True
247 self._use_echo_app = True
248 log.warning("Using EchoApp for VCS operations.")
248 log.warning("Using EchoApp for VCS operations.")
249 self.remote_wsgi = remote_wsgi_stub
249 self.remote_wsgi = remote_wsgi_stub
250
250
251 self._configure_settings(global_config, settings)
251 self._configure_settings(global_config, settings)
252
252
253 self._configure()
253 self._configure()
254
254
255 def _configure_settings(self, global_config, app_settings):
255 def _configure_settings(self, global_config, app_settings):
256 """
256 """
257 Configure the settings module.
257 Configure the settings module.
258 """
258 """
259 settings_merged = global_config.copy()
259 settings_merged = global_config.copy()
260 settings_merged.update(app_settings)
260 settings_merged.update(app_settings)
261
261
262 git_path = app_settings.get('git_path', None)
262 binary_dir = app_settings['core.binary_dir']
263 if git_path:
263
264 settings.GIT_EXECUTABLE = git_path
264 settings.BINARY_DIR = binary_dir
265 binary_dir = app_settings.get('core.binary_dir', None)
265
266 if binary_dir:
266 # from core.binary dir we set executable paths
267 settings.BINARY_DIR = binary_dir
267 settings.GIT_EXECUTABLE = os.path.join(binary_dir, settings.GIT_EXECUTABLE)
268 settings.SVN_EXECUTABLE = os.path.join(binary_dir, settings.SVN_EXECUTABLE)
269 settings.SVNLOOK_EXECUTABLE = os.path.join(binary_dir, settings.SVNLOOK_EXECUTABLE)
268
270
269 # Store the settings to make them available to other modules.
271 # Store the settings to make them available to other modules.
270 vcsserver.PYRAMID_SETTINGS = settings_merged
272 vcsserver.PYRAMID_SETTINGS = settings_merged
271 vcsserver.CONFIG = settings_merged
273 vcsserver.CONFIG = settings_merged
272
274
273 def _configure(self):
275 def _configure(self):
274 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
276 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
275
277
276 self.config.add_route('service', '/_service')
278 self.config.add_route('service', '/_service')
277 self.config.add_route('status', '/status')
279 self.config.add_route('status', '/status')
278 self.config.add_route('hg_proxy', '/proxy/hg')
280 self.config.add_route('hg_proxy', '/proxy/hg')
279 self.config.add_route('git_proxy', '/proxy/git')
281 self.config.add_route('git_proxy', '/proxy/git')
280
282
281 # rpc methods
283 # rpc methods
282 self.config.add_route('vcs', '/{backend}')
284 self.config.add_route('vcs', '/{backend}')
283
285
284 # streaming rpc remote methods
286 # streaming rpc remote methods
285 self.config.add_route('vcs_stream', '/{backend}/stream')
287 self.config.add_route('vcs_stream', '/{backend}/stream')
286
288
287 # vcs operations clone/push as streaming
289 # vcs operations clone/push as streaming
288 self.config.add_route('stream_git', '/stream/git/*repo_name')
290 self.config.add_route('stream_git', '/stream/git/*repo_name')
289 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
291 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
290
292
291 self.config.add_view(self.status_view, route_name='status', renderer='json')
293 self.config.add_view(self.status_view, route_name='status', renderer='json')
292 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
294 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
293
295
294 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
296 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
295 self.config.add_view(self.git_proxy(), route_name='git_proxy')
297 self.config.add_view(self.git_proxy(), route_name='git_proxy')
296 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
298 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
297 vcs_view=self._remotes)
299 vcs_view=self._remotes)
298 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
300 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
299 vcs_view=self._remotes)
301 vcs_view=self._remotes)
300
302
301 self.config.add_view(self.hg_stream(), route_name='stream_hg')
303 self.config.add_view(self.hg_stream(), route_name='stream_hg')
302 self.config.add_view(self.git_stream(), route_name='stream_git')
304 self.config.add_view(self.git_stream(), route_name='stream_git')
303
305
304 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
306 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
305
307
306 self.config.add_notfound_view(not_found, renderer='json')
308 self.config.add_notfound_view(not_found, renderer='json')
307
309
308 self.config.add_view(self.handle_vcs_exception, context=Exception)
310 self.config.add_view(self.handle_vcs_exception, context=Exception)
309
311
310 self.config.add_tween(
312 self.config.add_tween(
311 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
313 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
312 )
314 )
313 self.config.add_request_method(
315 self.config.add_request_method(
314 'vcsserver.lib.request_counter.get_request_counter',
316 'vcsserver.lib.request_counter.get_request_counter',
315 'request_count')
317 'request_count')
316
318
317 def wsgi_app(self):
319 def wsgi_app(self):
318 return self.config.make_wsgi_app()
320 return self.config.make_wsgi_app()
319
321
320 def _vcs_view_params(self, request):
322 def _vcs_view_params(self, request):
321 remote = self._remotes[request.matchdict['backend']]
323 remote = self._remotes[request.matchdict['backend']]
322 payload = msgpack.unpackb(request.body, use_list=True)
324 payload = msgpack.unpackb(request.body, use_list=True)
323
325
324 method = payload.get('method')
326 method = payload.get('method')
325 params = payload['params']
327 params = payload['params']
326 wire = params.get('wire')
328 wire = params.get('wire')
327 args = params.get('args')
329 args = params.get('args')
328 kwargs = params.get('kwargs')
330 kwargs = params.get('kwargs')
329 context_uid = None
331 context_uid = None
330
332
331 request.registry.vcs_call_context = {
333 request.registry.vcs_call_context = {
332 'method': method,
334 'method': method,
333 'repo_name': payload.get('_repo_name'),
335 'repo_name': payload.get('_repo_name'),
334 }
336 }
335
337
336 if wire:
338 if wire:
337 try:
339 try:
338 wire['context'] = context_uid = uuid.UUID(wire['context'])
340 wire['context'] = context_uid = uuid.UUID(wire['context'])
339 except KeyError:
341 except KeyError:
340 pass
342 pass
341 args.insert(0, wire)
343 args.insert(0, wire)
342 repo_state_uid = wire.get('repo_state_uid') if wire else None
344 repo_state_uid = wire.get('repo_state_uid') if wire else None
343
345
344 # NOTE(marcink): trading complexity for slight performance
346 # NOTE(marcink): trading complexity for slight performance
345 if log.isEnabledFor(logging.DEBUG):
347 if log.isEnabledFor(logging.DEBUG):
346 # also we SKIP printing out any of those methods args since they maybe excessive
348 # also we SKIP printing out any of those methods args since they maybe excessive
347 just_args_methods = {
349 just_args_methods = {
348 'commitctx': ('content', 'removed', 'updated'),
350 'commitctx': ('content', 'removed', 'updated'),
349 'commit': ('content', 'removed', 'updated')
351 'commit': ('content', 'removed', 'updated')
350 }
352 }
351 if method in just_args_methods:
353 if method in just_args_methods:
352 skip_args = just_args_methods[method]
354 skip_args = just_args_methods[method]
353 call_args = ''
355 call_args = ''
354 call_kwargs = {}
356 call_kwargs = {}
355 for k in kwargs:
357 for k in kwargs:
356 if k in skip_args:
358 if k in skip_args:
357 # replace our skip key with dummy
359 # replace our skip key with dummy
358 call_kwargs[k] = f'RemovedParam({k})'
360 call_kwargs[k] = f'RemovedParam({k})'
359 else:
361 else:
360 call_kwargs[k] = kwargs[k]
362 call_kwargs[k] = kwargs[k]
361 else:
363 else:
362 call_args = args[1:]
364 call_args = args[1:]
363 call_kwargs = kwargs
365 call_kwargs = kwargs
364
366
365 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
367 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
366 method, call_args, call_kwargs, context_uid, repo_state_uid)
368 method, call_args, call_kwargs, context_uid, repo_state_uid)
367
369
368 statsd = request.registry.statsd
370 statsd = request.registry.statsd
369 if statsd:
371 if statsd:
370 statsd.incr(
372 statsd.incr(
371 'vcsserver_method_total', tags=[
373 'vcsserver_method_total', tags=[
372 f"method:{method}",
374 f"method:{method}",
373 ])
375 ])
374 return payload, remote, method, args, kwargs
376 return payload, remote, method, args, kwargs
375
377
376 def vcs_view(self, request):
378 def vcs_view(self, request):
377
379
378 payload, remote, method, args, kwargs = self._vcs_view_params(request)
380 payload, remote, method, args, kwargs = self._vcs_view_params(request)
379 payload_id = payload.get('id')
381 payload_id = payload.get('id')
380
382
381 try:
383 try:
382 resp = getattr(remote, method)(*args, **kwargs)
384 resp = getattr(remote, method)(*args, **kwargs)
383 except Exception as e:
385 except Exception as e:
384 exc_info = list(sys.exc_info())
386 exc_info = list(sys.exc_info())
385 exc_type, exc_value, exc_traceback = exc_info
387 exc_type, exc_value, exc_traceback = exc_info
386
388
387 org_exc = getattr(e, '_org_exc', None)
389 org_exc = getattr(e, '_org_exc', None)
388 org_exc_name = None
390 org_exc_name = None
389 org_exc_tb = ''
391 org_exc_tb = ''
390 if org_exc:
392 if org_exc:
391 org_exc_name = org_exc.__class__.__name__
393 org_exc_name = org_exc.__class__.__name__
392 org_exc_tb = getattr(e, '_org_exc_tb', '')
394 org_exc_tb = getattr(e, '_org_exc_tb', '')
393 # replace our "faked" exception with our org
395 # replace our "faked" exception with our org
394 exc_info[0] = org_exc.__class__
396 exc_info[0] = org_exc.__class__
395 exc_info[1] = org_exc
397 exc_info[1] = org_exc
396
398
397 should_store_exc = True
399 should_store_exc = True
398 if org_exc:
400 if org_exc:
399 def get_exc_fqn(_exc_obj):
401 def get_exc_fqn(_exc_obj):
400 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
402 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
401 return module_name + '.' + org_exc_name
403 return module_name + '.' + org_exc_name
402
404
403 exc_fqn = get_exc_fqn(org_exc)
405 exc_fqn = get_exc_fqn(org_exc)
404
406
405 if exc_fqn in ['mercurial.error.RepoLookupError',
407 if exc_fqn in ['mercurial.error.RepoLookupError',
406 'vcsserver.exceptions.RefNotFoundException']:
408 'vcsserver.exceptions.RefNotFoundException']:
407 should_store_exc = False
409 should_store_exc = False
408
410
409 if should_store_exc:
411 if should_store_exc:
410 store_exception(id(exc_info), exc_info, request_path=request.path)
412 store_exception(id(exc_info), exc_info, request_path=request.path)
411
413
412 tb_info = format_exc(exc_info)
414 tb_info = format_exc(exc_info)
413
415
414 type_ = e.__class__.__name__
416 type_ = e.__class__.__name__
415 if type_ not in self.ALLOWED_EXCEPTIONS:
417 if type_ not in self.ALLOWED_EXCEPTIONS:
416 type_ = None
418 type_ = None
417
419
418 resp = {
420 resp = {
419 'id': payload_id,
421 'id': payload_id,
420 'error': {
422 'error': {
421 'message': str(e),
423 'message': str(e),
422 'traceback': tb_info,
424 'traceback': tb_info,
423 'org_exc': org_exc_name,
425 'org_exc': org_exc_name,
424 'org_exc_tb': org_exc_tb,
426 'org_exc_tb': org_exc_tb,
425 'type': type_
427 'type': type_
426 }
428 }
427 }
429 }
428
430
429 try:
431 try:
430 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
432 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
431 except AttributeError:
433 except AttributeError:
432 pass
434 pass
433 else:
435 else:
434 resp = {
436 resp = {
435 'id': payload_id,
437 'id': payload_id,
436 'result': resp
438 'result': resp
437 }
439 }
438 log.debug('Serving data for method %s', method)
440 log.debug('Serving data for method %s', method)
439 return resp
441 return resp
440
442
441 def vcs_stream_view(self, request):
443 def vcs_stream_view(self, request):
442 payload, remote, method, args, kwargs = self._vcs_view_params(request)
444 payload, remote, method, args, kwargs = self._vcs_view_params(request)
443 # this method has a stream: marker we remove it here
445 # this method has a stream: marker we remove it here
444 method = method.split('stream:')[-1]
446 method = method.split('stream:')[-1]
445 chunk_size = safe_int(payload.get('chunk_size')) or 4096
447 chunk_size = safe_int(payload.get('chunk_size')) or 4096
446
448
447 resp = getattr(remote, method)(*args, **kwargs)
449 resp = getattr(remote, method)(*args, **kwargs)
448
450
449 def get_chunked_data(method_resp):
451 def get_chunked_data(method_resp):
450 stream = io.BytesIO(method_resp)
452 stream = io.BytesIO(method_resp)
451 while 1:
453 while 1:
452 chunk = stream.read(chunk_size)
454 chunk = stream.read(chunk_size)
453 if not chunk:
455 if not chunk:
454 break
456 break
455 yield chunk
457 yield chunk
456
458
457 response = Response(app_iter=get_chunked_data(resp))
459 response = Response(app_iter=get_chunked_data(resp))
458 response.content_type = 'application/octet-stream'
460 response.content_type = 'application/octet-stream'
459
461
460 return response
462 return response
461
463
462 def status_view(self, request):
464 def status_view(self, request):
463 import vcsserver
465 import vcsserver
464 _platform_id = platform.uname()[1] or 'instance'
466 _platform_id = platform.uname()[1] or 'instance'
465
467
466 return {
468 return {
467 "status": "OK",
469 "status": "OK",
468 "vcsserver_version": vcsserver.get_version(),
470 "vcsserver_version": vcsserver.get_version(),
469 "platform": _platform_id,
471 "platform": _platform_id,
470 "pid": os.getpid(),
472 "pid": os.getpid(),
471 }
473 }
472
474
473 def service_view(self, request):
475 def service_view(self, request):
474 import vcsserver
476 import vcsserver
475
477
476 payload = msgpack.unpackb(request.body, use_list=True)
478 payload = msgpack.unpackb(request.body, use_list=True)
477 server_config, app_config = {}, {}
479 server_config, app_config = {}, {}
478
480
479 try:
481 try:
480 path = self.global_config['__file__']
482 path = self.global_config['__file__']
481 config = configparser.RawConfigParser()
483 config = configparser.RawConfigParser()
482
484
483 config.read(path)
485 config.read(path)
484
486
485 if config.has_section('server:main'):
487 if config.has_section('server:main'):
486 server_config = dict(config.items('server:main'))
488 server_config = dict(config.items('server:main'))
487 if config.has_section('app:main'):
489 if config.has_section('app:main'):
488 app_config = dict(config.items('app:main'))
490 app_config = dict(config.items('app:main'))
489
491
490 except Exception:
492 except Exception:
491 log.exception('Failed to read .ini file for display')
493 log.exception('Failed to read .ini file for display')
492
494
493 environ = list(os.environ.items())
495 environ = list(os.environ.items())
494
496
495 resp = {
497 resp = {
496 'id': payload.get('id'),
498 'id': payload.get('id'),
497 'result': dict(
499 'result': dict(
498 version=vcsserver.get_version(),
500 version=vcsserver.get_version(),
499 config=server_config,
501 config=server_config,
500 app_config=app_config,
502 app_config=app_config,
501 environ=environ,
503 environ=environ,
502 payload=payload,
504 payload=payload,
503 )
505 )
504 }
506 }
505 return resp
507 return resp
506
508
507 def _msgpack_renderer_factory(self, info):
509 def _msgpack_renderer_factory(self, info):
508
510
509 def _render(value, system):
511 def _render(value, system):
510 bin_type = False
512 bin_type = False
511 res = value.get('result')
513 res = value.get('result')
512 if isinstance(res, BytesEnvelope):
514 if isinstance(res, BytesEnvelope):
513 log.debug('Result is wrapped in BytesEnvelope type')
515 log.debug('Result is wrapped in BytesEnvelope type')
514 bin_type = True
516 bin_type = True
515 elif isinstance(res, BinaryEnvelope):
517 elif isinstance(res, BinaryEnvelope):
516 log.debug('Result is wrapped in BinaryEnvelope type')
518 log.debug('Result is wrapped in BinaryEnvelope type')
517 value['result'] = res.val
519 value['result'] = res.val
518 bin_type = True
520 bin_type = True
519
521
520 request = system.get('request')
522 request = system.get('request')
521 if request is not None:
523 if request is not None:
522 response = request.response
524 response = request.response
523 ct = response.content_type
525 ct = response.content_type
524 if ct == response.default_content_type:
526 if ct == response.default_content_type:
525 response.content_type = 'application/x-msgpack'
527 response.content_type = 'application/x-msgpack'
526 if bin_type:
528 if bin_type:
527 response.content_type = 'application/x-msgpack-bin'
529 response.content_type = 'application/x-msgpack-bin'
528
530
529 return msgpack.packb(value, use_bin_type=bin_type)
531 return msgpack.packb(value, use_bin_type=bin_type)
530 return _render
532 return _render
531
533
532 def set_env_from_config(self, environ, config):
534 def set_env_from_config(self, environ, config):
533 dict_conf = {}
535 dict_conf = {}
534 try:
536 try:
535 for elem in config:
537 for elem in config:
536 if elem[0] == 'rhodecode':
538 if elem[0] == 'rhodecode':
537 dict_conf = json.loads(elem[2])
539 dict_conf = json.loads(elem[2])
538 break
540 break
539 except Exception:
541 except Exception:
540 log.exception('Failed to fetch SCM CONFIG')
542 log.exception('Failed to fetch SCM CONFIG')
541 return
543 return
542
544
543 username = dict_conf.get('username')
545 username = dict_conf.get('username')
544 if username:
546 if username:
545 environ['REMOTE_USER'] = username
547 environ['REMOTE_USER'] = username
546 # mercurial specific, some extension api rely on this
548 # mercurial specific, some extension api rely on this
547 environ['HGUSER'] = username
549 environ['HGUSER'] = username
548
550
549 ip = dict_conf.get('ip')
551 ip = dict_conf.get('ip')
550 if ip:
552 if ip:
551 environ['REMOTE_HOST'] = ip
553 environ['REMOTE_HOST'] = ip
552
554
553 if _is_request_chunked(environ):
555 if _is_request_chunked(environ):
554 # set the compatibility flag for webob
556 # set the compatibility flag for webob
555 environ['wsgi.input_terminated'] = True
557 environ['wsgi.input_terminated'] = True
556
558
557 def hg_proxy(self):
559 def hg_proxy(self):
558 @wsgiapp
560 @wsgiapp
559 def _hg_proxy(environ, start_response):
561 def _hg_proxy(environ, start_response):
560 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
562 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
561 return app(environ, start_response)
563 return app(environ, start_response)
562 return _hg_proxy
564 return _hg_proxy
563
565
564 def git_proxy(self):
566 def git_proxy(self):
565 @wsgiapp
567 @wsgiapp
566 def _git_proxy(environ, start_response):
568 def _git_proxy(environ, start_response):
567 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
569 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
568 return app(environ, start_response)
570 return app(environ, start_response)
569 return _git_proxy
571 return _git_proxy
570
572
571 def hg_stream(self):
573 def hg_stream(self):
572 if self._use_echo_app:
574 if self._use_echo_app:
573 @wsgiapp
575 @wsgiapp
574 def _hg_stream(environ, start_response):
576 def _hg_stream(environ, start_response):
575 app = EchoApp('fake_path', 'fake_name', None)
577 app = EchoApp('fake_path', 'fake_name', None)
576 return app(environ, start_response)
578 return app(environ, start_response)
577 return _hg_stream
579 return _hg_stream
578 else:
580 else:
579 @wsgiapp
581 @wsgiapp
580 def _hg_stream(environ, start_response):
582 def _hg_stream(environ, start_response):
581 log.debug('http-app: handling hg stream')
583 log.debug('http-app: handling hg stream')
582 call_context = get_headers_call_context(environ)
584 call_context = get_headers_call_context(environ)
583
585
584 repo_path = call_context['repo_path']
586 repo_path = call_context['repo_path']
585 repo_name = call_context['repo_name']
587 repo_name = call_context['repo_name']
586 config = call_context['repo_config']
588 config = call_context['repo_config']
587
589
588 app = scm_app.create_hg_wsgi_app(
590 app = scm_app.create_hg_wsgi_app(
589 repo_path, repo_name, config)
591 repo_path, repo_name, config)
590
592
591 # Consistent path information for hgweb
593 # Consistent path information for hgweb
592 environ['PATH_INFO'] = call_context['path_info']
594 environ['PATH_INFO'] = call_context['path_info']
593 environ['REPO_NAME'] = repo_name
595 environ['REPO_NAME'] = repo_name
594 self.set_env_from_config(environ, config)
596 self.set_env_from_config(environ, config)
595
597
596 log.debug('http-app: starting app handler '
598 log.debug('http-app: starting app handler '
597 'with %s and process request', app)
599 'with %s and process request', app)
598 return app(environ, ResponseFilter(start_response))
600 return app(environ, ResponseFilter(start_response))
599 return _hg_stream
601 return _hg_stream
600
602
601 def git_stream(self):
603 def git_stream(self):
602 if self._use_echo_app:
604 if self._use_echo_app:
603 @wsgiapp
605 @wsgiapp
604 def _git_stream(environ, start_response):
606 def _git_stream(environ, start_response):
605 app = EchoApp('fake_path', 'fake_name', None)
607 app = EchoApp('fake_path', 'fake_name', None)
606 return app(environ, start_response)
608 return app(environ, start_response)
607 return _git_stream
609 return _git_stream
608 else:
610 else:
609 @wsgiapp
611 @wsgiapp
610 def _git_stream(environ, start_response):
612 def _git_stream(environ, start_response):
611 log.debug('http-app: handling git stream')
613 log.debug('http-app: handling git stream')
612
614
613 call_context = get_headers_call_context(environ)
615 call_context = get_headers_call_context(environ)
614
616
615 repo_path = call_context['repo_path']
617 repo_path = call_context['repo_path']
616 repo_name = call_context['repo_name']
618 repo_name = call_context['repo_name']
617 config = call_context['repo_config']
619 config = call_context['repo_config']
618
620
619 environ['PATH_INFO'] = call_context['path_info']
621 environ['PATH_INFO'] = call_context['path_info']
620 self.set_env_from_config(environ, config)
622 self.set_env_from_config(environ, config)
621
623
622 content_type = environ.get('CONTENT_TYPE', '')
624 content_type = environ.get('CONTENT_TYPE', '')
623
625
624 path = environ['PATH_INFO']
626 path = environ['PATH_INFO']
625 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
627 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
626 log.debug(
628 log.debug(
627 'LFS: Detecting if request `%s` is LFS server path based '
629 'LFS: Detecting if request `%s` is LFS server path based '
628 'on content type:`%s`, is_lfs:%s',
630 'on content type:`%s`, is_lfs:%s',
629 path, content_type, is_lfs_request)
631 path, content_type, is_lfs_request)
630
632
631 if not is_lfs_request:
633 if not is_lfs_request:
632 # fallback detection by path
634 # fallback detection by path
633 if GIT_LFS_PROTO_PAT.match(path):
635 if GIT_LFS_PROTO_PAT.match(path):
634 is_lfs_request = True
636 is_lfs_request = True
635 log.debug(
637 log.debug(
636 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
638 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
637 path, is_lfs_request)
639 path, is_lfs_request)
638
640
639 if is_lfs_request:
641 if is_lfs_request:
640 app = scm_app.create_git_lfs_wsgi_app(
642 app = scm_app.create_git_lfs_wsgi_app(
641 repo_path, repo_name, config)
643 repo_path, repo_name, config)
642 else:
644 else:
643 app = scm_app.create_git_wsgi_app(
645 app = scm_app.create_git_wsgi_app(
644 repo_path, repo_name, config)
646 repo_path, repo_name, config)
645
647
646 log.debug('http-app: starting app handler '
648 log.debug('http-app: starting app handler '
647 'with %s and process request', app)
649 'with %s and process request', app)
648
650
649 return app(environ, start_response)
651 return app(environ, start_response)
650
652
651 return _git_stream
653 return _git_stream
652
654
653 def handle_vcs_exception(self, exception, request):
655 def handle_vcs_exception(self, exception, request):
654 _vcs_kind = getattr(exception, '_vcs_kind', '')
656 _vcs_kind = getattr(exception, '_vcs_kind', '')
655
657
656 if _vcs_kind == 'repo_locked':
658 if _vcs_kind == 'repo_locked':
657 headers_call_context = get_headers_call_context(request.environ)
659 headers_call_context = get_headers_call_context(request.environ)
658 status_code = safe_int(headers_call_context['locked_status_code'])
660 status_code = safe_int(headers_call_context['locked_status_code'])
659
661
660 return HTTPRepoLocked(
662 return HTTPRepoLocked(
661 title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')])
663 title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')])
662
664
663 elif _vcs_kind == 'repo_branch_protected':
665 elif _vcs_kind == 'repo_branch_protected':
664 # Get custom repo-branch-protected status code if present.
666 # Get custom repo-branch-protected status code if present.
665 return HTTPRepoBranchProtected(
667 return HTTPRepoBranchProtected(
666 title=str(exception), headers=[('X-Rc-Branch-Protection', '1')])
668 title=str(exception), headers=[('X-Rc-Branch-Protection', '1')])
667
669
668 exc_info = request.exc_info
670 exc_info = request.exc_info
669 store_exception(id(exc_info), exc_info)
671 store_exception(id(exc_info), exc_info)
670
672
671 traceback_info = 'unavailable'
673 traceback_info = 'unavailable'
672 if request.exc_info:
674 if request.exc_info:
673 traceback_info = format_exc(request.exc_info)
675 traceback_info = format_exc(request.exc_info)
674
676
675 log.error(
677 log.error(
676 'error occurred handling this request for path: %s, \n%s',
678 'error occurred handling this request for path: %s, \n%s',
677 request.path, traceback_info)
679 request.path, traceback_info)
678
680
679 statsd = request.registry.statsd
681 statsd = request.registry.statsd
680 if statsd:
682 if statsd:
681 exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
683 exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
682 statsd.incr('vcsserver_exception_total',
684 statsd.incr('vcsserver_exception_total',
683 tags=[f"type:{exc_type}"])
685 tags=[f"type:{exc_type}"])
684 raise exception
686 raise exception
685
687
686
688
687 class ResponseFilter:
689 class ResponseFilter:
688
690
689 def __init__(self, start_response):
691 def __init__(self, start_response):
690 self._start_response = start_response
692 self._start_response = start_response
691
693
692 def __call__(self, status, response_headers, exc_info=None):
694 def __call__(self, status, response_headers, exc_info=None):
693 headers = tuple(
695 headers = tuple(
694 (h, v) for h, v in response_headers
696 (h, v) for h, v in response_headers
695 if not wsgiref.util.is_hop_by_hop(h))
697 if not wsgiref.util.is_hop_by_hop(h))
696 return self._start_response(status, headers, exc_info)
698 return self._start_response(status, headers, exc_info)
697
699
698
700
699 def sanitize_settings_and_apply_defaults(global_config, settings):
701 def sanitize_settings_and_apply_defaults(global_config, settings):
700 _global_settings_maker = SettingsMaker(global_config)
702 _global_settings_maker = SettingsMaker(global_config)
701 settings_maker = SettingsMaker(settings)
703 settings_maker = SettingsMaker(settings)
702
704
703 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
705 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
704
706
705 logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
707 logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
706 settings_maker.enable_logging(logging_conf)
708 settings_maker.enable_logging(logging_conf)
707
709
708 # Default includes, possible to change as a user
710 # Default includes, possible to change as a user
709 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
711 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
710 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
712 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
711
713
712 settings_maker.make_setting('__file__', global_config.get('__file__'))
714 settings_maker.make_setting('__file__', global_config.get('__file__'))
713
715
714 settings_maker.make_setting('pyramid.default_locale_name', 'en')
716 settings_maker.make_setting('pyramid.default_locale_name', 'en')
715 settings_maker.make_setting('locale', 'en_US.UTF-8')
717 settings_maker.make_setting('locale', 'en_US.UTF-8')
716
718
717 settings_maker.make_setting('core.binary_dir', '')
719 settings_maker.make_setting('core.binary_dir', '/usr/local/bin/rhodecode_bin/vcs_bin')
718
720
719 temp_store = tempfile.gettempdir()
721 temp_store = tempfile.gettempdir()
720 default_cache_dir = os.path.join(temp_store, 'rc_cache')
722 default_cache_dir = os.path.join(temp_store, 'rc_cache')
721 # save default, cache dir, and use it for all backends later.
723 # save default, cache dir, and use it for all backends later.
722 default_cache_dir = settings_maker.make_setting(
724 default_cache_dir = settings_maker.make_setting(
723 'cache_dir',
725 'cache_dir',
724 default=default_cache_dir, default_when_empty=True,
726 default=default_cache_dir, default_when_empty=True,
725 parser='dir:ensured')
727 parser='dir:ensured')
726
728
727 # exception store cache
729 # exception store cache
728 settings_maker.make_setting(
730 settings_maker.make_setting(
729 'exception_tracker.store_path',
731 'exception_tracker.store_path',
730 default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
732 default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
731 parser='dir:ensured'
733 parser='dir:ensured'
732 )
734 )
733
735
734 # repo_object cache defaults
736 # repo_object cache defaults
735 settings_maker.make_setting(
737 settings_maker.make_setting(
736 'rc_cache.repo_object.backend',
738 'rc_cache.repo_object.backend',
737 default='dogpile.cache.rc.file_namespace',
739 default='dogpile.cache.rc.file_namespace',
738 parser='string')
740 parser='string')
739 settings_maker.make_setting(
741 settings_maker.make_setting(
740 'rc_cache.repo_object.expiration_time',
742 'rc_cache.repo_object.expiration_time',
741 default=30 * 24 * 60 * 60, # 30days
743 default=30 * 24 * 60 * 60, # 30days
742 parser='int')
744 parser='int')
743 settings_maker.make_setting(
745 settings_maker.make_setting(
744 'rc_cache.repo_object.arguments.filename',
746 'rc_cache.repo_object.arguments.filename',
745 default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
747 default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
746 parser='string')
748 parser='string')
747
749
748 # statsd
750 # statsd
749 settings_maker.make_setting('statsd.enabled', False, parser='bool')
751 settings_maker.make_setting('statsd.enabled', False, parser='bool')
750 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
752 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
751 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
753 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
752 settings_maker.make_setting('statsd.statsd_prefix', '')
754 settings_maker.make_setting('statsd.statsd_prefix', '')
753 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
755 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
754
756
755 settings_maker.env_expand()
757 settings_maker.env_expand()
756
758
757
759
758 def main(global_config, **settings):
760 def main(global_config, **settings):
759 start_time = time.time()
761 start_time = time.time()
760 log.info('Pyramid app config starting')
762 log.info('Pyramid app config starting')
761
763
762 if MercurialFactory:
764 if MercurialFactory:
763 hgpatches.patch_largefiles_capabilities()
765 hgpatches.patch_largefiles_capabilities()
764 hgpatches.patch_subrepo_type_mapping()
766 hgpatches.patch_subrepo_type_mapping()
765
767
766 # Fill in and sanitize the defaults & do ENV expansion
768 # Fill in and sanitize the defaults & do ENV expansion
767 sanitize_settings_and_apply_defaults(global_config, settings)
769 sanitize_settings_and_apply_defaults(global_config, settings)
768
770
769 # init and bootstrap StatsdClient
771 # init and bootstrap StatsdClient
770 StatsdClient.setup(settings)
772 StatsdClient.setup(settings)
771
773
772 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
774 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
773 total_time = time.time() - start_time
775 total_time = time.time() - start_time
774 log.info('Pyramid app created and configured in %.2fs', total_time)
776 log.info('Pyramid app created and configured in %.2fs', total_time)
775 return pyramid_app
777 return pyramid_app
@@ -1,563 +1,563 b''
1 """
1 """
2 Module provides a class allowing to wrap communication over subprocess.Popen
2 Module provides a class allowing to wrap communication over subprocess.Popen
3 input, output, error streams into a meaningfull, non-blocking, concurrent
3 input, output, error streams into a meaningfull, non-blocking, concurrent
4 stream processor exposing the output data as an iterator fitting to be a
4 stream processor exposing the output data as an iterator fitting to be a
5 return value passed by a WSGI applicaiton to a WSGI server per PEP 3333.
5 return value passed by a WSGI applicaiton to a WSGI server per PEP 3333.
6
6
7 Copyright (c) 2011 Daniel Dotsenko <dotsa[at]hotmail.com>
7 Copyright (c) 2011 Daniel Dotsenko <dotsa[at]hotmail.com>
8
8
9 This file is part of git_http_backend.py Project.
9 This file is part of git_http_backend.py Project.
10
10
11 git_http_backend.py Project is free software: you can redistribute it and/or
11 git_http_backend.py Project is free software: you can redistribute it and/or
12 modify it under the terms of the GNU Lesser General Public License as
12 modify it under the terms of the GNU Lesser General Public License as
13 published by the Free Software Foundation, either version 2.1 of the License,
13 published by the Free Software Foundation, either version 2.1 of the License,
14 or (at your option) any later version.
14 or (at your option) any later version.
15
15
16 git_http_backend.py Project is distributed in the hope that it will be useful,
16 git_http_backend.py Project is distributed in the hope that it will be useful,
17 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 but WITHOUT ANY WARRANTY; without even the implied warranty of
18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 GNU Lesser General Public License for more details.
19 GNU Lesser General Public License for more details.
20
20
21 You should have received a copy of the GNU Lesser General Public License
21 You should have received a copy of the GNU Lesser General Public License
22 along with git_http_backend.py Project.
22 along with git_http_backend.py Project.
23 If not, see <http://www.gnu.org/licenses/>.
23 If not, see <http://www.gnu.org/licenses/>.
24 """
24 """
25 import os
25 import os
26 import collections
26 import collections
27 import logging
27 import logging
28 import subprocess
28 import subprocess
29 import threading
29 import threading
30
30
31 from vcsserver.str_utils import safe_str
31 from vcsserver.str_utils import safe_str
32
32
33 log = logging.getLogger(__name__)
33 log = logging.getLogger(__name__)
34
34
35
35
36 class StreamFeeder(threading.Thread):
36 class StreamFeeder(threading.Thread):
37 """
37 """
38 Normal writing into pipe-like is blocking once the buffer is filled.
38 Normal writing into pipe-like is blocking once the buffer is filled.
39 This thread allows a thread to seep data from a file-like into a pipe
39 This thread allows a thread to seep data from a file-like into a pipe
40 without blocking the main thread.
40 without blocking the main thread.
41 We close inpipe once the end of the source stream is reached.
41 We close inpipe once the end of the source stream is reached.
42 """
42 """
43
43
44 def __init__(self, source):
44 def __init__(self, source):
45 super().__init__()
45 super().__init__()
46 self.daemon = True
46 self.daemon = True
47 filelike = False
47 filelike = False
48 self.bytes = b''
48 self.bytes = b''
49 if type(source) in (str, bytes, bytearray): # string-like
49 if type(source) in (str, bytes, bytearray): # string-like
50 self.bytes = bytes(source)
50 self.bytes = bytes(source)
51 else: # can be either file pointer or file-like
51 else: # can be either file pointer or file-like
52 if isinstance(source, int): # file pointer it is
52 if isinstance(source, int): # file pointer it is
53 # converting file descriptor (int) stdin into file-like
53 # converting file descriptor (int) stdin into file-like
54 source = os.fdopen(source, 'rb', 16384)
54 source = os.fdopen(source, 'rb', 16384)
55 # let's see if source is file-like by now
55 # let's see if source is file-like by now
56 filelike = hasattr(source, 'read')
56 filelike = hasattr(source, 'read')
57 if not filelike and not self.bytes:
57 if not filelike and not self.bytes:
58 raise TypeError("StreamFeeder's source object must be a readable "
58 raise TypeError("StreamFeeder's source object must be a readable "
59 "file-like, a file descriptor, or a string-like.")
59 "file-like, a file descriptor, or a string-like.")
60 self.source = source
60 self.source = source
61 self.readiface, self.writeiface = os.pipe()
61 self.readiface, self.writeiface = os.pipe()
62
62
63 def run(self):
63 def run(self):
64 writer = self.writeiface
64 writer = self.writeiface
65 try:
65 try:
66 if self.bytes:
66 if self.bytes:
67 os.write(writer, self.bytes)
67 os.write(writer, self.bytes)
68 else:
68 else:
69 s = self.source
69 s = self.source
70
70
71 while 1:
71 while 1:
72 _bytes = s.read(4096)
72 _bytes = s.read(4096)
73 if not _bytes:
73 if not _bytes:
74 break
74 break
75 os.write(writer, _bytes)
75 os.write(writer, _bytes)
76
76
77 finally:
77 finally:
78 os.close(writer)
78 os.close(writer)
79
79
80 @property
80 @property
81 def output(self):
81 def output(self):
82 return self.readiface
82 return self.readiface
83
83
84
84
85 class InputStreamChunker(threading.Thread):
85 class InputStreamChunker(threading.Thread):
86 def __init__(self, source, target, buffer_size, chunk_size):
86 def __init__(self, source, target, buffer_size, chunk_size):
87
87
88 super().__init__()
88 super().__init__()
89
89
90 self.daemon = True # die die die.
90 self.daemon = True # die die die.
91
91
92 self.source = source
92 self.source = source
93 self.target = target
93 self.target = target
94 self.chunk_count_max = int(buffer_size / chunk_size) + 1
94 self.chunk_count_max = int(buffer_size / chunk_size) + 1
95 self.chunk_size = chunk_size
95 self.chunk_size = chunk_size
96
96
97 self.data_added = threading.Event()
97 self.data_added = threading.Event()
98 self.data_added.clear()
98 self.data_added.clear()
99
99
100 self.keep_reading = threading.Event()
100 self.keep_reading = threading.Event()
101 self.keep_reading.set()
101 self.keep_reading.set()
102
102
103 self.EOF = threading.Event()
103 self.EOF = threading.Event()
104 self.EOF.clear()
104 self.EOF.clear()
105
105
106 self.go = threading.Event()
106 self.go = threading.Event()
107 self.go.set()
107 self.go.set()
108
108
109 def stop(self):
109 def stop(self):
110 self.go.clear()
110 self.go.clear()
111 self.EOF.set()
111 self.EOF.set()
112 try:
112 try:
113 # this is not proper, but is done to force the reader thread let
113 # this is not proper, but is done to force the reader thread let
114 # go of the input because, if successful, .close() will send EOF
114 # go of the input because, if successful, .close() will send EOF
115 # down the pipe.
115 # down the pipe.
116 self.source.close()
116 self.source.close()
117 except Exception:
117 except Exception:
118 pass
118 pass
119
119
120 def run(self):
120 def run(self):
121 s = self.source
121 s = self.source
122 t = self.target
122 t = self.target
123 cs = self.chunk_size
123 cs = self.chunk_size
124 chunk_count_max = self.chunk_count_max
124 chunk_count_max = self.chunk_count_max
125 keep_reading = self.keep_reading
125 keep_reading = self.keep_reading
126 da = self.data_added
126 da = self.data_added
127 go = self.go
127 go = self.go
128
128
129 try:
129 try:
130 b = s.read(cs)
130 b = s.read(cs)
131 except ValueError:
131 except ValueError:
132 b = ''
132 b = ''
133
133
134 timeout_input = 20
134 timeout_input = 20
135 while b and go.is_set():
135 while b and go.is_set():
136 if len(t) > chunk_count_max:
136 if len(t) > chunk_count_max:
137 keep_reading.clear()
137 keep_reading.clear()
138 keep_reading.wait(timeout_input)
138 keep_reading.wait(timeout_input)
139 if len(t) > chunk_count_max + timeout_input:
139 if len(t) > chunk_count_max + timeout_input:
140 log.error("Timed out while waiting for input from subprocess.")
140 log.error("Timed out while waiting for input from subprocess.")
141 os._exit(-1) # this will cause the worker to recycle itself
141 os._exit(-1) # this will cause the worker to recycle itself
142
142
143 t.append(b)
143 t.append(b)
144 da.set()
144 da.set()
145
145
146 try:
146 try:
147 b = s.read(cs)
147 b = s.read(cs)
148 except ValueError: # probably "I/O operation on closed file"
148 except ValueError: # probably "I/O operation on closed file"
149 b = ''
149 b = ''
150
150
151 self.EOF.set()
151 self.EOF.set()
152 da.set() # for cases when done but there was no input.
152 da.set() # for cases when done but there was no input.
153
153
154
154
155 class BufferedGenerator:
155 class BufferedGenerator:
156 """
156 """
157 Class behaves as a non-blocking, buffered pipe reader.
157 Class behaves as a non-blocking, buffered pipe reader.
158 Reads chunks of data (through a thread)
158 Reads chunks of data (through a thread)
159 from a blocking pipe, and attaches these to an array (Deque) of chunks.
159 from a blocking pipe, and attaches these to an array (Deque) of chunks.
160 Reading is halted in the thread when max chunks is internally buffered.
160 Reading is halted in the thread when max chunks is internally buffered.
161 The .next() may operate in blocking or non-blocking fashion by yielding
161 The .next() may operate in blocking or non-blocking fashion by yielding
162 '' if no data is ready
162 '' if no data is ready
163 to be sent or by not returning until there is some data to send
163 to be sent or by not returning until there is some data to send
164 When we get EOF from underlying source pipe we raise the marker to raise
164 When we get EOF from underlying source pipe we raise the marker to raise
165 StopIteration after the last chunk of data is yielded.
165 StopIteration after the last chunk of data is yielded.
166 """
166 """
167
167
168 def __init__(self, name, source, buffer_size=65536, chunk_size=4096,
168 def __init__(self, name, source, buffer_size=65536, chunk_size=4096,
169 starting_values=None, bottomless=False):
169 starting_values=None, bottomless=False):
170 starting_values = starting_values or []
170 starting_values = starting_values or []
171 self.name = name
171 self.name = name
172 self.buffer_size = buffer_size
172 self.buffer_size = buffer_size
173 self.chunk_size = chunk_size
173 self.chunk_size = chunk_size
174
174
175 if bottomless:
175 if bottomless:
176 maxlen = int(buffer_size / chunk_size)
176 maxlen = int(buffer_size / chunk_size)
177 else:
177 else:
178 maxlen = None
178 maxlen = None
179
179
180 self.data_queue = collections.deque(starting_values, maxlen)
180 self.data_queue = collections.deque(starting_values, maxlen)
181 self.worker = InputStreamChunker(source, self.data_queue, buffer_size, chunk_size)
181 self.worker = InputStreamChunker(source, self.data_queue, buffer_size, chunk_size)
182 if starting_values:
182 if starting_values:
183 self.worker.data_added.set()
183 self.worker.data_added.set()
184 self.worker.start()
184 self.worker.start()
185
185
186 ####################
186 ####################
187 # Generator's methods
187 # Generator's methods
188 ####################
188 ####################
189 def __str__(self):
189 def __str__(self):
190 return f'BufferedGenerator(name={self.name} chunk: {self.chunk_size} on buffer: {self.buffer_size})'
190 return f'BufferedGenerator(name={self.name} chunk: {self.chunk_size} on buffer: {self.buffer_size})'
191
191
192 def __iter__(self):
192 def __iter__(self):
193 return self
193 return self
194
194
195 def __next__(self):
195 def __next__(self):
196
196
197 while not self.length and not self.worker.EOF.is_set():
197 while not self.length and not self.worker.EOF.is_set():
198 self.worker.data_added.clear()
198 self.worker.data_added.clear()
199 self.worker.data_added.wait(0.2)
199 self.worker.data_added.wait(0.2)
200
200
201 if self.length:
201 if self.length:
202 self.worker.keep_reading.set()
202 self.worker.keep_reading.set()
203 return bytes(self.data_queue.popleft())
203 return bytes(self.data_queue.popleft())
204 elif self.worker.EOF.is_set():
204 elif self.worker.EOF.is_set():
205 raise StopIteration
205 raise StopIteration
206
206
207 def throw(self, exc_type, value=None, traceback=None):
207 def throw(self, exc_type, value=None, traceback=None):
208 if not self.worker.EOF.is_set():
208 if not self.worker.EOF.is_set():
209 raise exc_type(value)
209 raise exc_type(value)
210
210
211 def start(self):
211 def start(self):
212 self.worker.start()
212 self.worker.start()
213
213
214 def stop(self):
214 def stop(self):
215 self.worker.stop()
215 self.worker.stop()
216
216
217 def close(self):
217 def close(self):
218 try:
218 try:
219 self.worker.stop()
219 self.worker.stop()
220 self.throw(GeneratorExit)
220 self.throw(GeneratorExit)
221 except (GeneratorExit, StopIteration):
221 except (GeneratorExit, StopIteration):
222 pass
222 pass
223
223
224 ####################
224 ####################
225 # Threaded reader's infrastructure.
225 # Threaded reader's infrastructure.
226 ####################
226 ####################
227 @property
227 @property
228 def input(self):
228 def input(self):
229 return self.worker.w
229 return self.worker.w
230
230
231 @property
231 @property
232 def data_added_event(self):
232 def data_added_event(self):
233 return self.worker.data_added
233 return self.worker.data_added
234
234
235 @property
235 @property
236 def data_added(self):
236 def data_added(self):
237 return self.worker.data_added.is_set()
237 return self.worker.data_added.is_set()
238
238
239 @property
239 @property
240 def reading_paused(self):
240 def reading_paused(self):
241 return not self.worker.keep_reading.is_set()
241 return not self.worker.keep_reading.is_set()
242
242
243 @property
243 @property
244 def done_reading_event(self):
244 def done_reading_event(self):
245 """
245 """
246 Done_reding does not mean that the iterator's buffer is empty.
246 Done_reding does not mean that the iterator's buffer is empty.
247 Iterator might have done reading from underlying source, but the read
247 Iterator might have done reading from underlying source, but the read
248 chunks might still be available for serving through .next() method.
248 chunks might still be available for serving through .next() method.
249
249
250 :returns: An Event class instance.
250 :returns: An Event class instance.
251 """
251 """
252 return self.worker.EOF
252 return self.worker.EOF
253
253
254 @property
254 @property
255 def done_reading(self):
255 def done_reading(self):
256 """
256 """
257 Done_reading does not mean that the iterator's buffer is empty.
257 Done_reading does not mean that the iterator's buffer is empty.
258 Iterator might have done reading from underlying source, but the read
258 Iterator might have done reading from underlying source, but the read
259 chunks might still be available for serving through .next() method.
259 chunks might still be available for serving through .next() method.
260
260
261 :returns: An Bool value.
261 :returns: An Bool value.
262 """
262 """
263 return self.worker.EOF.is_set()
263 return self.worker.EOF.is_set()
264
264
265 @property
265 @property
266 def length(self):
266 def length(self):
267 """
267 """
268 returns int.
268 returns int.
269
269
270 This is the length of the queue of chunks, not the length of
270 This is the length of the queue of chunks, not the length of
271 the combined contents in those chunks.
271 the combined contents in those chunks.
272
272
273 __len__() cannot be meaningfully implemented because this
273 __len__() cannot be meaningfully implemented because this
274 reader is just flying through a bottomless pit content and
274 reader is just flying through a bottomless pit content and
275 can only know the length of what it already saw.
275 can only know the length of what it already saw.
276
276
277 If __len__() on WSGI server per PEP 3333 returns a value,
277 If __len__() on WSGI server per PEP 3333 returns a value,
278 the response's length will be set to that. In order not to
278 the response's length will be set to that. In order not to
279 confuse WSGI PEP3333 servers, we will not implement __len__
279 confuse WSGI PEP3333 servers, we will not implement __len__
280 at all.
280 at all.
281 """
281 """
282 return len(self.data_queue)
282 return len(self.data_queue)
283
283
284 def prepend(self, x):
284 def prepend(self, x):
285 self.data_queue.appendleft(x)
285 self.data_queue.appendleft(x)
286
286
287 def append(self, x):
287 def append(self, x):
288 self.data_queue.append(x)
288 self.data_queue.append(x)
289
289
290 def extend(self, o):
290 def extend(self, o):
291 self.data_queue.extend(o)
291 self.data_queue.extend(o)
292
292
293 def __getitem__(self, i):
293 def __getitem__(self, i):
294 return self.data_queue[i]
294 return self.data_queue[i]
295
295
296
296
297 class SubprocessIOChunker:
297 class SubprocessIOChunker:
298 """
298 """
299 Processor class wrapping handling of subprocess IO.
299 Processor class wrapping handling of subprocess IO.
300
300
301 .. important::
301 .. important::
302
302
303 Watch out for the method `__del__` on this class. If this object
303 Watch out for the method `__del__` on this class. If this object
304 is deleted, it will kill the subprocess, so avoid to
304 is deleted, it will kill the subprocess, so avoid to
305 return the `output` attribute or usage of it like in the following
305 return the `output` attribute or usage of it like in the following
306 example::
306 example::
307
307
308 # `args` expected to run a program that produces a lot of output
308 # `args` expected to run a program that produces a lot of output
309 output = ''.join(SubprocessIOChunker(
309 output = ''.join(SubprocessIOChunker(
310 args, shell=False, inputstream=inputstream, env=environ).output)
310 args, shell=False, inputstream=inputstream, env=environ).output)
311
311
312 # `output` will not contain all the data, because the __del__ method
312 # `output` will not contain all the data, because the __del__ method
313 # has already killed the subprocess in this case before all output
313 # has already killed the subprocess in this case before all output
314 # has been consumed.
314 # has been consumed.
315
315
316
316
317
317
318 In a way, this is a "communicate()" replacement with a twist.
318 In a way, this is a "communicate()" replacement with a twist.
319
319
320 - We are multithreaded. Writing in and reading out, err are all sep threads.
320 - We are multithreaded. Writing in and reading out, err are all sep threads.
321 - We support concurrent (in and out) stream processing.
321 - We support concurrent (in and out) stream processing.
322 - The output is not a stream. It's a queue of read string (bytes, not str)
322 - The output is not a stream. It's a queue of read string (bytes, not str)
323 chunks. The object behaves as an iterable. You can "for chunk in obj:" us.
323 chunks. The object behaves as an iterable. You can "for chunk in obj:" us.
324 - We are non-blocking in more respects than communicate()
324 - We are non-blocking in more respects than communicate()
325 (reading from subprocess out pauses when internal buffer is full, but
325 (reading from subprocess out pauses when internal buffer is full, but
326 does not block the parent calling code. On the flip side, reading from
326 does not block the parent calling code. On the flip side, reading from
327 slow-yielding subprocess may block the iteration until data shows up. This
327 slow-yielding subprocess may block the iteration until data shows up. This
328 does not block the parallel inpipe reading occurring parallel thread.)
328 does not block the parallel inpipe reading occurring parallel thread.)
329
329
330 The purpose of the object is to allow us to wrap subprocess interactions into
330 The purpose of the object is to allow us to wrap subprocess interactions into
331 an iterable that can be passed to a WSGI server as the application's return
331 an iterable that can be passed to a WSGI server as the application's return
332 value. Because of stream-processing-ability, WSGI does not have to read ALL
332 value. Because of stream-processing-ability, WSGI does not have to read ALL
333 of the subprocess's output and buffer it, before handing it to WSGI server for
333 of the subprocess's output and buffer it, before handing it to WSGI server for
334 HTTP response. Instead, the class initializer reads just a bit of the stream
334 HTTP response. Instead, the class initializer reads just a bit of the stream
335 to figure out if error occurred or likely to occur and if not, just hands the
335 to figure out if error occurred or likely to occur and if not, just hands the
336 further iteration over subprocess output to the server for completion of HTTP
336 further iteration over subprocess output to the server for completion of HTTP
337 response.
337 response.
338
338
339 The real or perceived subprocess error is trapped and raised as one of
339 The real or perceived subprocess error is trapped and raised as one of
340 OSError family of exceptions
340 OSError family of exceptions
341
341
342 Example usage:
342 Example usage:
343 # try:
343 # try:
344 # answer = SubprocessIOChunker(
344 # answer = SubprocessIOChunker(
345 # cmd,
345 # cmd,
346 # input,
346 # input,
347 # buffer_size = 65536,
347 # buffer_size = 65536,
348 # chunk_size = 4096
348 # chunk_size = 4096
349 # )
349 # )
350 # except (OSError) as e:
350 # except (OSError) as e:
351 # print str(e)
351 # print str(e)
352 # raise e
352 # raise e
353 #
353 #
354 # return answer
354 # return answer
355
355
356
356
357 """
357 """
358
358
359 # TODO: johbo: This is used to make sure that the open end of the PIPE
359 # TODO: johbo: This is used to make sure that the open end of the PIPE
360 # is closed in the end. It would be way better to wrap this into an
360 # is closed in the end. It would be way better to wrap this into an
361 # object, so that it is closed automatically once it is consumed or
361 # object, so that it is closed automatically once it is consumed or
362 # something similar.
362 # something similar.
363 _close_input_fd = None
363 _close_input_fd = None
364
364
365 _closed = False
365 _closed = False
366 _stdout = None
366 _stdout = None
367 _stderr = None
367 _stderr = None
368
368
369 def __init__(self, cmd, input_stream=None, buffer_size=65536,
369 def __init__(self, cmd, input_stream=None, buffer_size=65536,
370 chunk_size=4096, starting_values=None, fail_on_stderr=True,
370 chunk_size=4096, starting_values=None, fail_on_stderr=True,
371 fail_on_return_code=True, **kwargs):
371 fail_on_return_code=True, **kwargs):
372 """
372 """
373 Initializes SubprocessIOChunker
373 Initializes SubprocessIOChunker
374
374
375 :param cmd: A Subprocess.Popen style "cmd". Can be string or array of strings
375 :param cmd: A Subprocess.Popen style "cmd". Can be string or array of strings
376 :param input_stream: (Default: None) A file-like, string, or file pointer.
376 :param input_stream: (Default: None) A file-like, string, or file pointer.
377 :param buffer_size: (Default: 65536) A size of total buffer per stream in bytes.
377 :param buffer_size: (Default: 65536) A size of total buffer per stream in bytes.
378 :param chunk_size: (Default: 4096) A max size of a chunk. Actual chunk may be smaller.
378 :param chunk_size: (Default: 4096) A max size of a chunk. Actual chunk may be smaller.
379 :param starting_values: (Default: []) An array of strings to put in front of output que.
379 :param starting_values: (Default: []) An array of strings to put in front of output que.
380 :param fail_on_stderr: (Default: True) Whether to raise an exception in
380 :param fail_on_stderr: (Default: True) Whether to raise an exception in
381 case something is written to stderr.
381 case something is written to stderr.
382 :param fail_on_return_code: (Default: True) Whether to raise an
382 :param fail_on_return_code: (Default: True) Whether to raise an
383 exception if the return code is not 0.
383 exception if the return code is not 0.
384 """
384 """
385
385
386 kwargs['shell'] = kwargs.get('shell', True)
386 kwargs['shell'] = kwargs.get('shell', True)
387
387
388 starting_values = starting_values or []
388 starting_values = starting_values or []
389 if input_stream:
389 if input_stream:
390 input_streamer = StreamFeeder(input_stream)
390 input_streamer = StreamFeeder(input_stream)
391 input_streamer.start()
391 input_streamer.start()
392 input_stream = input_streamer.output
392 input_stream = input_streamer.output
393 self._close_input_fd = input_stream
393 self._close_input_fd = input_stream
394
394
395 self._fail_on_stderr = fail_on_stderr
395 self._fail_on_stderr = fail_on_stderr
396 self._fail_on_return_code = fail_on_return_code
396 self._fail_on_return_code = fail_on_return_code
397 self.cmd = cmd
397 self.cmd = cmd
398
398
399 _p = subprocess.Popen(cmd, bufsize=-1, stdin=input_stream, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
399 _p = subprocess.Popen(cmd, bufsize=-1, stdin=input_stream, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
400 **kwargs)
400 **kwargs)
401 self.process = _p
401 self.process = _p
402
402
403 bg_out = BufferedGenerator('stdout', _p.stdout, buffer_size, chunk_size, starting_values)
403 bg_out = BufferedGenerator('stdout', _p.stdout, buffer_size, chunk_size, starting_values)
404 bg_err = BufferedGenerator('stderr', _p.stderr, 10240, 1, bottomless=True)
404 bg_err = BufferedGenerator('stderr', _p.stderr, 10240, 1, bottomless=True)
405
405
406 while not bg_out.done_reading and not bg_out.reading_paused and not bg_err.length:
406 while not bg_out.done_reading and not bg_out.reading_paused and not bg_err.length:
407 # doing this until we reach either end of file, or end of buffer.
407 # doing this until we reach either end of file, or end of buffer.
408 bg_out.data_added_event.wait(0.2)
408 bg_out.data_added_event.wait(0.2)
409 bg_out.data_added_event.clear()
409 bg_out.data_added_event.clear()
410
410
411 # at this point it's still ambiguous if we are done reading or just full buffer.
411 # at this point it's still ambiguous if we are done reading or just full buffer.
412 # Either way, if error (returned by ended process, or implied based on
412 # Either way, if error (returned by ended process, or implied based on
413 # presence of stuff in stderr output) we error out.
413 # presence of stuff in stderr output) we error out.
414 # Else, we are happy.
414 # Else, we are happy.
415 return_code = _p.poll()
415 return_code = _p.poll()
416 ret_code_ok = return_code in [None, 0]
416 ret_code_ok = return_code in [None, 0]
417 ret_code_fail = return_code is not None and return_code != 0
417 ret_code_fail = return_code is not None and return_code != 0
418 if (
418 if (
419 (ret_code_fail and fail_on_return_code) or
419 (ret_code_fail and fail_on_return_code) or
420 (ret_code_ok and fail_on_stderr and bg_err.length)
420 (ret_code_ok and fail_on_stderr and bg_err.length)
421 ):
421 ):
422
422
423 try:
423 try:
424 _p.terminate()
424 _p.terminate()
425 except Exception:
425 except Exception:
426 pass
426 pass
427
427
428 bg_out.stop()
428 bg_out.stop()
429 out = b''.join(bg_out)
429 out = b''.join(bg_out)
430 self._stdout = out
430 self._stdout = out
431
431
432 bg_err.stop()
432 bg_err.stop()
433 err = b''.join(bg_err)
433 err = b''.join(bg_err)
434 self._stderr = err
434 self._stderr = err
435
435
436 # code from https://github.com/schacon/grack/pull/7
436 # code from https://github.com/schacon/grack/pull/7
437 if err.strip() == b'fatal: The remote end hung up unexpectedly' and out.startswith(b'0034shallow '):
437 if err.strip() == b'fatal: The remote end hung up unexpectedly' and out.startswith(b'0034shallow '):
438 bg_out = iter([out])
438 bg_out = iter([out])
439 _p = None
439 _p = None
440 elif err and fail_on_stderr:
440 elif err and fail_on_stderr:
441 text_err = err.decode()
441 text_err = err.decode()
442 raise OSError(
442 raise OSError(
443 f"Subprocess exited due to an error:\n{text_err}")
443 f"Subprocess exited due to an error:\n{text_err}")
444
444
445 if ret_code_fail and fail_on_return_code:
445 if ret_code_fail and fail_on_return_code:
446 text_err = err.decode()
446 text_err = err.decode()
447 if not err:
447 if not err:
448 # maybe get empty stderr, try stdout instead
448 # maybe get empty stderr, try stdout instead
449 # in many cases git reports the errors on stdout too
449 # in many cases git reports the errors on stdout too
450 text_err = out.decode()
450 text_err = out.decode()
451 raise OSError(
451 raise OSError(
452 f"Subprocess exited with non 0 ret code:{return_code}: stderr:{text_err}")
452 f"Subprocess exited with non 0 ret code:{return_code}: stderr:{text_err}")
453
453
454 self.stdout = bg_out
454 self.stdout = bg_out
455 self.stderr = bg_err
455 self.stderr = bg_err
456 self.inputstream = input_stream
456 self.inputstream = input_stream
457
457
458 def __str__(self):
458 def __str__(self):
459 proc = getattr(self, 'process', 'NO_PROCESS')
459 proc = getattr(self, 'process', 'NO_PROCESS')
460 return f'SubprocessIOChunker: {proc}'
460 return f'SubprocessIOChunker: {proc}'
461
461
462 def __iter__(self):
462 def __iter__(self):
463 return self
463 return self
464
464
465 def __next__(self):
465 def __next__(self):
466 # Note: mikhail: We need to be sure that we are checking the return
466 # Note: mikhail: We need to be sure that we are checking the return
467 # code after the stdout stream is closed. Some processes, e.g. git
467 # code after the stdout stream is closed. Some processes, e.g. git
468 # are doing some magic in between closing stdout and terminating the
468 # are doing some magic in between closing stdout and terminating the
469 # process and, as a result, we are not getting return code on "slow"
469 # process and, as a result, we are not getting return code on "slow"
470 # systems.
470 # systems.
471 result = None
471 result = None
472 stop_iteration = None
472 stop_iteration = None
473 try:
473 try:
474 result = next(self.stdout)
474 result = next(self.stdout)
475 except StopIteration as e:
475 except StopIteration as e:
476 stop_iteration = e
476 stop_iteration = e
477
477
478 if self.process:
478 if self.process:
479 return_code = self.process.poll()
479 return_code = self.process.poll()
480 ret_code_fail = return_code is not None and return_code != 0
480 ret_code_fail = return_code is not None and return_code != 0
481 if ret_code_fail and self._fail_on_return_code:
481 if ret_code_fail and self._fail_on_return_code:
482 self.stop_streams()
482 self.stop_streams()
483 err = self.get_stderr()
483 err = self.get_stderr()
484 raise OSError(
484 raise OSError(
485 f"Subprocess exited (exit_code:{return_code}) due to an error during iteration:\n{err}")
485 f"Subprocess exited (exit_code:{return_code}) due to an error during iteration:\n{err}")
486
486
487 if stop_iteration:
487 if stop_iteration:
488 raise stop_iteration
488 raise stop_iteration
489 return result
489 return result
490
490
491 def throw(self, exc_type, value=None, traceback=None):
491 def throw(self, exc_type, value=None, traceback=None):
492 if self.stdout.length or not self.stdout.done_reading:
492 if self.stdout.length or not self.stdout.done_reading:
493 raise exc_type(value)
493 raise exc_type(value)
494
494
495 def close(self):
495 def close(self):
496 if self._closed:
496 if self._closed:
497 return
497 return
498
498
499 try:
499 try:
500 self.process.terminate()
500 self.process.terminate()
501 except Exception:
501 except Exception:
502 pass
502 pass
503 if self._close_input_fd:
503 if self._close_input_fd:
504 os.close(self._close_input_fd)
504 os.close(self._close_input_fd)
505 try:
505 try:
506 self.stdout.close()
506 self.stdout.close()
507 except Exception:
507 except Exception:
508 pass
508 pass
509 try:
509 try:
510 self.stderr.close()
510 self.stderr.close()
511 except Exception:
511 except Exception:
512 pass
512 pass
513 try:
513 try:
514 os.close(self.inputstream)
514 os.close(self.inputstream)
515 except Exception:
515 except Exception:
516 pass
516 pass
517
517
518 self._closed = True
518 self._closed = True
519
519
520 def stop_streams(self):
520 def stop_streams(self):
521 getattr(self.stdout, 'stop', lambda: None)()
521 getattr(self.stdout, 'stop', lambda: None)()
522 getattr(self.stderr, 'stop', lambda: None)()
522 getattr(self.stderr, 'stop', lambda: None)()
523
523
524 def get_stdout(self):
524 def get_stdout(self):
525 if self._stdout:
525 if self._stdout:
526 return self._stdout
526 return self._stdout
527 else:
527 else:
528 return b''.join(self.stdout)
528 return b''.join(self.stdout)
529
529
530 def get_stderr(self):
530 def get_stderr(self):
531 if self._stderr:
531 if self._stderr:
532 return self._stderr
532 return self._stderr
533 else:
533 else:
534 return b''.join(self.stderr)
534 return b''.join(self.stderr)
535
535
536
536
537 def run_command(arguments, env=None):
537 def run_command(arguments, env=None):
538 """
538 """
539 Run the specified command and return the stdout.
539 Run the specified command and return the stdout.
540
540
541 :param arguments: sequence of program arguments (including the program name)
541 :param arguments: sequence of program arguments (including the program name)
542 :type arguments: list[str]
542 :type arguments: list[str]
543 """
543 """
544
544
545 cmd = arguments
545 cmd = arguments
546 log.debug('Running subprocessio command %s', cmd)
546 log.debug('Running subprocessio command %s', cmd)
547 proc = None
547 proc = None
548 try:
548 try:
549 _opts = {'shell': False, 'fail_on_stderr': False}
549 _opts = {'shell': False, 'fail_on_stderr': False}
550 if env:
550 if env:
551 _opts.update({'env': env})
551 _opts.update({'env': env})
552 proc = SubprocessIOChunker(cmd, **_opts)
552 proc = SubprocessIOChunker(cmd, **_opts)
553 return b''.join(proc), b''.join(proc.stderr)
553 return b''.join(proc), b''.join(proc.stderr)
554 except OSError as err:
554 except OSError as err:
555 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
555 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
556 tb_err = ("Couldn't run subprocessio command (%s).\n"
556 tb_err = ("Couldn't run subprocessio command (%s).\n"
557 "Original error was:%s\n" % (cmd, err))
557 "Original error was:%s\n" % (cmd, err))
558 log.exception(tb_err)
558 log.exception(tb_err)
559 raise Exception(tb_err)
559 raise Exception(tb_err)
560 finally:
560 finally:
561 if proc:
561 if proc:
562 proc.close()
562 proc.close()
563
563
General Comments 0
You need to be logged in to leave comments. Login now