##// END OF EJS Templates
lint: more autofixes
super-admin -
r1153:bc27ecc0 default
parent child Browse files
Show More
@@ -1,175 +1,175 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 19 import shutil
20 20 import logging
21 21 from collections import OrderedDict
22 22
23 23 log = logging.getLogger(__name__)
24 24
25 25
26 26 class OidHandler:
27 27
28 28 def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
29 29 obj_verify_href=None):
30 30 self.current_store = store
31 31 self.repo_name = repo_name
32 32 self.auth = auth
33 33 self.oid = oid
34 34 self.obj_size = obj_size
35 35 self.obj_data = obj_data
36 36 self.obj_href = obj_href
37 37 self.obj_verify_href = obj_verify_href
38 38
39 39 def get_store(self, mode=None):
40 40 return self.current_store
41 41
42 42 def get_auth(self):
43 43 """returns auth header for re-use in upload/download"""
44 44 return " ".join(self.auth)
45 45
46 46 def download(self):
47 47
48 48 store = self.get_store()
49 49 response = None
50 50 has_errors = None
51 51
52 52 if not store.has_oid():
53 53 # error reply back to client that something is wrong with dl
54 54 err_msg = f'object: {store.oid} does not exist in store'
55 55 has_errors = OrderedDict(
56 56 error=OrderedDict(
57 57 code=404,
58 58 message=err_msg
59 59 )
60 60 )
61 61
62 62 download_action = OrderedDict(
63 63 href=self.obj_href,
64 64 header=OrderedDict([("Authorization", self.get_auth())])
65 65 )
66 66 if not has_errors:
67 67 response = OrderedDict(download=download_action)
68 68 return response, has_errors
69 69
70 70 def upload(self, skip_existing=True):
71 71 """
72 72 Write upload action for git-lfs server
73 73 """
74 74
75 75 store = self.get_store()
76 76 response = None
77 77 has_errors = None
78 78
79 79 # verify if we have the OID before, if we do, reply with empty
80 80 if store.has_oid():
81 81 log.debug('LFS: store already has oid %s', store.oid)
82 82
83 83 # validate size
84 84 store_size = store.size_oid()
85 85 size_match = store_size == self.obj_size
86 86 if not size_match:
87 87 log.warning(
88 88 'LFS: size mismatch for oid:%s, in store:%s expected: %s',
89 89 self.oid, store_size, self.obj_size)
90 90 elif skip_existing:
91 91 log.debug('LFS: skipping further action as oid is existing')
92 92 return response, has_errors
93 93
94 94 chunked = ("Transfer-Encoding", "chunked")
95 95 upload_action = OrderedDict(
96 96 href=self.obj_href,
97 97 header=OrderedDict([("Authorization", self.get_auth()), chunked])
98 98 )
99 99 if not has_errors:
100 100 response = OrderedDict(upload=upload_action)
101 101 # if specified in handler, return the verification endpoint
102 102 if self.obj_verify_href:
103 103 verify_action = OrderedDict(
104 104 href=self.obj_verify_href,
105 105 header=OrderedDict([("Authorization", self.get_auth())])
106 106 )
107 107 response['verify'] = verify_action
108 108 return response, has_errors
109 109
110 110 def exec_operation(self, operation, *args, **kwargs):
111 111 handler = getattr(self, operation)
112 112 log.debug('LFS: handling request using %s handler', handler)
113 113 return handler(*args, **kwargs)
114 114
115 115
116 116 class LFSOidStore:
117 117
118 118 def __init__(self, oid, repo, store_location=None):
119 119 self.oid = oid
120 120 self.repo = repo
121 121 self.store_path = store_location or self.get_default_store()
122 122 self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
123 123 self.oid_path = os.path.join(self.store_path, oid)
124 124 self.fd = None
125 125
126 126 def get_engine(self, mode):
127 127 """
128 128 engine = .get_engine(mode='wb')
129 129 with engine as f:
130 130 f.write('...')
131 131 """
132 132
133 class StoreEngine(object):
133 class StoreEngine:
134 134 def __init__(self, mode, store_path, oid_path, tmp_oid_path):
135 135 self.mode = mode
136 136 self.store_path = store_path
137 137 self.oid_path = oid_path
138 138 self.tmp_oid_path = tmp_oid_path
139 139
140 140 def __enter__(self):
141 141 if not os.path.isdir(self.store_path):
142 142 os.makedirs(self.store_path)
143 143
144 144 # TODO(marcink): maybe write metadata here with size/oid ?
145 145 fd = open(self.tmp_oid_path, self.mode)
146 146 self.fd = fd
147 147 return fd
148 148
149 149 def __exit__(self, exc_type, exc_value, traceback):
150 150 # close tmp file, and rename to final destination
151 151 self.fd.close()
152 152 shutil.move(self.tmp_oid_path, self.oid_path)
153 153
154 154 return StoreEngine(
155 155 mode, self.store_path, self.oid_path, self.tmp_oid_path)
156 156
157 157 def get_default_store(self):
158 158 """
159 159 Default store, consistent with defaults of Mercurial large files store
160 160 which is /home/username/.cache/largefiles
161 161 """
162 162 user_home = os.path.expanduser("~")
163 163 return os.path.join(user_home, '.cache', 'lfs-store')
164 164
165 165 def has_oid(self):
166 166 return os.path.exists(os.path.join(self.store_path, self.oid))
167 167
168 168 def size_oid(self):
169 169 size = -1
170 170
171 171 if self.has_oid():
172 172 oid = os.path.join(self.store_path, self.oid)
173 173 size = os.stat(oid).st_size
174 174
175 175 return size
@@ -1,154 +1,154 b''
1 1 import re
2 2 import random
3 3 from collections import deque
4 4 from datetime import timedelta
5 5 from repoze.lru import lru_cache
6 6
7 7 from .timer import Timer
8 8
9 9 TAG_INVALID_CHARS_RE = re.compile(
10 10 r"[^\w\d_\-:/\.]",
11 11 #re.UNICODE
12 12 )
13 13 TAG_INVALID_CHARS_SUBS = "_"
14 14
15 15 # we save and expose methods called by statsd for discovery
16 16 buckets_dict = {
17 17
18 18 }
19 19
20 20
21 21 @lru_cache(maxsize=500)
22 22 def _normalize_tags_with_cache(tag_list):
23 23 return [TAG_INVALID_CHARS_RE.sub(TAG_INVALID_CHARS_SUBS, tag) for tag in tag_list]
24 24
25 25
26 26 def normalize_tags(tag_list):
27 27 # We have to turn our input tag list into a non-mutable tuple for it to
28 28 # be hashable (and thus usable) by the @lru_cache decorator.
29 29 return _normalize_tags_with_cache(tuple(tag_list))
30 30
31 31
32 32 class StatsClientBase:
33 33 """A Base class for various statsd clients."""
34 34
35 35 def close(self):
36 36 """Used to close and clean up any underlying resources."""
37 37 raise NotImplementedError()
38 38
39 39 def _send(self):
40 40 raise NotImplementedError()
41 41
42 42 def pipeline(self):
43 43 raise NotImplementedError()
44 44
45 45 def timer(self, stat, rate=1, tags=None, auto_send=True):
46 46 """
47 47 statsd = StatsdClient.statsd
48 48 with statsd.timer('bucket_name', auto_send=True) as tmr:
49 49 # This block will be timed.
50 50 for i in range(0, 100000):
51 51 i ** 2
52 52 # you can access time here...
53 53 elapsed_ms = tmr.ms
54 54 """
55 55 return Timer(self, stat, rate, tags, auto_send=auto_send)
56 56
57 57 def timing(self, stat, delta, rate=1, tags=None, use_decimals=True):
58 58 """
59 59 Send new timing information.
60 60
61 61 `delta` can be either a number of milliseconds or a timedelta.
62 62 """
63 63 if isinstance(delta, timedelta):
64 64 # Convert timedelta to number of milliseconds.
65 65 delta = delta.total_seconds() * 1000.
66 66 if use_decimals:
67 67 fmt = '%0.6f|ms'
68 68 else:
69 69 fmt = '%s|ms'
70 70 self._send_stat(stat, fmt % delta, rate, tags)
71 71
72 72 def incr(self, stat, count=1, rate=1, tags=None):
73 73 """Increment a stat by `count`."""
74 self._send_stat(stat, '%s|c' % count, rate, tags)
74 self._send_stat(stat, f'{count}|c', rate, tags)
75 75
76 76 def decr(self, stat, count=1, rate=1, tags=None):
77 77 """Decrement a stat by `count`."""
78 78 self.incr(stat, -count, rate, tags)
79 79
80 80 def gauge(self, stat, value, rate=1, delta=False, tags=None):
81 81 """Set a gauge value."""
82 82 if value < 0 and not delta:
83 83 if rate < 1:
84 84 if random.random() > rate:
85 85 return
86 86 with self.pipeline() as pipe:
87 87 pipe._send_stat(stat, '0|g', 1)
88 pipe._send_stat(stat, '%s|g' % value, 1)
88 pipe._send_stat(stat, f'{value}|g', 1)
89 89 else:
90 90 prefix = '+' if delta and value >= 0 else ''
91 self._send_stat(stat, '%s%s|g' % (prefix, value), rate, tags)
91 self._send_stat(stat, f'{prefix}{value}|g', rate, tags)
92 92
93 93 def set(self, stat, value, rate=1):
94 94 """Set a set value."""
95 self._send_stat(stat, '%s|s' % value, rate)
95 self._send_stat(stat, f'{value}|s', rate)
96 96
97 97 def histogram(self, stat, value, rate=1, tags=None):
98 98 """Set a histogram"""
99 self._send_stat(stat, '%s|h' % value, rate, tags)
99 self._send_stat(stat, f'{value}|h', rate, tags)
100 100
101 101 def _send_stat(self, stat, value, rate, tags=None):
102 102 self._after(self._prepare(stat, value, rate, tags))
103 103
104 104 def _prepare(self, stat, value, rate, tags=None):
105 105 global buckets_dict
106 106 buckets_dict[stat] = 1
107 107
108 108 if rate < 1:
109 109 if random.random() > rate:
110 110 return
111 value = '%s|@%s' % (value, rate)
111 value = f'{value}|@{rate}'
112 112
113 113 if self._prefix:
114 stat = '%s.%s' % (self._prefix, stat)
114 stat = f'{self._prefix}.{stat}'
115 115
116 116 res = '%s:%s%s' % (
117 117 stat,
118 118 value,
119 119 ("|#" + ",".join(normalize_tags(tags))) if tags else "",
120 120 )
121 121 return res
122 122
123 123 def _after(self, data):
124 124 if data:
125 125 self._send(data)
126 126
127 127
128 128 class PipelineBase(StatsClientBase):
129 129
130 130 def __init__(self, client):
131 131 self._client = client
132 132 self._prefix = client._prefix
133 133 self._stats = deque()
134 134
135 135 def _send(self):
136 136 raise NotImplementedError()
137 137
138 138 def _after(self, data):
139 139 if data is not None:
140 140 self._stats.append(data)
141 141
142 142 def __enter__(self):
143 143 return self
144 144
145 145 def __exit__(self, typ, value, tb):
146 146 self.send()
147 147
148 148 def send(self):
149 149 if not self._stats:
150 150 return
151 151 self._send()
152 152
153 153 def pipeline(self):
154 154 return self.__class__(self)
@@ -1,417 +1,417 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 """Handles the Git smart protocol."""
19 19
20 20 import os
21 21 import socket
22 22 import logging
23 23
24 24 import dulwich.protocol
25 25 from dulwich.protocol import CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K
26 26 from webob import Request, Response, exc
27 27
28 28 from vcsserver.lib.rc_json import json
29 29 from vcsserver import hooks, subprocessio
30 30 from vcsserver.str_utils import ascii_bytes
31 31
32 32
33 33 log = logging.getLogger(__name__)
34 34
35 35
36 36 class FileWrapper:
37 37 """File wrapper that ensures how much data is read from it."""
38 38
39 39 def __init__(self, fd, content_length):
40 40 self.fd = fd
41 41 self.content_length = content_length
42 42 self.remain = content_length
43 43
44 44 def read(self, size):
45 45 if size <= self.remain:
46 46 try:
47 47 data = self.fd.read(size)
48 48 except socket.error:
49 49 raise IOError(self)
50 50 self.remain -= size
51 51 elif self.remain:
52 52 data = self.fd.read(self.remain)
53 53 self.remain = 0
54 54 else:
55 55 data = None
56 56 return data
57 57
58 58 def __repr__(self):
59 59 return '<FileWrapper {} len: {}, read: {}>'.format(
60 60 self.fd, self.content_length, self.content_length - self.remain
61 61 )
62 62
63 63
64 64 class GitRepository:
65 65 """WSGI app for handling Git smart protocol endpoints."""
66 66
67 67 git_folder_signature = frozenset(('config', 'head', 'info', 'objects', 'refs'))
68 68 commands = frozenset(('git-upload-pack', 'git-receive-pack'))
69 69 valid_accepts = frozenset(f'application/x-{c}-result' for c in commands)
70 70
71 71 # The last bytes are the SHA1 of the first 12 bytes.
72 72 EMPTY_PACK = (
73 73 b'PACK\x00\x00\x00\x02\x00\x00\x00\x00\x02\x9d\x08' +
74 74 b'\x82;\xd8\xa8\xea\xb5\x10\xadj\xc7\\\x82<\xfd>\xd3\x1e'
75 75 )
76 76 FLUSH_PACKET = b"0000"
77 77
78 78 SIDE_BAND_CAPS = frozenset((CAPABILITY_SIDE_BAND, CAPABILITY_SIDE_BAND_64K))
79 79
80 80 def __init__(self, repo_name, content_path, git_path, update_server_info, extras):
81 81 files = frozenset(f.lower() for f in os.listdir(content_path))
82 82 valid_dir_signature = self.git_folder_signature.issubset(files)
83 83
84 84 if not valid_dir_signature:
85 85 raise OSError(f'{content_path} missing git signature')
86 86
87 87 self.content_path = content_path
88 88 self.repo_name = repo_name
89 89 self.extras = extras
90 90 self.git_path = git_path
91 91 self.update_server_info = update_server_info
92 92
93 93 def _get_fixedpath(self, path):
94 94 """
95 95 Small fix for repo_path
96 96
97 97 :param path:
98 98 """
99 99 path = path.split(self.repo_name, 1)[-1]
100 100 if path.startswith('.git'):
101 101 # for bare repos we still get the .git prefix inside, we skip it
102 102 # here, and remove from the service command
103 103 path = path[4:]
104 104
105 105 return path.strip('/')
106 106
107 107 def inforefs(self, request, unused_environ):
108 108 """
109 109 WSGI Response producer for HTTP GET Git Smart
110 110 HTTP /info/refs request.
111 111 """
112 112
113 113 git_command = request.GET.get('service')
114 114 if git_command not in self.commands:
115 115 log.debug('command %s not allowed', git_command)
116 116 return exc.HTTPForbidden()
117 117
118 118 # please, resist the urge to add '\n' to git capture and increment
119 119 # line count by 1.
120 120 # by git docs: Documentation/technical/http-protocol.txt#L214 \n is
121 121 # a part of protocol.
122 122 # The code in Git client not only does NOT need '\n', but actually
123 123 # blows up if you sprinkle "flush" (0000) as "0001\n".
124 124 # It reads binary, per number of bytes specified.
125 125 # if you do add '\n' as part of data, count it.
126 server_advert = '# service=%s\n' % git_command
126 server_advert = f'# service={git_command}\n'
127 127 packet_len = hex(len(server_advert) + 4)[2:].rjust(4, '0').lower()
128 128 try:
129 129 gitenv = dict(os.environ)
130 130 # forget all configs
131 131 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
132 132 command = [self.git_path, git_command[4:], '--stateless-rpc',
133 133 '--advertise-refs', self.content_path]
134 134 out = subprocessio.SubprocessIOChunker(
135 135 command,
136 136 env=gitenv,
137 137 starting_values=[ascii_bytes(packet_len + server_advert) + self.FLUSH_PACKET],
138 138 shell=False
139 139 )
140 140 except OSError:
141 141 log.exception('Error processing command')
142 142 raise exc.HTTPExpectationFailed()
143 143
144 144 resp = Response()
145 145 resp.content_type = f'application/x-{git_command}-advertisement'
146 146 resp.charset = None
147 147 resp.app_iter = out
148 148
149 149 return resp
150 150
151 151 def _get_want_capabilities(self, request):
152 152 """Read the capabilities found in the first want line of the request."""
153 153 pos = request.body_file_seekable.tell()
154 154 first_line = request.body_file_seekable.readline()
155 155 request.body_file_seekable.seek(pos)
156 156
157 157 return frozenset(
158 158 dulwich.protocol.extract_want_line_capabilities(first_line)[1])
159 159
160 160 def _build_failed_pre_pull_response(self, capabilities, pre_pull_messages):
161 161 """
162 162 Construct a response with an empty PACK file.
163 163
164 164 We use an empty PACK file, as that would trigger the failure of the pull
165 165 or clone command.
166 166
167 167 We also print in the error output a message explaining why the command
168 168 was aborted.
169 169
170 170 If additionally, the user is accepting messages we send them the output
171 171 of the pre-pull hook.
172 172
173 173 Note that for clients not supporting side-band we just send them the
174 174 emtpy PACK file.
175 175 """
176 176
177 177 if self.SIDE_BAND_CAPS.intersection(capabilities):
178 178 response = []
179 179 proto = dulwich.protocol.Protocol(None, response.append)
180 180 proto.write_pkt_line(dulwich.protocol.NAK_LINE)
181 181
182 182 self._write_sideband_to_proto(proto, ascii_bytes(pre_pull_messages, allow_bytes=True), capabilities)
183 183 # N.B.(skreft): Do not change the sideband channel to 3, as that
184 184 # produces a fatal error in the client:
185 185 # fatal: error in sideband demultiplexer
186 186 proto.write_sideband(
187 187 dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS,
188 188 ascii_bytes('Pre pull hook failed: aborting\n', allow_bytes=True))
189 189 proto.write_sideband(
190 190 dulwich.protocol.SIDE_BAND_CHANNEL_DATA,
191 191 ascii_bytes(self.EMPTY_PACK, allow_bytes=True))
192 192
193 193 # writes b"0000" as default
194 194 proto.write_pkt_line(None)
195 195
196 196 return response
197 197 else:
198 198 return [ascii_bytes(self.EMPTY_PACK, allow_bytes=True)]
199 199
200 200 def _build_post_pull_response(self, response, capabilities, start_message, end_message):
201 201 """
202 202 Given a list response we inject the post-pull messages.
203 203
204 204 We only inject the messages if the client supports sideband, and the
205 205 response has the format:
206 206 0008NAK\n...0000
207 207
208 208 Note that we do not check the no-progress capability as by default, git
209 209 sends it, which effectively would block all messages.
210 210 """
211 211
212 212 if not self.SIDE_BAND_CAPS.intersection(capabilities):
213 213 return response
214 214
215 215 if not start_message and not end_message:
216 216 return response
217 217
218 218 try:
219 219 iter(response)
220 220 # iterator probably will work, we continue
221 221 except TypeError:
222 222 raise TypeError(f'response must be an iterator: got {type(response)}')
223 223 if isinstance(response, (list, tuple)):
224 224 raise TypeError(f'response must be an iterator: got {type(response)}')
225 225
226 226 def injected_response():
227 227
228 228 do_loop = 1
229 229 header_injected = 0
230 230 next_item = None
231 231 has_item = False
232 232 item = b''
233 233
234 234 while do_loop:
235 235
236 236 try:
237 237 next_item = next(response)
238 238 except StopIteration:
239 239 do_loop = 0
240 240
241 241 if has_item:
242 242 # last item ! alter it now
243 243 if do_loop == 0 and item.endswith(self.FLUSH_PACKET):
244 244 new_response = [item[:-4]]
245 245 new_response.extend(self._get_messages(end_message, capabilities))
246 246 new_response.append(self.FLUSH_PACKET)
247 247 item = b''.join(new_response)
248 248
249 249 yield item
250 250
251 251 has_item = True
252 252 item = next_item
253 253
254 254 # alter item if it's the initial chunk
255 255 if not header_injected and item.startswith(b'0008NAK\n'):
256 256 new_response = [b'0008NAK\n']
257 257 new_response.extend(self._get_messages(start_message, capabilities))
258 258 new_response.append(item[8:])
259 259 item = b''.join(new_response)
260 260 header_injected = 1
261 261
262 262 return injected_response()
263 263
264 264 def _write_sideband_to_proto(self, proto, data, capabilities):
265 265 """
266 266 Write the data to the proto's sideband number 2 == SIDE_BAND_CHANNEL_PROGRESS
267 267
268 268 We do not use dulwich's write_sideband directly as it only supports
269 269 side-band-64k.
270 270 """
271 271 if not data:
272 272 return
273 273
274 274 # N.B.(skreft): The values below are explained in the pack protocol
275 275 # documentation, section Packfile Data.
276 276 # https://github.com/git/git/blob/master/Documentation/technical/pack-protocol.txt
277 277 if CAPABILITY_SIDE_BAND_64K in capabilities:
278 278 chunk_size = 65515
279 279 elif CAPABILITY_SIDE_BAND in capabilities:
280 280 chunk_size = 995
281 281 else:
282 282 return
283 283
284 284 chunker = (data[i:i + chunk_size] for i in range(0, len(data), chunk_size))
285 285
286 286 for chunk in chunker:
287 287 proto.write_sideband(dulwich.protocol.SIDE_BAND_CHANNEL_PROGRESS, ascii_bytes(chunk, allow_bytes=True))
288 288
289 289 def _get_messages(self, data, capabilities):
290 290 """Return a list with packets for sending data in sideband number 2."""
291 291 response = []
292 292 proto = dulwich.protocol.Protocol(None, response.append)
293 293
294 294 self._write_sideband_to_proto(proto, data, capabilities)
295 295
296 296 return response
297 297
298 298 def backend(self, request, environ):
299 299 """
300 300 WSGI Response producer for HTTP POST Git Smart HTTP requests.
301 301 Reads commands and data from HTTP POST's body.
302 302 returns an iterator obj with contents of git command's
303 303 response to stdout
304 304 """
305 305 # TODO(skreft): think how we could detect an HTTPLockedException, as
306 306 # we probably want to have the same mechanism used by mercurial and
307 307 # simplevcs.
308 308 # For that we would need to parse the output of the command looking for
309 309 # some signs of the HTTPLockedError, parse the data and reraise it in
310 310 # pygrack. However, that would interfere with the streaming.
311 311 #
312 312 # Now the output of a blocked push is:
313 313 # Pushing to http://test_regular:test12@127.0.0.1:5001/vcs_test_git
314 314 # POST git-receive-pack (1047 bytes)
315 315 # remote: ERROR: Repository `vcs_test_git` locked by user `test_admin`. Reason:`lock_auto`
316 316 # To http://test_regular:test12@127.0.0.1:5001/vcs_test_git
317 317 # ! [remote rejected] master -> master (pre-receive hook declined)
318 318 # error: failed to push some refs to 'http://test_regular:test12@127.0.0.1:5001/vcs_test_git'
319 319
320 320 git_command = self._get_fixedpath(request.path_info)
321 321 if git_command not in self.commands:
322 322 log.debug('command %s not allowed', git_command)
323 323 return exc.HTTPForbidden()
324 324
325 325 capabilities = None
326 326 if git_command == 'git-upload-pack':
327 327 capabilities = self._get_want_capabilities(request)
328 328
329 329 if 'CONTENT_LENGTH' in environ:
330 330 inputstream = FileWrapper(request.body_file_seekable,
331 331 request.content_length)
332 332 else:
333 333 inputstream = request.body_file_seekable
334 334
335 335 resp = Response()
336 336 resp.content_type = f'application/x-{git_command}-result'
337 337 resp.charset = None
338 338
339 339 pre_pull_messages = ''
340 340 # Upload-pack == clone
341 341 if git_command == 'git-upload-pack':
342 342 hook_response = hooks.git_pre_pull(self.extras)
343 343 if hook_response.status != 0:
344 344 pre_pull_messages = hook_response.output
345 345 resp.app_iter = self._build_failed_pre_pull_response(
346 346 capabilities, pre_pull_messages)
347 347 return resp
348 348
349 349 gitenv = dict(os.environ)
350 350 # forget all configs
351 351 gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
352 352 gitenv['RC_SCM_DATA'] = json.dumps(self.extras)
353 353 cmd = [self.git_path, git_command[4:], '--stateless-rpc',
354 354 self.content_path]
355 355 log.debug('handling cmd %s', cmd)
356 356
357 357 out = subprocessio.SubprocessIOChunker(
358 358 cmd,
359 359 input_stream=inputstream,
360 360 env=gitenv,
361 361 cwd=self.content_path,
362 362 shell=False,
363 363 fail_on_stderr=False,
364 364 fail_on_return_code=False
365 365 )
366 366
367 367 if self.update_server_info and git_command == 'git-receive-pack':
368 368 # We need to fully consume the iterator here, as the
369 369 # update-server-info command needs to be run after the push.
370 370 out = list(out)
371 371
372 372 # Updating refs manually after each push.
373 373 # This is required as some clients are exposing Git repos internally
374 374 # with the dumb protocol.
375 375 cmd = [self.git_path, 'update-server-info']
376 376 log.debug('handling cmd %s', cmd)
377 377 output = subprocessio.SubprocessIOChunker(
378 378 cmd,
379 379 input_stream=inputstream,
380 380 env=gitenv,
381 381 cwd=self.content_path,
382 382 shell=False,
383 383 fail_on_stderr=False,
384 384 fail_on_return_code=False
385 385 )
386 386 # Consume all the output so the subprocess finishes
387 387 for _ in output:
388 388 pass
389 389
390 390 # Upload-pack == clone
391 391 if git_command == 'git-upload-pack':
392 392 hook_response = hooks.git_post_pull(self.extras)
393 393 post_pull_messages = hook_response.output
394 394 resp.app_iter = self._build_post_pull_response(out, capabilities, pre_pull_messages, post_pull_messages)
395 395 else:
396 396 resp.app_iter = out
397 397
398 398 return resp
399 399
400 400 def __call__(self, environ, start_response):
401 401 request = Request(environ)
402 402 _path = self._get_fixedpath(request.path_info)
403 403 if _path.startswith('info/refs'):
404 404 app = self.inforefs
405 405 else:
406 406 app = self.backend
407 407
408 408 try:
409 409 resp = app(request, environ)
410 410 except exc.HTTPException as error:
411 411 log.exception('HTTP Error')
412 412 resp = error
413 413 except Exception:
414 414 log.exception('Unknown error')
415 415 resp = exc.HTTPInternalServerError()
416 416
417 417 return resp(environ, start_response)
@@ -1,206 +1,206 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 19 import sys
20 20 import stat
21 21 import pytest
22 22 import vcsserver
23 23 import tempfile
24 24 from vcsserver import hook_utils
25 25 from vcsserver.tests.fixture import no_newline_id_generator
26 26 from vcsserver.str_utils import safe_bytes, safe_str
27 27 from vcsserver.utils import AttributeDict
28 28
29 29
30 30 class TestCheckRhodecodeHook:
31 31
32 32 def test_returns_false_when_hook_file_is_wrong_found(self, tmpdir):
33 33 hook = os.path.join(str(tmpdir), 'fake_hook_file.py')
34 34 with open(hook, 'wb') as f:
35 35 f.write(b'dummy test')
36 36 result = hook_utils.check_rhodecode_hook(hook)
37 37 assert result is False
38 38
39 39 def test_returns_true_when_no_hook_file_found(self, tmpdir):
40 40 hook = os.path.join(str(tmpdir), 'fake_hook_file_not_existing.py')
41 41 result = hook_utils.check_rhodecode_hook(hook)
42 42 assert result
43 43
44 44 @pytest.mark.parametrize("file_content, expected_result", [
45 45 ("RC_HOOK_VER = '3.3.3'\n", True),
46 46 ("RC_HOOK = '3.3.3'\n", False),
47 47 ], ids=no_newline_id_generator)
48 48 def test_signatures(self, file_content, expected_result, tmpdir):
49 49 hook = os.path.join(str(tmpdir), 'fake_hook_file_1.py')
50 50 with open(hook, 'wb') as f:
51 51 f.write(safe_bytes(file_content))
52 52
53 53 result = hook_utils.check_rhodecode_hook(hook)
54 54
55 55 assert result is expected_result
56 56
57 57
58 58 class BaseInstallHooks:
59 59 HOOK_FILES = ()
60 60
61 61 def _check_hook_file_mode(self, file_path):
62 assert os.path.exists(file_path), 'path %s missing' % file_path
62 assert os.path.exists(file_path), f'path {file_path} missing'
63 63 stat_info = os.stat(file_path)
64 64
65 65 file_mode = stat.S_IMODE(stat_info.st_mode)
66 66 expected_mode = int('755', 8)
67 67 assert expected_mode == file_mode
68 68
69 69 def _check_hook_file_content(self, file_path, executable):
70 70 executable = executable or sys.executable
71 71 with open(file_path, 'rt') as hook_file:
72 72 content = hook_file.read()
73 73
74 74 expected_env = '#!{}'.format(executable)
75 75 expected_rc_version = "\nRC_HOOK_VER = '{}'\n".format(vcsserver.__version__)
76 76 assert content.strip().startswith(expected_env)
77 77 assert expected_rc_version in content
78 78
79 79 def _create_fake_hook(self, file_path, content):
80 80 with open(file_path, 'w') as hook_file:
81 81 hook_file.write(content)
82 82
83 83 def create_dummy_repo(self, repo_type):
84 84 tmpdir = tempfile.mkdtemp()
85 85 repo = AttributeDict()
86 86 if repo_type == 'git':
87 87 repo.path = os.path.join(tmpdir, 'test_git_hooks_installation_repo')
88 88 os.makedirs(repo.path)
89 89 os.makedirs(os.path.join(repo.path, 'hooks'))
90 90 repo.bare = True
91 91
92 92 elif repo_type == 'svn':
93 93 repo.path = os.path.join(tmpdir, 'test_svn_hooks_installation_repo')
94 94 os.makedirs(repo.path)
95 95 os.makedirs(os.path.join(repo.path, 'hooks'))
96 96
97 97 return repo
98 98
99 99 def check_hooks(self, repo_path, repo_bare=True):
100 100 for file_name in self.HOOK_FILES:
101 101 if repo_bare:
102 102 file_path = os.path.join(repo_path, 'hooks', file_name)
103 103 else:
104 104 file_path = os.path.join(repo_path, '.git', 'hooks', file_name)
105 105 self._check_hook_file_mode(file_path)
106 106 self._check_hook_file_content(file_path, sys.executable)
107 107
108 108
109 109 class TestInstallGitHooks(BaseInstallHooks):
110 110 HOOK_FILES = ('pre-receive', 'post-receive')
111 111
112 112 def test_hooks_are_installed(self):
113 113 repo = self.create_dummy_repo('git')
114 114 result = hook_utils.install_git_hooks(repo.path, repo.bare)
115 115 assert result
116 116 self.check_hooks(repo.path, repo.bare)
117 117
118 118 def test_hooks_are_replaced(self):
119 119 repo = self.create_dummy_repo('git')
120 120 hooks_path = os.path.join(repo.path, 'hooks')
121 121 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
122 122 self._create_fake_hook(
123 123 file_path, content="RC_HOOK_VER = 'abcde'\n")
124 124
125 125 result = hook_utils.install_git_hooks(repo.path, repo.bare)
126 126 assert result
127 127 self.check_hooks(repo.path, repo.bare)
128 128
129 129 def test_non_rc_hooks_are_not_replaced(self):
130 130 repo = self.create_dummy_repo('git')
131 131 hooks_path = os.path.join(repo.path, 'hooks')
132 132 non_rc_content = 'echo "non rc hook"\n'
133 133 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
134 134 self._create_fake_hook(
135 135 file_path, content=non_rc_content)
136 136
137 137 result = hook_utils.install_git_hooks(repo.path, repo.bare)
138 138 assert result
139 139
140 140 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
141 141 with open(file_path, 'rt') as hook_file:
142 142 content = hook_file.read()
143 143 assert content == non_rc_content
144 144
145 145 def test_non_rc_hooks_are_replaced_with_force_flag(self):
146 146 repo = self.create_dummy_repo('git')
147 147 hooks_path = os.path.join(repo.path, 'hooks')
148 148 non_rc_content = 'echo "non rc hook"\n'
149 149 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
150 150 self._create_fake_hook(
151 151 file_path, content=non_rc_content)
152 152
153 153 result = hook_utils.install_git_hooks(
154 154 repo.path, repo.bare, force_create=True)
155 155 assert result
156 156 self.check_hooks(repo.path, repo.bare)
157 157
158 158
159 159 class TestInstallSvnHooks(BaseInstallHooks):
160 160 HOOK_FILES = ('pre-commit', 'post-commit')
161 161
162 162 def test_hooks_are_installed(self):
163 163 repo = self.create_dummy_repo('svn')
164 164 result = hook_utils.install_svn_hooks(repo.path)
165 165 assert result
166 166 self.check_hooks(repo.path)
167 167
168 168 def test_hooks_are_replaced(self):
169 169 repo = self.create_dummy_repo('svn')
170 170 hooks_path = os.path.join(repo.path, 'hooks')
171 171 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
172 172 self._create_fake_hook(
173 173 file_path, content="RC_HOOK_VER = 'abcde'\n")
174 174
175 175 result = hook_utils.install_svn_hooks(repo.path)
176 176 assert result
177 177 self.check_hooks(repo.path)
178 178
179 179 def test_non_rc_hooks_are_not_replaced(self):
180 180 repo = self.create_dummy_repo('svn')
181 181 hooks_path = os.path.join(repo.path, 'hooks')
182 182 non_rc_content = 'echo "non rc hook"\n'
183 183 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
184 184 self._create_fake_hook(
185 185 file_path, content=non_rc_content)
186 186
187 187 result = hook_utils.install_svn_hooks(repo.path)
188 188 assert result
189 189
190 190 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
191 191 with open(file_path, 'rt') as hook_file:
192 192 content = hook_file.read()
193 193 assert content == non_rc_content
194 194
195 195 def test_non_rc_hooks_are_replaced_with_force_flag(self):
196 196 repo = self.create_dummy_repo('svn')
197 197 hooks_path = os.path.join(repo.path, 'hooks')
198 198 non_rc_content = 'echo "non rc hook"\n'
199 199 for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]:
200 200 self._create_fake_hook(
201 201 file_path, content=non_rc_content)
202 202
203 203 result = hook_utils.install_svn_hooks(
204 204 repo.path, force_create=True)
205 205 assert result
206 206 self.check_hooks(repo.path, )
General Comments 0
You need to be logged in to leave comments. Login now