Show More
@@ -202,6 +202,18 b' instructions when a failure occurs, thus' | |||||
202 | Mercurial server when the bundle hosting service fails. |
|
202 | Mercurial server when the bundle hosting service fails. | |
203 |
|
203 | |||
204 |
|
204 | |||
|
205 | inline clonebundles | |||
|
206 | ------------------- | |||
|
207 | ||||
|
208 | It is possible to transmit clonebundles inline in case repositories are | |||
|
209 | accessed over SSH. This avoids having to setup an external HTTPS server | |||
|
210 | and results in the same access control as already present for the SSH setup. | |||
|
211 | ||||
|
212 | Inline clonebundles should be placed into the `.hg/bundle-cache` directory. | |||
|
213 | A clonebundle at `.hg/bundle-cache/mybundle.bundle` is referred to | |||
|
214 | in the `clonebundles.manifest` file as `peer-bundle-cache://mybundle.bundle`. | |||
|
215 | ||||
|
216 | ||||
205 | auto-generation of clone bundles |
|
217 | auto-generation of clone bundles | |
206 | -------------------------------- |
|
218 | -------------------------------- | |
207 |
|
219 |
@@ -23,7 +23,9 b' from .utils import stringutil' | |||||
23 |
|
23 | |||
24 | urlreq = util.urlreq |
|
24 | urlreq = util.urlreq | |
25 |
|
25 | |||
|
26 | BUNDLE_CACHE_DIR = b'bundle-cache' | |||
26 | CB_MANIFEST_FILE = b'clonebundles.manifest' |
|
27 | CB_MANIFEST_FILE = b'clonebundles.manifest' | |
|
28 | CLONEBUNDLESCHEME = b"peer-bundle-cache://" | |||
27 |
|
29 | |||
28 |
|
30 | |||
29 | def get_manifest(repo): |
|
31 | def get_manifest(repo): |
@@ -2834,7 +2834,7 b' def _maybeapplyclonebundle(pullop):' | |||||
2834 |
|
2834 | |||
2835 | url = entries[0][b'URL'] |
|
2835 | url = entries[0][b'URL'] | |
2836 | repo.ui.status(_(b'applying clone bundle from %s\n') % url) |
|
2836 | repo.ui.status(_(b'applying clone bundle from %s\n') % url) | |
2837 | if trypullbundlefromurl(repo.ui, repo, url): |
|
2837 | if trypullbundlefromurl(repo.ui, repo, url, remote): | |
2838 | repo.ui.status(_(b'finished applying clone bundle\n')) |
|
2838 | repo.ui.status(_(b'finished applying clone bundle\n')) | |
2839 | # Bundle failed. |
|
2839 | # Bundle failed. | |
2840 | # |
|
2840 | # | |
@@ -2855,11 +2855,22 b' def _maybeapplyclonebundle(pullop):' | |||||
2855 | ) |
|
2855 | ) | |
2856 |
|
2856 | |||
2857 |
|
2857 | |||
2858 | def trypullbundlefromurl(ui, repo, url): |
|
2858 | def inline_clone_bundle_open(ui, url, peer): | |
|
2859 | if not peer: | |||
|
2860 | raise error.Abort(_(b'no remote repository supplied for %s' % url)) | |||
|
2861 | clonebundleid = url[len(bundlecaches.CLONEBUNDLESCHEME) :] | |||
|
2862 | peerclonebundle = peer.get_inline_clone_bundle(clonebundleid) | |||
|
2863 | return util.chunkbuffer(peerclonebundle) | |||
|
2864 | ||||
|
2865 | ||||
|
2866 | def trypullbundlefromurl(ui, repo, url, peer): | |||
2859 | """Attempt to apply a bundle from a URL.""" |
|
2867 | """Attempt to apply a bundle from a URL.""" | |
2860 | with repo.lock(), repo.transaction(b'bundleurl') as tr: |
|
2868 | with repo.lock(), repo.transaction(b'bundleurl') as tr: | |
2861 | try: |
|
2869 | try: | |
2862 | fh = urlmod.open(ui, url) |
|
2870 | if url.startswith(bundlecaches.CLONEBUNDLESCHEME): | |
|
2871 | fh = inline_clone_bundle_open(ui, url, peer) | |||
|
2872 | else: | |||
|
2873 | fh = urlmod.open(ui, url) | |||
2863 | cg = readbundle(ui, fh, b'stream') |
|
2874 | cg = readbundle(ui, fh, b'stream') | |
2864 |
|
2875 | |||
2865 | if isinstance(cg, streamclone.streamcloneapplier): |
|
2876 | if isinstance(cg, streamclone.streamcloneapplier): |
@@ -1318,6 +1318,12 b' be ``$HG_HOOKTYPE=incoming`` and ``$HG_H' | |||||
1318 | changeset to tag is in ``$HG_NODE``. The name of tag is in ``$HG_TAG``. The |
|
1318 | changeset to tag is in ``$HG_NODE``. The name of tag is in ``$HG_TAG``. The | |
1319 | tag is local if ``$HG_LOCAL=1``, or in the repository if ``$HG_LOCAL=0``. |
|
1319 | tag is local if ``$HG_LOCAL=1``, or in the repository if ``$HG_LOCAL=0``. | |
1320 |
|
1320 | |||
|
1321 | ``pretransmit-inline-clone-bundle`` | |||
|
1322 | Run before transferring an inline clonebundle to the peer. | |||
|
1323 | If the exit status is 0, the inline clonebundle will be allowed to be | |||
|
1324 | transferred. A non-zero status will cause the transfer to fail. | |||
|
1325 | The path of the inline clonebundle is in ``$HG_CLONEBUNDLEPATH``. | |||
|
1326 | ||||
1321 | ``pretxnopen`` |
|
1327 | ``pretxnopen`` | |
1322 | Run before any new repository transaction is open. The reason for the |
|
1328 | Run before any new repository transaction is open. The reason for the | |
1323 | transaction will be in ``$HG_TXNNAME``, and a unique identifier for the |
|
1329 | transaction will be in ``$HG_TXNNAME``, and a unique identifier for the |
@@ -441,6 +441,13 b' class httppeer(wireprotov1peer.wirepeer)' | |||||
441 | def capabilities(self): |
|
441 | def capabilities(self): | |
442 | return self._caps |
|
442 | return self._caps | |
443 |
|
443 | |||
|
444 | def _finish_inline_clone_bundle(self, stream): | |||
|
445 | # HTTP streams must hit the end to process the last empty | |||
|
446 | # chunk of Chunked-Encoding so the connection can be reused. | |||
|
447 | chunk = stream.read(1) | |||
|
448 | if chunk: | |||
|
449 | self._abort(error.ResponseError(_(b"unexpected response:"), chunk)) | |||
|
450 | ||||
444 | # End of ipeercommands interface. |
|
451 | # End of ipeercommands interface. | |
445 |
|
452 | |||
446 | def _callstream(self, cmd, _compressible=False, **args): |
|
453 | def _callstream(self, cmd, _compressible=False, **args): |
@@ -176,6 +176,12 b' class ipeercommands(interfaceutil.Interf' | |||||
176 | Returns a set of string capabilities. |
|
176 | Returns a set of string capabilities. | |
177 | """ |
|
177 | """ | |
178 |
|
178 | |||
|
179 | def get_inline_clone_bundle(path): | |||
|
180 | """Retrieve clonebundle across the wire. | |||
|
181 | ||||
|
182 | Returns a chunkbuffer | |||
|
183 | """ | |||
|
184 | ||||
179 | def clonebundles(): |
|
185 | def clonebundles(): | |
180 | """Obtains the clone bundles manifest for the repo. |
|
186 | """Obtains the clone bundles manifest for the repo. | |
181 |
|
187 |
@@ -348,6 +348,10 b' class localpeer(repository.peer):' | |||||
348 | def capabilities(self): |
|
348 | def capabilities(self): | |
349 | return self._caps |
|
349 | return self._caps | |
350 |
|
350 | |||
|
351 | def get_inline_clone_bundle(self, path): | |||
|
352 | # not needed with local peer | |||
|
353 | raise NotImplementedError | |||
|
354 | ||||
351 | def clonebundles(self): |
|
355 | def clonebundles(self): | |
352 | return bundlecaches.get_manifest(self._repo) |
|
356 | return bundlecaches.get_manifest(self._repo) | |
353 |
|
357 |
@@ -213,7 +213,7 b' def _clientcapabilities():' | |||||
213 |
|
213 | |||
214 | Returns a list of capabilities that are supported by this client. |
|
214 | Returns a list of capabilities that are supported by this client. | |
215 | """ |
|
215 | """ | |
216 | protoparams = {b'partial-pull'} |
|
216 | protoparams = {b'partial-pull', b'inlineclonebundles'} | |
217 | comps = [ |
|
217 | comps = [ | |
218 | e.wireprotosupport().name |
|
218 | e.wireprotosupport().name | |
219 | for e in util.compengines.supportedwireengines(util.CLIENTROLE) |
|
219 | for e in util.compengines.supportedwireengines(util.CLIENTROLE) |
@@ -428,7 +428,16 b' def consumev1(repo, fp, filecount, bytec' | |||||
428 | with repo.svfs.backgroundclosing(repo.ui, expectedcount=filecount): |
|
428 | with repo.svfs.backgroundclosing(repo.ui, expectedcount=filecount): | |
429 | for i in range(filecount): |
|
429 | for i in range(filecount): | |
430 | # XXX doesn't support '\n' or '\r' in filenames |
|
430 | # XXX doesn't support '\n' or '\r' in filenames | |
431 |
|
|
431 | if util.safehasattr(fp, 'readline'): | |
|
432 | l = fp.readline() | |||
|
433 | else: | |||
|
434 | # inline clonebundles use a chunkbuffer, so no readline | |||
|
435 | # --> this should be small anyway, the first line | |||
|
436 | # only contains the size of the bundle | |||
|
437 | l_buf = [] | |||
|
438 | while not (l_buf and l_buf[-1] == b'\n'): | |||
|
439 | l_buf.append(fp.read(1)) | |||
|
440 | l = b''.join(l_buf) | |||
432 | try: |
|
441 | try: | |
433 | name, size = l.split(b'\0', 1) |
|
442 | name, size = l.split(b'\0', 1) | |
434 | size = int(size) |
|
443 | size = int(size) |
@@ -341,6 +341,19 b' class wirepeer(repository.peer):' | |||||
341 | self.requirecap(b'clonebundles', _(b'clone bundles')) |
|
341 | self.requirecap(b'clonebundles', _(b'clone bundles')) | |
342 | return self._call(b'clonebundles') |
|
342 | return self._call(b'clonebundles') | |
343 |
|
343 | |||
|
344 | def _finish_inline_clone_bundle(self, stream): | |||
|
345 | pass # allow override for httppeer | |||
|
346 | ||||
|
347 | def get_inline_clone_bundle(self, path): | |||
|
348 | stream = self._callstream(b"get_inline_clone_bundle", path=path) | |||
|
349 | length = util.uvarintdecodestream(stream) | |||
|
350 | ||||
|
351 | # SSH streams will block if reading more than length | |||
|
352 | for chunk in util.filechunkiter(stream, limit=length): | |||
|
353 | yield chunk | |||
|
354 | ||||
|
355 | self._finish_inline_clone_bundle(stream) | |||
|
356 | ||||
344 | @batchable |
|
357 | @batchable | |
345 | def lookup(self, key): |
|
358 | def lookup(self, key): | |
346 | self.requirecap(b'lookup', _(b'look up remote revision')) |
|
359 | self.requirecap(b'lookup', _(b'look up remote revision')) |
@@ -21,6 +21,7 b' from . import (' | |||||
21 | encoding, |
|
21 | encoding, | |
22 | error, |
|
22 | error, | |
23 | exchange, |
|
23 | exchange, | |
|
24 | hook, | |||
24 | pushkey as pushkeymod, |
|
25 | pushkey as pushkeymod, | |
25 | pycompat, |
|
26 | pycompat, | |
26 | repoview, |
|
27 | repoview, | |
@@ -264,6 +265,40 b' def branches(repo, proto, nodes):' | |||||
264 | return wireprototypes.bytesresponse(b''.join(r)) |
|
265 | return wireprototypes.bytesresponse(b''.join(r)) | |
265 |
|
266 | |||
266 |
|
267 | |||
|
268 | @wireprotocommand(b'get_inline_clone_bundle', b'path', permission=b'pull') | |||
|
269 | def get_inline_clone_bundle(repo, proto, path): | |||
|
270 | """ | |||
|
271 | Server command to send a clonebundle to the client | |||
|
272 | """ | |||
|
273 | if hook.hashook(repo.ui, b'pretransmit-inline-clone-bundle'): | |||
|
274 | hook.hook( | |||
|
275 | repo.ui, | |||
|
276 | repo, | |||
|
277 | b'pretransmit-inline-clone-bundle', | |||
|
278 | throw=True, | |||
|
279 | clonebundlepath=path, | |||
|
280 | ) | |||
|
281 | ||||
|
282 | bundle_dir = repo.vfs.join(bundlecaches.BUNDLE_CACHE_DIR) | |||
|
283 | clonebundlepath = repo.vfs.join(bundle_dir, path) | |||
|
284 | if not repo.vfs.exists(clonebundlepath): | |||
|
285 | raise error.Abort(b'clonebundle %s does not exist' % path) | |||
|
286 | ||||
|
287 | clonebundles_dir = os.path.realpath(bundle_dir) | |||
|
288 | if not os.path.realpath(clonebundlepath).startswith(clonebundles_dir): | |||
|
289 | raise error.Abort(b'clonebundle %s is using an illegal path' % path) | |||
|
290 | ||||
|
291 | def generator(vfs, bundle_path): | |||
|
292 | with vfs(bundle_path) as f: | |||
|
293 | length = os.fstat(f.fileno())[6] | |||
|
294 | yield util.uvarintencode(length) | |||
|
295 | for chunk in util.filechunkiter(f): | |||
|
296 | yield chunk | |||
|
297 | ||||
|
298 | stream = generator(repo.vfs, clonebundlepath) | |||
|
299 | return wireprototypes.streamres(gen=stream, prefer_uncompressed=True) | |||
|
300 | ||||
|
301 | ||||
267 | @wireprotocommand(b'clonebundles', b'', permission=b'pull') |
|
302 | @wireprotocommand(b'clonebundles', b'', permission=b'pull') | |
268 | def clonebundles(repo, proto): |
|
303 | def clonebundles(repo, proto): | |
269 | """Server command for returning info for available bundles to seed clones. |
|
304 | """Server command for returning info for available bundles to seed clones. | |
@@ -273,9 +308,21 b' def clonebundles(repo, proto):' | |||||
273 | Extensions may wrap this command to filter or dynamically emit data |
|
308 | Extensions may wrap this command to filter or dynamically emit data | |
274 | depending on the request. e.g. you could advertise URLs for the closest |
|
309 | depending on the request. e.g. you could advertise URLs for the closest | |
275 | data center given the client's IP address. |
|
310 | data center given the client's IP address. | |
|
311 | ||||
|
312 | The only filter on the server side is filtering out inline clonebundles | |||
|
313 | in case a client does not support them. | |||
|
314 | Otherwise, older clients would retrieve and error out on those. | |||
276 | """ |
|
315 | """ | |
277 | manifest = bundlecaches.get_manifest(repo) |
|
316 | manifest_contents = bundlecaches.get_manifest(repo) | |
278 | return wireprototypes.bytesresponse(manifest) |
|
317 | clientcapabilities = proto.getprotocaps() | |
|
318 | if b'inlineclonebundles' in clientcapabilities: | |||
|
319 | return wireprototypes.bytesresponse(manifest_contents) | |||
|
320 | modified_manifest = [] | |||
|
321 | for line in manifest_contents.splitlines(): | |||
|
322 | if line.startswith(bundlecaches.CLONEBUNDLESCHEME): | |||
|
323 | continue | |||
|
324 | modified_manifest.append(line) | |||
|
325 | return wireprototypes.bytesresponse(b'\n'.join(modified_manifest)) | |||
279 |
|
326 | |||
280 |
|
327 | |||
281 | wireprotocaps = [ |
|
328 | wireprotocaps = [ |
@@ -219,6 +219,59 b' Feature works over SSH' | |||||
219 | no changes found |
|
219 | no changes found | |
220 | 2 local changesets published |
|
220 | 2 local changesets published | |
221 |
|
221 | |||
|
222 | Feature works over SSH with inline bundle | |||
|
223 | $ mkdir server/.hg/bundle-cache/ | |||
|
224 | $ cp full.hg server/.hg/bundle-cache/ | |||
|
225 | $ echo "peer-bundle-cache://full.hg" > server/.hg/clonebundles.manifest | |||
|
226 | $ hg clone -U ssh://user@dummy/server ssh-inline-clone | |||
|
227 | applying clone bundle from peer-bundle-cache://full.hg | |||
|
228 | adding changesets | |||
|
229 | adding manifests | |||
|
230 | adding file changes | |||
|
231 | added 2 changesets with 2 changes to 2 files | |||
|
232 | finished applying clone bundle | |||
|
233 | searching for changes | |||
|
234 | no changes found | |||
|
235 | 2 local changesets published | |||
|
236 | ||||
|
237 | Hooks work with inline bundle | |||
|
238 | $ cp server/.hg/hgrc server/.hg/hgrc-beforeinlinehooks | |||
|
239 | $ echo "[hooks]" >> server/.hg/hgrc | |||
|
240 | $ echo "pretransmit-inline-clone-bundle=echo foo" >> server/.hg/hgrc | |||
|
241 | $ hg clone -U ssh://user@dummy/server ssh-inline-clone-hook | |||
|
242 | applying clone bundle from peer-bundle-cache://full.hg | |||
|
243 | remote: foo | |||
|
244 | adding changesets | |||
|
245 | adding manifests | |||
|
246 | adding file changes | |||
|
247 | added 2 changesets with 2 changes to 2 files | |||
|
248 | finished applying clone bundle | |||
|
249 | searching for changes | |||
|
250 | no changes found | |||
|
251 | 2 local changesets published | |||
|
252 | ||||
|
253 | Hooks can make an inline bundle fail | |||
|
254 | $ cp server/.hg/hgrc-beforeinlinehooks server/.hg/hgrc | |||
|
255 | $ echo "[hooks]" >> server/.hg/hgrc | |||
|
256 | $ echo "pretransmit-inline-clone-bundle=echo bar && false" >> server/.hg/hgrc | |||
|
257 | $ hg clone -U ssh://user@dummy/server ssh-inline-clone-hook-fail | |||
|
258 | applying clone bundle from peer-bundle-cache://full.hg | |||
|
259 | remote: bar | |||
|
260 | remote: abort: pretransmit-inline-clone-bundle hook exited with status 1 | |||
|
261 | abort: stream ended unexpectedly (got 0 bytes, expected 1) | |||
|
262 | [255] | |||
|
263 | $ cp server/.hg/hgrc-beforeinlinehooks server/.hg/hgrc | |||
|
264 | ||||
|
265 | Feature does not use inline bundle over HTTP(S) because there is no protocaps support | |||
|
266 | (so no way for the client to announce that it supports inline clonebundles) | |||
|
267 | $ hg clone -U http://localhost:$HGPORT http-inline-clone | |||
|
268 | requesting all changes | |||
|
269 | adding changesets | |||
|
270 | adding manifests | |||
|
271 | adding file changes | |||
|
272 | added 2 changesets with 2 changes to 2 files | |||
|
273 | new changesets 53245c60e682:aaff8d2ffbbf | |||
|
274 | ||||
222 | Entry with unknown BUNDLESPEC is filtered and not used |
|
275 | Entry with unknown BUNDLESPEC is filtered and not used | |
223 |
|
276 | |||
224 | $ cat > server/.hg/clonebundles.manifest << EOF |
|
277 | $ cat > server/.hg/clonebundles.manifest << EOF |
General Comments 0
You need to be logged in to leave comments.
Login now