##// END OF EJS Templates
clonebundles: adds a auto-generate.serve-inline option...
marmoute -
r51604:40638610 default
parent child Browse files
Show More
@@ -254,9 +254,28 b' command, or automatically on each reposi'
254 auto-generate.on-change=yes
254 auto-generate.on-change=yes
255 auto-generate.formats= zstd-v2, gzip-v2
255 auto-generate.formats= zstd-v2, gzip-v2
256
256
257 Automatic Inline serving
258 ........................
259
260 The simplest way to serve the generated bundle is through the Mercurial
261 protocol. However it is not the most efficient as request will still be served
262 by that main server. It is useful in case where authentication is complexe or
263 when an efficient mirror system is already in use anyway. See the `inline
264 clonebundles` section above for details about inline clonebundles
265
266 To automatically serve generated bundle through inline clonebundle, simply set
267 the following option::
268
269 auto-generate.serve-inline=yes
270
271 Enabling this option disable the managed upload and serving explained below.
272
257 Bundles Upload and Serving:
273 Bundles Upload and Serving:
258 ...........................
274 ...........................
259
275
276 This is the most efficient way to serve automatically generated clone bundles,
277 but requires some setup.
278
260 The generated bundles need to be made available to users through a "public" URL.
279 The generated bundles need to be made available to users through a "public" URL.
261 This should be donne through `clone-bundles.upload-command` configuration. The
280 This should be donne through `clone-bundles.upload-command` configuration. The
262 value of this command should be a shell command. It will have access to the
281 value of this command should be a shell command. It will have access to the
@@ -344,6 +363,7 b' command = registrar.command(cmdtable)'
344
363
345 configitem(b'clone-bundles', b'auto-generate.on-change', default=False)
364 configitem(b'clone-bundles', b'auto-generate.on-change', default=False)
346 configitem(b'clone-bundles', b'auto-generate.formats', default=list)
365 configitem(b'clone-bundles', b'auto-generate.formats', default=list)
366 configitem(b'clone-bundles', b'auto-generate.serve-inline', default=False)
347 configitem(b'clone-bundles', b'trigger.below-bundled-ratio', default=0.95)
367 configitem(b'clone-bundles', b'trigger.below-bundled-ratio', default=0.95)
348 configitem(b'clone-bundles', b'trigger.revs', default=1000)
368 configitem(b'clone-bundles', b'trigger.revs', default=1000)
349
369
@@ -753,45 +773,67 b' def upload_bundle(repo, bundle):'
753
773
754 The upload is done using the `clone-bundles.upload-command`
774 The upload is done using the `clone-bundles.upload-command`
755 """
775 """
756 cmd = repo.ui.config(b'clone-bundles', b'upload-command')
776 inline = repo.ui.config(b'clone-bundles', b'auto-generate.serve-inline')
757 url = repo.ui.config(b'clone-bundles', b'url-template')
758 basename = repo.vfs.basename(bundle.filepath)
777 basename = repo.vfs.basename(bundle.filepath)
759 filepath = procutil.shellquote(bundle.filepath)
778 if inline:
760 variables = {
779 dest_dir = repo.vfs.join(bundlecaches.BUNDLE_CACHE_DIR)
761 b'HGCB_BUNDLE_PATH': filepath,
780 repo.vfs.makedirs(dest_dir)
762 b'HGCB_BUNDLE_BASENAME': basename,
781 dest = repo.vfs.join(dest_dir, basename)
763 }
782 util.copyfiles(bundle.filepath, dest, hardlink=True)
764 env = procutil.shellenviron(environ=variables)
783 url = bundlecaches.CLONEBUNDLESCHEME + basename
765 ret = repo.ui.system(cmd, environ=env)
784 return bundle.uploaded(url, basename)
766 if ret:
785 else:
767 raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
786 cmd = repo.ui.config(b'clone-bundles', b'upload-command')
768 url = (
787 url = repo.ui.config(b'clone-bundles', b'url-template')
769 url.decode('utf8')
788 filepath = procutil.shellquote(bundle.filepath)
770 .format(basename=basename.decode('utf8'))
789 variables = {
771 .encode('utf8')
790 b'HGCB_BUNDLE_PATH': filepath,
772 )
791 b'HGCB_BUNDLE_BASENAME': basename,
773 return bundle.uploaded(url, basename)
792 }
793 env = procutil.shellenviron(environ=variables)
794 ret = repo.ui.system(cmd, environ=env)
795 if ret:
796 raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
797 url = (
798 url.decode('utf8')
799 .format(basename=basename.decode('utf8'))
800 .encode('utf8')
801 )
802 return bundle.uploaded(url, basename)
774
803
775
804
776 def delete_bundle(repo, bundle):
805 def delete_bundle(repo, bundle):
777 """delete a bundle from storage"""
806 """delete a bundle from storage"""
778 assert bundle.ready
807 assert bundle.ready
779 msg = b'clone-bundles: deleting bundle %s\n'
808
809 inline = bundle.file_url.startswith(bundlecaches.CLONEBUNDLESCHEME)
810
811 if inline:
812 msg = b'clone-bundles: deleting inline bundle %s\n'
813 else:
814 msg = b'clone-bundles: deleting bundle %s\n'
780 msg %= bundle.basename
815 msg %= bundle.basename
781 if repo.ui.configbool(b'devel', b'debug.clonebundles'):
816 if repo.ui.configbool(b'devel', b'debug.clonebundles'):
782 repo.ui.write(msg)
817 repo.ui.write(msg)
783 else:
818 else:
784 repo.ui.debug(msg)
819 repo.ui.debug(msg)
785
820
786 cmd = repo.ui.config(b'clone-bundles', b'delete-command')
821 if inline:
787 variables = {
822 inline_path = repo.vfs.join(
788 b'HGCB_BUNDLE_URL': bundle.file_url,
823 bundlecaches.BUNDLE_CACHE_DIR,
789 b'HGCB_BASENAME': bundle.basename,
824 bundle.basename,
790 }
825 )
791 env = procutil.shellenviron(environ=variables)
826 util.tryunlink(inline_path)
792 ret = repo.ui.system(cmd, environ=env)
827 else:
793 if ret:
828 cmd = repo.ui.config(b'clone-bundles', b'delete-command')
794 raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
829 variables = {
830 b'HGCB_BUNDLE_URL': bundle.file_url,
831 b'HGCB_BASENAME': bundle.basename,
832 }
833 env = procutil.shellenviron(environ=variables)
834 ret = repo.ui.system(cmd, environ=env)
835 if ret:
836 raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
795
837
796
838
797 def auto_bundle_needed_actions(repo, bundles, op_id):
839 def auto_bundle_needed_actions(repo, bundles, op_id):
@@ -382,3 +382,95 b' bundles should have been generated with '
382 $ cat ../server/.hg/clonebundles.manifest
382 $ cat ../server/.hg/clonebundles.manifest
383 https://example.com/final-upload/full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v1 REQUIRESNI=true (glob)
383 https://example.com/final-upload/full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v1 REQUIRESNI=true (glob)
384 https://example.com/final-upload/full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v2 REQUIRESNI=true (glob)
384 https://example.com/final-upload/full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v2 REQUIRESNI=true (glob)
385
386 Test serving them through inline-clone bundle
387 =============================================
388
389 $ cat >> ../server/.hg/hgrc << EOF
390 > [clone-bundles]
391 > auto-generate.serve-inline=yes
392 > EOF
393 $ hg -R ../server/ admin::clone-bundles-clear
394 clone-bundles: deleting bundle full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
395 clone-bundles: deleting bundle full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
396
397 initial generation
398 ------------------
399
400
401 $ hg -R ../server/ admin::clone-bundles-refresh
402 clone-bundles: starting bundle generation: v1
403 11 changesets found
404 clone-bundles: starting bundle generation: v2
405 11 changesets found
406 $ cat ../server/.hg/clonebundles.manifest
407 peer-bundle-cache://full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v1 (glob)
408 peer-bundle-cache://full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg BUNDLESPEC=v2 (glob)
409 $ ls -1 ../server/.hg/bundle-cache
410 full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
411 full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
412 $ ls -1 ../final-upload
413
414 Regeneration eventually cleanup the old ones
415 --------------------------------------------
416
417 create more content
418 $ touch voit
419 $ hg -q commit -A -m 'add voit'
420 $ touch ar
421 $ hg -q commit -A -m 'add ar'
422 $ hg push
423 pushing to $TESTTMP/server
424 searching for changes
425 adding changesets
426 adding manifests
427 adding file changes
428 added 2 changesets with 2 changes to 2 files
429
430 check first regeneration
431
432 $ hg -R ../server/ admin::clone-bundles-refresh
433 clone-bundles: starting bundle generation: v1
434 13 changesets found
435 clone-bundles: starting bundle generation: v2
436 13 changesets found
437 $ cat ../server/.hg/clonebundles.manifest
438 peer-bundle-cache://full-v1-13_revs-8a81f9be54ea_tip-*_acbr.hg BUNDLESPEC=v1 (glob)
439 peer-bundle-cache://full-v2-13_revs-8a81f9be54ea_tip-*_acbr.hg BUNDLESPEC=v2 (glob)
440 $ ls -1 ../server/.hg/bundle-cache
441 full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
442 full-v1-13_revs-8a81f9be54ea_tip-*_acbr.hg (glob)
443 full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
444 full-v2-13_revs-8a81f9be54ea_tip-*_acbr.hg (glob)
445 $ ls -1 ../final-upload
446
447 check first regeneration (should cleanup the one before that last)
448
449 $ touch "investi"
450 $ hg -q commit -A -m 'add investi'
451 $ touch "lesgisla"
452 $ hg -q commit -A -m 'add lesgisla'
453 $ hg push
454 pushing to $TESTTMP/server
455 searching for changes
456 adding changesets
457 adding manifests
458 adding file changes
459 added 2 changesets with 2 changes to 2 files
460
461 $ hg -R ../server/ admin::clone-bundles-refresh
462 clone-bundles: deleting inline bundle full-v1-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
463 clone-bundles: deleting inline bundle full-v2-11_revs-4226b1cd5fda_tip-*_acbr.hg (glob)
464 clone-bundles: starting bundle generation: v1
465 15 changesets found
466 clone-bundles: starting bundle generation: v2
467 15 changesets found
468 $ cat ../server/.hg/clonebundles.manifest
469 peer-bundle-cache://full-v1-15_revs-17615b3984c2_tip-*_acbr.hg BUNDLESPEC=v1 (glob)
470 peer-bundle-cache://full-v2-15_revs-17615b3984c2_tip-*_acbr.hg BUNDLESPEC=v2 (glob)
471 $ ls -1 ../server/.hg/bundle-cache
472 full-v1-13_revs-8a81f9be54ea_tip-*_acbr.hg (glob)
473 full-v1-15_revs-17615b3984c2_tip-*_acbr.hg (glob)
474 full-v2-13_revs-8a81f9be54ea_tip-*_acbr.hg (glob)
475 full-v2-15_revs-17615b3984c2_tip-*_acbr.hg (glob)
476 $ ls -1 ../final-upload
General Comments 0
You need to be logged in to leave comments. Login now