clone-bundles: garbage collect older bundle when generating new ones

See inline documentation for details.

hgext/clonebundles.py

@@ -209,7 +209,8 b' It is possible to set Mercurial to autom'
 new content is available.
 
 Mercurial will take care of the process asynchronously. The defined list of
-bundle type will be generated, uploaded, and advertised.
+bundle-type will be generated, uploaded, and advertised. Older bundles will get
+decommissioned as newer ones replace them.
 
 Bundles Generation:
 ...................
@@ -235,11 +236,26 b' basename in the "public" URL is accessib'
   upload-command=sftp put $HGCB_BUNDLE_PATH \
       sftp://bundles.host/clone-bundles/$HGCB_BUNDLE_BASENAME
 
+If the file was already uploaded, the command must still succeed.
+
 After upload, the file should be available at an url defined by
 `clone-bundles.url-template`.
 
   [clone-bundles]
   url-template=https://bundles.host/cache/clone-bundles/{basename}
+
+Old bundles cleanup:
+....................
+
+When new bundles are generated, the older ones are no longer necessary and can
+be removed from storage. This is done through the `clone-bundles.delete-command`
+configuration. The command is given the url of the artifact to delete through
+the `$HGCB_BUNDLE_URL` environment variable.
+
+  [clone-bundles]
+  delete-command=sftp rm sftp://bundles.host/clone-bundles/$HGCB_BUNDLE_BASENAME
+
+If the file was already deleted, the command must still succeed.
 """
 
 
@@ -299,6 +315,8 b" configitem(b'clone-bundles', b'auto-gene"
 
 configitem(b'clone-bundles', b'upload-command', default=None)
 
+configitem(b'clone-bundles', b'delete-command', default=None)
+
 configitem(b'clone-bundles', b'url-template', default=None)
 
 configitem(b'devel', b'debug.clonebundles', default=False)
@@ -666,6 +684,34 b' def finalize_one_bundle(repo, target):'
     cleanup_tmp_bundle(repo, target)
 
 
+def find_outdated_bundles(repo, bundles):
+    """finds outdated bundles"""
+    olds = []
+    per_types = {}
+    for b in bundles:
+        if not b.valid_for(repo):
+            olds.append(b)
+            continue
+        l = per_types.setdefault(b.bundle_type, [])
+        l.append(b)
+    for key in sorted(per_types):
+        all = per_types[key]
+        if len(all) > 1:
+            all.sort(key=lambda b: b.revs, reverse=True)
+            olds.extend(all[1:])
+    return olds
+
+
+def collect_garbage(repo):
+    """finds outdated bundles and get them deleted"""
+    with repo.clonebundles_lock():
+        bundles = read_auto_gen(repo)
+        olds = find_outdated_bundles(repo, bundles)
+        for o in olds:
+            delete_bundle(repo, o)
+        update_bundle_list(repo, del_bundles=olds)
+
+
 def upload_bundle(repo, bundle):
     """upload the result of a GeneratingBundle and return a GeneratedBundle
 
@@ -691,12 +737,34 b' def upload_bundle(repo, bundle):'
     return bundle.uploaded(url, basename)
 
 
+def delete_bundle(repo, bundle):
+    """delete a bundle from storage"""
+    assert bundle.ready
+    msg = b'clone-bundles: deleting bundle %s\n'
+    msg %= bundle.basename
+    if repo.ui.configbool(b'devel', b'debug.clonebundles'):
+        repo.ui.write(msg)
+    else:
+        repo.ui.debug(msg)
+
+    cmd = repo.ui.config(b'clone-bundles', b'delete-command')
+    variables = {
+        b'HGCB_BUNDLE_URL': bundle.file_url,
+        b'HGCB_BASENAME': bundle.basename,
+    }
+    env = procutil.shellenviron(environ=variables)
+    ret = repo.ui.system(cmd, environ=env)
+    if ret:
+        raise error.Abort(b"command returned status %d: %s" % (ret, cmd))
+
+
 def auto_bundle_needed_actions(repo, bundles, op_id):
     """find the list of bundles that need action
 
     returns a list of RequestedBundle objects that need to be generated and
     uploaded."""
     create_bundles = []
+    delete_bundles = []
     repo = repo.filtered(b"immutable")
     targets = repo.ui.configlist(b'clone-bundles', b'auto-generate.formats')
     revs = len(repo.changelog)
@@ -712,7 +780,8 b' def auto_bundle_needed_actions(repo, bun'
         data['bundle_type'] = t
         b = RequestedBundle(**data)
         create_bundles.append(b)
-    return create_bundles
+    delete_bundles.extend(find_outdated_bundles(repo, bundles))
+    return create_bundles, delete_bundles
 
 
 def start_one_bundle(repo, bundle):
@@ -759,6 +828,8 b' def debugmakeclonebundles(ui, repo):'
     requested_bundle = util.pickle.load(procutil.stdin)
     procutil.stdin.close()
 
+    collect_garbage(repo)
+
     fname = requested_bundle.suggested_filename
     fpath = repo.vfs.makedirs(b'tmp-bundles')
     fpath = repo.vfs.join(b'tmp-bundles', fname)
@@ -778,7 +849,7 b' def make_auto_bundler(source_repo):'
         repo = reporef()
         assert repo is not None
         bundles = read_auto_gen(repo)
-        new = auto_bundle_needed_actions(repo, bundles, b"%d_txn" % id(tr))
+        new, __ = auto_bundle_needed_actions(repo, bundles, b"%d_txn" % id(tr))
         for data in new:
             start_one_bundle(repo, data)
         return None

tests/test-clonebundles-autogen.t

@@ -11,6 +11,7 b' initial setup'
   > [clone-bundles]
   > auto-generate.formats = v2
   > upload-command = cp "\$HGCB_BUNDLE_PATH" "$TESTTMP"/final-upload/
+  > delete-command = rm -f "$TESTTMP/final-upload/\$HGCB_BASENAME"
   > url-template = file://$TESTTMP/final-upload/{basename}
   > 
   > [devel]
@@ -68,3 +69,28 b' Newer bundles are generated with more pu'
   full-v2-2_revs-aaff8d2ffbbf_tip-*_txn.hg (glob)
   full-v2-4_revs-6427147b985a_tip-*_txn.hg (glob)
   $ ls -1 ../server/.hg/tmp-bundles
+
+Older bundles are cleaned up with more pushes
+---------------------------------------------
+
+  $ touch faz
+  $ hg -q commit -A -m 'add faz'
+  $ touch fuz
+  $ hg -q commit -A -m 'add fuz'
+  $ hg push
+  pushing to $TESTTMP/server
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  clone-bundles: deleting bundle full-v2-2_revs-aaff8d2ffbbf_tip-*_txn.hg (glob)
+  6 changesets found
+  added 2 changesets with 2 changes to 2 files
+  clone-bundles: starting bundle generation: v2
+
+  $ cat ../server/.hg/clonebundles.manifest
+  file:/*/$TESTTMP/final-upload/full-v2-6_revs-b1010e95ea00_tip-*_txn.hg BUNDLESPEC=v2 REQUIRESNI=true (glob)
+  $ ls -1 ../final-upload
+  full-v2-4_revs-6427147b985a_tip-*_txn.hg (glob)
+  full-v2-6_revs-b1010e95ea00_tip-*_txn.hg (glob)
+  $ ls -1 ../server/.hg/tmp-bundles