remotefilelog: include file contents in bundles produced during strip

`hg strip` and other things that use repair.strip (such as the narrow
extension's `hg tracked --removeinclude`) will "save" some commits that have a
higher revision number than the oldest commit we're stripping, but aren't
actually descended from any of the commits that we're stripping. It saves them
in a bundle, and then reapplies them to the repo.

Remotefilelog doesn't generally participate in strip, it doesn't contribute
files to either the backup bundle or the "saved" bundle, and doesn't adjust
linknodes when commits are stripped. This can break things like push, which
rely on the linknodes.

This change makes it so that remotefilelog includes files in these bundles
during strip operations. During reapplication, the files are reapplied from the
bundle, and the linknode is properly updated.

Differential Revision: https://phab.mercurial-scm.org/D10320

hgext/remotefilelog/__init__.py

@@ -215,6 +215,8 b" configitem(b'remotefilelog', b'pullprefe"
 configitem(b'remotefilelog', b'backgroundprefetch', default=False)
 configitem(b'remotefilelog', b'prefetchdelay', default=120)
 configitem(b'remotefilelog', b'prefetchdays', default=14)
+# Other values include 'local' or 'none'. Any unrecognized value is 'all'.
+configitem(b'remotefilelog', b'strip.includefiles', default='all')
 
 configitem(b'remotefilelog', b'getfilesstep', default=10000)
 configitem(b'remotefilelog', b'getfilestype', default=b'optimistic')

hgext/remotefilelog/shallowbundle.py

@@ -104,6 +104,18 b' class shallowcg1packer(changegroup.cgpac'
         if source == b"push" or source == b"bundle":
             return AllFiles
 
+        # We won't actually strip the files, but we should put them in any
+        # backup bundle generated by strip (especially for cases like narrow's
+        # `hg tracked --removeinclude`, as failing to do so means that the
+        # "saved" changesets during a strip won't have their files reapplied and
+        # thus their linknode adjusted, if necessary).
+        if source == b"strip":
+            cfg = repo.ui.config(b'remotefilelog', b'strip.includefiles')
+            if cfg == b'local':
+                return LocalFiles
+            elif cfg != b'none':
+                return AllFiles
+
         caps = self._bundlecaps or []
         if source == b"serve" or source == b"pull":
             if constants.BUNDLE2_CAPABLITY in caps:

tests/test-remotefilelog-bgprefetch.t

@@ -63,6 +63,7 b''
   > EOF
   $ hg strip tip
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/6b4b6f66ef8c-b4b8bdaf-backup.hg (glob)
+  1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
 
   $ clearcache
   $ hg pull

tests/test-remotefilelog-bundles.t

@@ -26,12 +26,12 b' Unbundling a shallow bundle'
   $ hg strip -r 66ee28d0328c
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/66ee28d0328c-3d7aafd1-backup.hg (glob)
-  1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
+  2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
   $ hg unbundle .hg/strip-backup/66ee28d0328c-3d7aafd1-backup.hg
   adding changesets
   adding manifests
   adding file changes
-  added 2 changesets with 0 changes to 0 files
+  added 2 changesets with 2 changes to 1 files
   new changesets 66ee28d0328c:16db62c5946f
   (run 'hg update' to get a working copy)
 
@@ -51,7 +51,7 b' Unbundling a full bundle'
 
 Pulling from a shallow bundle
 
-  $ hg strip -r 66ee28d0328c
+  $ hg strip -r 66ee28d0328c --config remotefilelog.strip.includefiles=none
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/66ee28d0328c-3d7aafd1-backup.hg (glob)
   $ hg pull -r 66ee28d0328c .hg/strip-backup/66ee28d0328c-3d7aafd1-backup.hg
   pulling from .hg/strip-backup/66ee28d0328c-3d7aafd1-backup.hg
@@ -63,12 +63,13 b' Pulling from a shallow bundle'
   new changesets 66ee28d0328c (1 drafts)
   (run 'hg update' to get a working copy)
 
-Pulling from a full bundle
+Pulling from a full bundle, also testing that strip produces a full bundle by
+default.
 
   $ hg strip -r 66ee28d0328c
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/66ee28d0328c-b6ee89e7-backup.hg (glob)
-  $ hg pull -r 66ee28d0328c ../fullbundle.hg
-  pulling from ../fullbundle.hg
+  $ hg pull -r 66ee28d0328c .hg/strip-backup/66ee28d0328c-b6ee89e7-backup.hg
+  pulling from .hg/strip-backup/66ee28d0328c-b6ee89e7-backup.hg
   searching for changes
   abort: cannot pull from full bundles
   (use `hg unbundle` instead)

tests/test-remotefilelog-local.t

@@ -116,7 +116,7 b''
   $ hg strip -r .
   2 files updated, 0 files merged, 1 files removed, 0 files unresolved
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/19edf50f4de7-df3d0f74-backup.hg (glob)
-  4 files fetched over 2 fetches - (4 misses, 0.00% hit ratio) over *s (glob)
+  3 files fetched over 2 fetches - (3 misses, 0.00% hit ratio) over *s (glob)
 
 # unbundle
 
@@ -133,13 +133,14 b''
   adding changesets
   adding manifests
   adding file changes
-  added 1 changesets with 0 changes to 0 files
+  added 1 changesets with 3 changes to 3 files
   new changesets 19edf50f4de7 (1 drafts)
   (run 'hg update' to get a working copy)
+  2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
 
   $ hg up
   3 files updated, 0 files merged, 0 files removed, 0 files unresolved
-  4 files fetched over 1 fetches - (4 misses, 0.00% hit ratio) over *s (glob)
+  1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
   $ cat a
   a
 
@@ -148,7 +149,7 b''
   $ clearcache
   $ hg revert -r .~2 y z
   no changes needed to z
-  2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
+  1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
   $ hg checkout -C -r . -q
 
 # explicit bundle should produce full bundle file
@@ -159,7 +160,7 b''
   $ cd ..
 
   $ hgcloneshallow ssh://user@dummy/master shallow2 -q
-  1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
+  2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
   $ cd shallow2
   $ hg unbundle ../local.bundle
   adding changesets

tests/test-remotefilelog-prefetch.t

@@ -86,6 +86,7 b''
   $ printf "[remotefilelog]\npullprefetch=bookmark()\n" >> .hg/hgrc
   $ hg strip tip
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/109c3a557a73-3f43405e-backup.hg (glob)
+  1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
 
   $ clearcache
   $ hg pull

tests/test-remotefilelog-sparse.t

@@ -48,6 +48,7 b''
   $ printf "[remotefilelog]\npullprefetch=bookmark()\n" >> .hg/hgrc
   $ hg strip tip
   saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/876b1317060d-b2e91d8d-backup.hg (glob)
+  2 files fetched over 2 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
 
   $ hg debugsparse --delete z
 

tests/test-remotefilelog-strip.t

@@ -60,8 +60,9 b''
   |
   @  0 b292c1e3311f
   
-FIXME: This should point to a commit that actually exists in the repo. Otherwise
-remotefilelog has to search every commit in the repository looking for a valid
-linkrev every time it's queried, such as during push.
+Demonstrate that the linknode points to a commit that is actually in the repo
+after the strip operation. Otherwise remotefilelog has to search every commit in
+the repository looking for a valid linkrev every time it's queried, such as
+during push.
   $ hg debug-file-linknode -r 70494d a
-  df91f74b871e064c89afa1fe9e2f66afa2c125df
+  70494d7ec5ef6cd3cd6939a9fd2812f9956bf553