merge with stable

hgext/remotefilelog/__init__.py

@@ -229,7 +229,7 b" configitem(b'repack', b'chainorphansbysi"
 configitem(b'packs', b'maxpacksize', default=0)
 configitem(b'packs', b'maxchainlen', default=1000)
 
-configitem(b'devel', b'remotefilelog.ensurestart', default=False)
+configitem(b'devel', b'remotefilelog.bg-wait', default=False)
 
 #  default TTL limit is 30 days
 _defaultlimit = 60 * 60 * 24 * 30
@@ -1080,26 +1080,19 b' def pull(orig, ui, repo, *pats, **opts):'
         prefetchrevset = ui.config(b'remotefilelog', b'pullprefetch')
         bgrepack = repo.ui.configbool(b'remotefilelog', b'backgroundrepack')
         bgprefetch = repo.ui.configbool(b'remotefilelog', b'backgroundprefetch')
-        ensurestart = repo.ui.configbool(b'devel', b'remotefilelog.ensurestart')
 
         if prefetchrevset:
             ui.status(_(b"prefetching file contents\n"))
             revs = scmutil.revrange(repo, [prefetchrevset])
             base = repo[b'.'].rev()
             if bgprefetch:
-                repo.backgroundprefetch(
-                    prefetchrevset, repack=bgrepack, ensurestart=ensurestart
-                )
+                repo.backgroundprefetch(prefetchrevset, repack=bgrepack)
             else:
                 repo.prefetch(revs, base=base)
                 if bgrepack:
-                    repackmod.backgroundrepack(
-                        repo, incremental=True, ensurestart=ensurestart
-                    )
+                    repackmod.backgroundrepack(repo, incremental=True)
         elif bgrepack:
-            repackmod.backgroundrepack(
-                repo, incremental=True, ensurestart=ensurestart
-            )
+            repackmod.backgroundrepack(repo, incremental=True)
 
     return result
 
@@ -1250,13 +1243,9 b' def prefetch(ui, repo, *pats, **opts):'
     revs = scmutil.revrange(repo, opts.get(b'rev'))
     repo.prefetch(revs, opts.get(b'base'), pats, opts)
 
-    ensurestart = repo.ui.configbool(b'devel', b'remotefilelog.ensurestart')
-
     # Run repack in background
     if opts.get(b'repack'):
-        repackmod.backgroundrepack(
-            repo, incremental=True, ensurestart=ensurestart
-        )
+        repackmod.backgroundrepack(repo, incremental=True)
 
 
 @command(
@@ -1276,12 +1265,10 b' def prefetch(ui, repo, *pats, **opts):'
 )
 def repack_(ui, repo, *pats, **opts):
     if opts.get('background'):
-        ensurestart = repo.ui.configbool(b'devel', b'remotefilelog.ensurestart')
         repackmod.backgroundrepack(
             repo,
             incremental=opts.get('incremental'),
             packsonly=opts.get('packsonly', False),
-            ensurestart=ensurestart,
         )
         return
 

hgext/remotefilelog/repack.py

@@ -36,9 +36,7 b' class RepackAlreadyRunning(error.Abort):'
     pass
 
 
-def backgroundrepack(
-    repo, incremental=True, packsonly=False, ensurestart=False
-):
+def backgroundrepack(repo, incremental=True, packsonly=False):
     cmd = [procutil.hgexecutable(), b'-R', repo.origroot, b'repack']
     msg = _(b"(running background repack)\n")
     if incremental:
@@ -48,7 +46,11 b' def backgroundrepack('
         cmd.append(b'--packsonly')
     repo.ui.warn(msg)
     # We know this command will find a binary, so don't block on it starting.
-    procutil.runbgcommand(cmd, encoding.environ, ensurestart=ensurestart)
+    kwargs = {}
+    if repo.ui.configbool(b'devel', b'remotefilelog.bg-wait'):
+        kwargs['record_wait'] = repo.ui.atexit
+
+    procutil.runbgcommand(cmd, encoding.environ, ensurestart=False, **kwargs)
 
 
 def fullrepack(repo, options=None):

hgext/remotefilelog/shallowrepo.py

@@ -215,13 +215,7 b' def wraprepo(repo):'
             )
 
         def backgroundprefetch(
-            self,
-            revs,
-            base=None,
-            repack=False,
-            pats=None,
-            opts=None,
-            ensurestart=False,
+            self, revs, base=None, repack=False, pats=None, opts=None
         ):
             """Runs prefetch in background with optional repack
             """
@@ -232,8 +226,12 b' def wraprepo(repo):'
                 cmd += [b'-r', revs]
             # We know this command will find a binary, so don't block
             # on it starting.
+            kwargs = {}
+            if repo.ui.configbool(b'devel', b'remotefilelog.bg-wait'):
+                kwargs['record_wait'] = repo.ui.atexit
+
             procutil.runbgcommand(
-                cmd, encoding.environ, ensurestart=ensurestart
+                cmd, encoding.environ, ensurestart=False, **kwargs
             )
 
         def prefetch(self, revs, base=None, pats=None, opts=None):

mercurial/utils/procutil.py

@@ -540,12 +540,18 b' if pycompat.iswindows:'
     )
 
     def runbgcommand(
-        script, env, shell=False, stdout=None, stderr=None, ensurestart=True
+        script,
+        env,
+        shell=False,
+        stdout=None,
+        stderr=None,
+        ensurestart=True,
+        record_wait=None,
     ):
         '''Spawn a command without waiting for it to finish.'''
         # we can't use close_fds *and* redirect stdin. I'm not sure that we
         # need to because the detached process has no console connection.
-        subprocess.Popen(
+        p = subprocess.Popen(
             tonativestr(script),
             shell=shell,
             env=tonativeenv(env),
@@ -554,46 +560,64 b' if pycompat.iswindows:'
             stdout=stdout,
             stderr=stderr,
         )
+        if record_wait is not None:
+            record_wait(p.wait)
 
 
 else:
 
     def runbgcommand(
-        cmd, env, shell=False, stdout=None, stderr=None, ensurestart=True
+        cmd,
+        env,
+        shell=False,
+        stdout=None,
+        stderr=None,
+        ensurestart=True,
+        record_wait=None,
     ):
-        '''Spawn a command without waiting for it to finish.'''
+        '''Spawn a command without waiting for it to finish.
+
+
+        When `record_wait` is not None, the spawned process will not be fully
+        detached and the `record_wait` argument will be called with a the
+        `Subprocess.wait` function for the spawned process.  This is mostly
+        useful for developers that need to make sure the spawned process
+        finished before a certain point. (eg: writing test)'''
         # double-fork to completely detach from the parent process
         # based on http://code.activestate.com/recipes/278731
-        pid = os.fork()
-        if pid:
-            if not ensurestart:
+        if record_wait is None:
+            pid = os.fork()
+            if pid:
+                if not ensurestart:
+                    return
+                # Parent process
+                (_pid, status) = os.waitpid(pid, 0)
+                if os.WIFEXITED(status):
+                    returncode = os.WEXITSTATUS(status)
+                else:
+                    returncode = -(os.WTERMSIG(status))
+                if returncode != 0:
+                    # The child process's return code is 0 on success, an errno
+                    # value on failure, or 255 if we don't have a valid errno
+                    # value.
+                    #
+                    # (It would be slightly nicer to return the full exception info
+                    # over a pipe as the subprocess module does.  For now it
+                    # doesn't seem worth adding that complexity here, though.)
+                    if returncode == 255:
+                        returncode = errno.EINVAL
+                    raise OSError(
+                        returncode,
+                        b'error running %r: %s'
+                        % (cmd, os.strerror(returncode)),
+                    )
                 return
-            # Parent process
-            (_pid, status) = os.waitpid(pid, 0)
-            if os.WIFEXITED(status):
-                returncode = os.WEXITSTATUS(status)
-            else:
-                returncode = -(os.WTERMSIG(status))
-            if returncode != 0:
-                # The child process's return code is 0 on success, an errno
-                # value on failure, or 255 if we don't have a valid errno
-                # value.
-                #
-                # (It would be slightly nicer to return the full exception info
-                # over a pipe as the subprocess module does.  For now it
-                # doesn't seem worth adding that complexity here, though.)
-                if returncode == 255:
-                    returncode = errno.EINVAL
-                raise OSError(
-                    returncode,
-                    b'error running %r: %s' % (cmd, os.strerror(returncode)),
-                )
-            return
 
         returncode = 255
         try:
-            # Start a new session
-            os.setsid()
+            if record_wait is None:
+                # Start a new session
+                os.setsid()
 
             stdin = open(os.devnull, b'r')
             if stdout is None:
@@ -603,7 +627,7 b' else:'
 
             # connect stdin to devnull to make sure the subprocess can't
             # muck up that stream for mercurial.
-            subprocess.Popen(
+            p = subprocess.Popen(
                 cmd,
                 shell=shell,
                 env=env,
@@ -612,6 +636,8 b' else:'
                 stdout=stdout,
                 stderr=stderr,
             )
+            if record_wait is not None:
+                record_wait(p.wait)
             returncode = 0
         except EnvironmentError as ex:
             returncode = ex.errno & 0xFF
@@ -624,4 +650,5 b' else:'
         finally:
             # mission accomplished, this child needs to exit and not
             # continue the hg process here.
-            os._exit(returncode)
+            if record_wait is None:
+                os._exit(returncode)

tests/test-remotefilelog-bgprefetch.t

@@ -1,11 +1,9 b''
 #require no-windows
 
   $ . "$TESTDIR/remotefilelog-library.sh"
-# devel.remotefilelog.ensurestart: reduce race condition with
-# waiton{repack/prefetch}
   $ cat >> $HGRCPATH <<EOF
   > [devel]
-  > remotefilelog.ensurestart=True
+  > remotefilelog.bg-wait=True
   > EOF
 
   $ hg init master
@@ -78,8 +76,6 b''
   new changesets 6b4b6f66ef8c
   (run 'hg update' to get a working copy)
   prefetching file contents
-  $ sleep 0.5
-  $ hg debugwaitonprefetch >/dev/null 2>%1
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/ef95c5376f34698742fe34f315fd82136f8f68c0
   $TESTTMP/hgcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a/076f5e2225b3ff0400b98c92aa6cdf403ee24cca
@@ -107,11 +103,6 b''
   new changesets 6b4b6f66ef8c
   (run 'hg update' to get a working copy)
   prefetching file contents
-  $ sleep 0.5
-  $ hg debugwaitonprefetch >/dev/null 2>%1
-  $ sleep 0.5
-  $ hg debugwaitonrepack >/dev/null 2>%1
-  $ sleep 0.5
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histidx
   $TESTTMP/hgcache/master/packs/6e8633deba6e544e5f8edbd7b996d6e31a2c42ae.histpack
@@ -143,11 +134,6 b''
   $ hg up -r 0
   1 files updated, 0 files merged, 1 files removed, 0 files unresolved
   * files fetched over * fetches - (* misses, 0.00% hit ratio) over *s (glob)
-  $ sleep 1
-  $ hg debugwaitonprefetch >/dev/null 2>%1
-  $ sleep 1
-  $ hg debugwaitonrepack >/dev/null 2>%1
-  $ sleep 1
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
@@ -195,11 +181,6 b''
   $ hg commit -qAm b
   * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob)
   $ hg bookmark temporary
-  $ sleep 1
-  $ hg debugwaitonprefetch >/dev/null 2>%1
-  $ sleep 1
-  $ hg debugwaitonrepack >/dev/null 2>%1
-  $ sleep 1
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
@@ -237,19 +218,14 b''
 # background prefetch with repack on rebase when wcprevset configured
 
   $ hg up -r 2
-  3 files updated, 0 files merged, 3 files removed, 0 files unresolved
+  3 files updated, 0 files merged, 2 files removed, 0 files unresolved
   (leaving bookmark temporary)
   $ clearcache
   $ find $CACHEDIR -type f | sort
   $ hg rebase -s temporary -d foo
-  rebasing 3:58147a5b5242 "b" (temporary tip)
-  saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/58147a5b5242-c3678817-rebase.hg
+  rebasing 3:d9cf06e3b5b6 "b" (temporary tip)
+  saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/d9cf06e3b5b6-e5c3dc63-rebase.hg
   3 files fetched over 1 fetches - (3 misses, 0.00% hit ratio) over *s (glob)
-  $ sleep 1
-  $ hg debugwaitonprefetch >/dev/null 2>%1
-  $ sleep 1
-  $ hg debugwaitonrepack >/dev/null 2>%1
-  $ sleep 1
 
 # Ensure that file 'y' was prefetched - it was not part of the rebase operation and therefore
 # could only be downloaded by the background prefetch
@@ -281,7 +257,7 b''
 # Check that foregound prefetch with no arguments blocks until background prefetches finish
 
   $ hg up -r 3
-  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
   $ clearcache
   $ hg prefetch --repack
   waiting for lock on prefetching in $TESTTMP/shallow held by process * on host * (glob) (?)
@@ -289,10 +265,6 b''
   (running background incremental repack)
   * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) (?)
 
-  $ sleep 0.5
-  $ hg debugwaitonrepack >/dev/null 2>%1
-  $ sleep 0.5
-
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histpack
@@ -333,9 +305,6 b''
   got lock after * seconds (glob) (?)
   (running background incremental repack)
   * files fetched over 1 fetches - (* misses, 0.00% hit ratio) over *s (glob) (?)
-  $ sleep 0.5
-  $ hg debugwaitonrepack >/dev/null 2>%1
-  $ sleep 0.5
 
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/8f1443d44e57fec96f72fb2412e01d2818767ef2.histidx

tests/test-remotefilelog-gcrepack.t

@@ -40,8 +40,6 b''
   $ hg prefetch
   2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
   $ hg repack
-  $ sleep 0.5
-  $ hg debugwaitonrepack >/dev/null 2>%1
 
   $ find $CACHEDIR | sort | egrep ".datapack|.histpack"
   $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack
@@ -73,8 +71,6 b''
   > EOF
 
   $ hg repack
-  $ sleep 0.5
-  $ hg debugwaitonrepack >/dev/null 2>%1
 
   $ find $CACHEDIR | sort | egrep ".datapack|.histpack"
   $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack
@@ -101,8 +97,6 b''
   $ hg prefetch
   2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
   $ hg repack
-  $ sleep 0.5
-  $ hg debugwaitonrepack >/dev/null 2>%1
 
   $ find $CACHEDIR | sort | egrep ".datapack|.histpack"
   $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack
@@ -135,8 +129,6 b''
   > EOF
 
   $ hg repack
-  $ sleep 0.5
-  $ hg debugwaitonrepack >/dev/null 2>%1
 
   $ find $CACHEDIR | sort | egrep ".datapack|.histpack"
   $TESTTMP/hgcache/master/packs/7bcd2d90b99395ca43172a0dd24e18860b2902f9.histpack

tests/test-remotefilelog-repack-fast.t

@@ -1,13 +1,12 b''
 #require no-windows
 
   $ . "$TESTDIR/remotefilelog-library.sh"
-# devel.remotefilelog.ensurestart: reduce race condition with
-# waiton{repack/prefetch}
+
   $ cat >> $HGRCPATH <<EOF
   > [remotefilelog]
   > fastdatapack=True
   > [devel]
-  > remotefilelog.ensurestart=True
+  > remotefilelog.bg-wait=True
   > EOF
 
   $ hg init master
@@ -142,8 +141,6 b''
 
   $ hg repack --background
   (running background repack)
-  $ sleep 0.5
-  $ hg debugwaitonrepack >/dev/null 2>&1
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.dataidx
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.datapack
@@ -342,8 +339,6 b' Pull should run background repack'
   searching for changes
   no changes found
   (running background incremental repack)
-  $ sleep 0.5
-  $ hg debugwaitonrepack >/dev/null 2>&1
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--     303 156a6c1c83aeb69422d7936e0a46ba9bc06a71c0.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack

tests/test-remotefilelog-repack.t

@@ -1,11 +1,9 b''
 #require no-windows
 
   $ . "$TESTDIR/remotefilelog-library.sh"
-# devel.remotefilelog.ensurestart: reduce race condition with
-# waiton{repack/prefetch}
   $ cat >> $HGRCPATH <<EOF
   > [devel]
-  > remotefilelog.ensurestart=True
+  > remotefilelog.bg-wait=True
   > EOF
 
   $ hg init master
@@ -152,8 +150,6 b''
 
   $ hg repack --background
   (running background repack)
-  $ sleep 0.5
-  $ hg debugwaitonrepack >/dev/null 2>&1
   $ find $CACHEDIR -type f | sort
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.dataidx
   $TESTTMP/hgcache/master/packs/39443fa1064182e93d968b5cba292eb5283260d0.datapack
@@ -371,8 +367,6 b' Pull should run background repack'
   searching for changes
   no changes found
   (running background incremental repack)
-  $ sleep 0.5
-  $ hg debugwaitonrepack >/dev/null 2>&1
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
   -r--r--r--     303 156a6c1c83aeb69422d7936e0a46ba9bc06a71c0.datapack
   $ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack