diff --git a/hgext/largefiles/overrides.py b/hgext/largefiles/overrides.py
--- a/hgext/largefiles/overrides.py
+++ b/hgext/largefiles/overrides.py
@@ -681,9 +681,9 @@ def overridepull(orig, ui, repo, source=
             def _dummy(*args, **kwargs):
                 pass
             commands.postincoming = _dummy
-            repo.lfpullsource = source
             if not source:
                 source = 'default'
+            repo.lfpullsource = source
             try:
                 result = commands.pull(ui, repo, source, **opts)
             finally:
@@ -694,9 +694,9 @@ def overridepull(orig, ui, repo, source=
         finally:
             repo._isrebasing = False
     else:
-        repo.lfpullsource = source
         if not source:
             source = 'default'
+        repo.lfpullsource = source
         oldheads = lfutil.getcurrentheads(repo)
         result = orig(ui, repo, source, **opts)
         # If we do not have the new largefiles for any new heads we pulled, we
diff --git a/tests/test-largefiles.t b/tests/test-largefiles.t
--- a/tests/test-largefiles.t
+++ b/tests/test-largefiles.t
@@ -765,7 +765,9 @@ Ensure base clone command argument valid
   abort: --all-largefiles is incompatible with non-local destination ssh://localhost/a
   [255]
 
-Test pulling with --all-largefiles flag
+Test pulling with --all-largefiles flag.  Also test that the largefiles are
+downloaded from 'default' instead of 'default-push' when no source is specified
+(issue3584)
 
   $ rm -Rf a-backup
   $ hg clone -r 1 a a-backup
@@ -779,7 +781,7 @@ Test pulling with --all-largefiles flag
   2 largefiles updated, 0 removed
   $ rm "${USERCACHE}"/*
   $ cd a-backup
-  $ hg pull --all-largefiles
+  $ hg pull --all-largefiles --config paths.default-push=bogus/path
   pulling from $TESTTMP/a (glob)
   searching for changes
   adding changesets
@@ -821,8 +823,12 @@ revisions (this was a very bad bug that 
   getting changed largefiles
   3 largefiles updated, 0 removed
   $ cd d
-  $ hg pull --rebase ../b
-  pulling from ../b
+
+More rebase testing, but also test that the largefiles are downloaded from
+'default' instead of 'default-push' when no source is specified (issue3584).
+The error messages go away if repo 'b' is created with --all-largefiles.
+  $ hg pull --rebase --all-largefiles --config paths.default-push=bogus/path --config paths.default=../b
+  pulling from $TESTTMP/b (glob)
   searching for changes
   adding changesets
   adding manifests
@@ -832,6 +838,26 @@ revisions (this was a very bad bug that 
   M sub/normal4
   M sub2/large6
   saved backup bundle to $TESTTMP/d/.hg/strip-backup/f574fb32bb45-backup.hg (glob)
+  large3: can't get file locally
+  (no default or default-push path set in hgrc)
+  sub/large4: can't get file locally
+  (no default or default-push path set in hgrc)
+  large1: can't get file locally
+  (no default or default-push path set in hgrc)
+  sub/large2: can't get file locally
+  (no default or default-push path set in hgrc)
+  sub/large2: can't get file locally
+  (no default or default-push path set in hgrc)
+  large1: can't get file locally
+  (no default or default-push path set in hgrc)
+  sub/large2: can't get file locally
+  (no default or default-push path set in hgrc)
+  large1: can't get file locally
+  (no default or default-push path set in hgrc)
+  sub/large2: can't get file locally
+  (no default or default-push path set in hgrc)
+  0 additional largefiles cached
+  9 largefiles failed to download
   nothing to rebase
   $ hg log --template '{rev}:{node|short}  {desc|firstline}\n'
   9:598410d3eb9a  modify normal file largefile in repo d