diff --git a/hgext/largefiles/lfutil.py b/hgext/largefiles/lfutil.py
--- a/hgext/largefiles/lfutil.py
+++ b/hgext/largefiles/lfutil.py
@@ -123,9 +123,13 @@ def openlfdirstate(ui, repo, create=True
     # it. This ensures that we create it on the first meaningful
     # largefiles operation in a new clone.
     if create and not os.path.exists(os.path.join(lfstoredir, 'dirstate')):
-        util.makedirs(lfstoredir)
         matcher = getstandinmatcher(repo)
-        for standin in repo.dirstate.walk(matcher, [], False, False):
+        standins = repo.dirstate.walk(matcher, [], False, False)
+
+        if len(standins) > 0:
+            util.makedirs(lfstoredir)
+
+        for standin in standins:
             lfile = splitstandin(standin)
             lfdirstate.normallookup(lfile)
     return lfdirstate
diff --git a/tests/test-issue3084.t b/tests/test-issue3084.t
--- a/tests/test-issue3084.t
+++ b/tests/test-issue3084.t
@@ -154,7 +154,20 @@ Prepare test repo:
   $ hg init merges
   $ cd merges
   $ touch f1
-  $ hg ci -Aqm "0-root"
+  $ hg ci -Aqm "0-root" --config extensions.largefiles=!
+
+Ensure that .hg/largefiles isn't created before largefiles are added
+#if unix-permissions
+  $ chmod 555 .hg
+#endif
+  $ hg status
+#if unix-permissions
+  $ chmod 755 .hg
+#endif
+
+  $ find .hg/largefiles
+  find: `.hg/largefiles': No such file or directory
+  [1]
 
 ancestor is "normal":
   $ echo normal > f