diff --git a/hgext/largefiles/lfutil.py b/hgext/largefiles/lfutil.py
--- a/hgext/largefiles/lfutil.py
+++ b/hgext/largefiles/lfutil.py
@@ -141,7 +141,7 @@ class largefilesdirstate(dirstate.dirsta
     def normallookup(self, f):
         return super(largefilesdirstate, self).normallookup(unixpath(f))
 
-def openlfdirstate(ui, repo):
+def openlfdirstate(ui, repo, create=True):
     '''
     Return a dirstate object that tracks largefiles: i.e. its root is
     the repo root, but it is saved in .hg/largefiles/dirstate.
@@ -154,7 +154,7 @@ def openlfdirstate(ui, repo):
     # If the largefiles dirstate does not exist, populate and create
     # it. This ensures that we create it on the first meaningful
     # largefiles operation in a new clone.
-    if not os.path.exists(os.path.join(admin, 'dirstate')):
+    if create and not os.path.exists(os.path.join(admin, 'dirstate')):
         util.makedirs(admin)
         matcher = getstandinmatcher(repo)
         for standin in dirstatewalk(repo.dirstate, matcher):
@@ -435,8 +435,11 @@ def unixpath(path):
     return util.pconvert(os.path.normpath(path))
 
 def islfilesrepo(repo):
-    return ('largefiles' in repo.requirements and
-            util.any(shortname + '/' in f[0] for f in repo.store.datafiles()))
+    if ('largefiles' in repo.requirements and
+            util.any(shortname + '/' in f[0] for f in repo.store.datafiles())):
+        return True
+
+    return util.any(openlfdirstate(repo.ui, repo, False))
 
 class storeprotonotcapable(Exception):
     def __init__(self, storetypes):
diff --git a/tests/test-largefiles.t b/tests/test-largefiles.t
--- a/tests/test-largefiles.t
+++ b/tests/test-largefiles.t
@@ -1583,3 +1583,40 @@ Lock in subrepo, otherwise the change is
   lf_subrepo_archive/subrepo/normal.txt
 
   $ cd ..
+
+Test that addremove picks up largefiles prior to the initial commit (issue3541)
+
+  $ hg init addrm2
+  $ cd addrm2
+  $ touch large.dat
+  $ touch large2.dat
+  $ touch normal
+  $ hg add --large large.dat
+  $ hg addremove -v
+  adding large2.dat as a largefile
+  adding normal
+
+Test that forgetting all largefiles reverts to islfilesrepo() == False
+(addremove will add *.dat as normal files now)
+  $ hg forget large.dat
+  $ hg forget large2.dat
+  $ hg addremove -v
+  adding large.dat
+  adding large2.dat
+
+Test commit's addremove option prior to the first commit
+  $ hg forget large.dat
+  $ hg forget large2.dat
+  $ hg add --large large.dat
+  $ hg ci -Am "commit"
+  adding large2.dat as a largefile
+  Invoking status precommit hook
+  A large.dat
+  A large2.dat
+  A normal
+  $ find .hglf/ | sort
+  .hglf/
+  .hglf/large.dat
+  .hglf/large2.dat
+
+  $ cd ..