diff --git a/IPython/config/loader.py b/IPython/config/loader.py
index a5eae75..fc22aab 100644
--- a/IPython/config/loader.py
+++ b/IPython/config/loader.py
@@ -246,11 +246,13 @@ class Config(dict):
                 c = Config()
                 dict.__setitem__(self, key, c)
                 return c
-            else:
+            elif not key.startswith('_'):
                 # undefined, create lazy value, used for container methods
                 v = LazyConfigValue()
                 dict.__setitem__(self, key, v)
                 return v
+            else:
+                raise KeyError
 
     def __setitem__(self, key, value):
         if _is_section_key(key):
diff --git a/IPython/config/tests/test_loader.py b/IPython/config/tests/test_loader.py
index b9e8076..0238285 100644
--- a/IPython/config/tests/test_loader.py
+++ b/IPython/config/tests/test_loader.py
@@ -369,6 +369,14 @@ class TestConfig(TestCase):
         assert isinstance(foo, LazyConfigValue)
         self.assertIn('foo', cfg)
 
+    def test_getattr_private_missing(self):
+        cfg = Config()
+        self.assertNotIn('_repr_html_', cfg)
+        with self.assertRaises(AttributeError):
+            _ = cfg._repr_html_
+        self.assertNotIn('_repr_html_', cfg)
+        self.assertEqual(len(cfg), 0)
+
     def test_getitem_not_section(self):
         cfg = Config()
         self.assertNotIn('foo', cfg)