diff --git a/IPython/sphinxext/ipython_directive.py b/IPython/sphinxext/ipython_directive.py
index cde36cf..eb1219f 100644
--- a/IPython/sphinxext/ipython_directive.py
+++ b/IPython/sphinxext/ipython_directive.py
@@ -881,10 +881,8 @@ class IPythonDirective(Directive):
             # EmbeddedSphinxShell is created, its interactive shell member
             # is the same for each instance.
 
-            if mplbackend:
+            if mplbackend and 'matplotlib.backends' not in sys.modules:
                 import matplotlib
-                # Repeated calls to use() will not hurt us since `mplbackend`
-                # is the same each time.
                 matplotlib.use(mplbackend)
 
             # Must be called after (potentially) importing matplotlib and