diff --git a/IPython/core/pylabtools.py b/IPython/core/pylabtools.py
index 687486c..fa6c6f9 100644
--- a/IPython/core/pylabtools.py
+++ b/IPython/core/pylabtools.py
@@ -354,6 +354,7 @@ def configure_inline_support(shell, backend):
             shell._saved_rcParams[k] = pyplot.rcParams[k]
         # load inline_rc
         pyplot.rcParams.update(cfg.rc)
+        new_backend_name = "inline"
     else:
         from ipykernel.pylab.backend_inline import flush_figures
         try:
@@ -363,7 +364,13 @@ def configure_inline_support(shell, backend):
         if hasattr(shell, '_saved_rcParams'):
             pyplot.rcParams.update(shell._saved_rcParams)
             del shell._saved_rcParams
-
-    # Setup the default figure format
-    select_figure_formats(shell, cfg.figure_formats, **cfg.print_figure_kwargs)
-
+        new_backend_name = "other"
+
+    # only enable the formats once -> don't change the enabled formats (which the user may
+    # has changed) when getting another "%matplotlib inline" call.
+    # See https://github.com/ipython/ipykernel/issues/29
+    cur_backend = getattr(configure_inline_support, "current_backend", "unset")
+    if new_backend_name != cur_backend:
+        # Setup the default figure format
+        select_figure_formats(shell, cfg.figure_formats, **cfg.print_figure_kwargs)
+        configure_inline_support.current_backend = new_backend_name
diff --git a/IPython/core/tests/test_pylabtools.py b/IPython/core/tests/test_pylabtools.py
index 317e41f..df23f08 100644
--- a/IPython/core/tests/test_pylabtools.py
+++ b/IPython/core/tests/test_pylabtools.py
@@ -229,6 +229,26 @@ class TestPylabSwitch(object):
         nt.assert_equal(gui, 'qt')
         nt.assert_equal(s.pylab_gui_select, 'qt')
 
+    def test_inline_twice(self):
+        "Using '%matplotlib inline' twice should not reset formatters"
+
+        ip = get_ipython()
+        gui, backend = ip.enable_matplotlib('inline')
+        nt.assert_equal(gui, 'inline')
+
+        fmts =  {'png'}
+        active_mimes = {_fmt_mime_map[fmt] for fmt in fmts}
+        pt.select_figure_formats(ip, fmts)
+
+        gui, backend = ip.enable_matplotlib('inline')
+        nt.assert_equal(gui, 'inline')
+
+        for mime, f in ip.display_formatter.formatters.items():
+            if mime in active_mimes:
+                nt.assert_in(Figure, f)
+            else:
+                nt.assert_not_in(Figure, f)
+
     def test_qt_gtk(self):
         s = self.Shell()
         gui, backend = s.enable_matplotlib('qt')