add utils.path.ensure_dir_exists

Ensures that a directory exists,
and protects against a common race condition when multiple
processes try to create the same directory.

IPython/core/application.py

@@ -39,7 +39,7 from IPython.config.application import Application, catch_config_error
 from IPython.config.loader import ConfigFileNotFound
 from IPython.core import release, crashhandler
 from IPython.core.profiledir import ProfileDir, ProfileDirError
-from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
+from IPython.utils.path import get_ipython_dir, get_ipython_package_dir, ensure_dir_exists
 from IPython.utils import py3compat
 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict, Set, Instance
 
@@ -220,20 +220,18 class BaseIPythonApplication(Application):
             sys.getfilesystemencoding()
         )
         sys.path.append(str_path)
-        if not os.path.isdir(new):
-            os.makedirs(new, mode=0o777)
+        ensure_dir_exists(new)
         readme = os.path.join(new, 'README')
         readme_src = os.path.join(get_ipython_package_dir(), u'config', u'profile', 'README')
         if not os.path.exists(readme) and os.path.exists(readme_src):
             shutil.copy(readme_src, readme)
         for d in ('extensions', 'nbextensions'):
             path = os.path.join(new, d)
-            if not os.path.exists(path):
-                try:
-                    os.mkdir(path)
-                except OSError as e:
-                    if e.errno != errno.EEXIST:
-                        self.log.error("couldn't create path %s: %s", path, e)
+            try:
+                ensure_dir_exists(path)
+            except OSError:
+                # this will not be EEXIST
+                self.log.error("couldn't create path %s: %s", path, e)
         self.log.debug("IPYTHONDIR set to: %s" % new)
 
     def load_config_file(self, suppress_errors=True):

IPython/core/extensions.py

@@ -1,27 +1,15
 # encoding: utf-8
-"""A class for managing IPython extensions.
+"""A class for managing IPython extensions."""
 
-Authors:
-
-* Brian Granger
-"""
-
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2010-2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 import os
 from shutil import copyfile
 import sys
 
 from IPython.config.configurable import Configurable
+from IPython.utils.path import ensure_dir_exists
 from IPython.utils.traitlets import Instance
 from IPython.utils.py3compat import PY3
 if PY3:
@@ -77,8 +65,7 class ExtensionManager(Configurable):
         return os.path.join(self.shell.ipython_dir, u'extensions')
 
     def _on_ipython_dir_changed(self):
-        if not os.path.isdir(self.ipython_extension_dir):
-            os.makedirs(self.ipython_extension_dir, mode = 0o777)
+        ensure_dir_exists(self.ipython_extension_dir)
 
     def load_extension(self, module_str):
         """Load an IPython extension by its module name.
@@ -162,8 +149,7 class ExtensionManager(Configurable):
         Returns the full path to the installed file.
         """
         # Ensure the extension directory exists
-        if not os.path.isdir(self.ipython_extension_dir):
-            os.makedirs(self.ipython_extension_dir, mode = 0o777)
+        ensure_dir_exists(self.ipython_extension_dir)
 
         if os.path.isfile(url):
             src_filename = os.path.basename(url)

IPython/core/interactiveshell.py

@@ -66,7 +66,7 from IPython.utils import openpy
 from IPython.utils.decorators import undoc
 from IPython.utils.io import ask_yes_no
 from IPython.utils.ipstruct import Struct
-from IPython.utils.path import get_home_dir, get_ipython_dir, get_py_filename, unquote_filename
+from IPython.utils.path import get_home_dir, get_ipython_dir, get_py_filename, unquote_filename, ensure_dir_exists
 from IPython.utils.pickleshare import PickleShareDB
 from IPython.utils.process import system, getoutput
 from IPython.utils.py3compat import (builtin_mod, unicode_type, string_types,
@@ -524,8 +524,7 class InteractiveShell(SingletonConfigurable):
     #-------------------------------------------------------------------------
 
     def _ipython_dir_changed(self, name, new):
-        if not os.path.isdir(new):
-            os.makedirs(new, mode = 0o777)
+        ensure_dir_exists(new)
 
     def set_autoindent(self,value=None):
         """Set the autoindent flag, checking for readline support.

IPython/core/profiledir.py

@@ -1,25 +1,8
 # encoding: utf-8
-"""
-An object for managing IPython profile directories.
+"""An object for managing IPython profile directories."""
 
-Authors:
-
-* Brian Granger
-* Fernando Perez
-* Min RK
-
-"""
-
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2011 The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 import os
 import shutil
@@ -27,16 +10,11 import errno
 import time
 
 from IPython.config.configurable import LoggingConfigurable
-from IPython.utils.path import get_ipython_package_dir, expand_path
+from IPython.utils.path import get_ipython_package_dir, expand_path, ensure_dir_exists
 from IPython.utils import py3compat
 from IPython.utils.traitlets import Unicode, Bool
 
 #-----------------------------------------------------------------------------
-# Classes and functions
-#-----------------------------------------------------------------------------
-
-
-#-----------------------------------------------------------------------------
 # Module errors
 #-----------------------------------------------------------------------------
 
@@ -80,16 +58,7 class ProfileDir(LoggingConfigurable):
         if self._location_isset:
             raise RuntimeError("Cannot set profile location more than once.")
         self._location_isset = True
-        num_tries = 0
-        max_tries = 5
-        while not os.path.isdir(new):
-            try:
-                os.makedirs(new)
-            except OSError:
-                if num_tries > max_tries:
-                    raise
-                num_tries += 1
-                time.sleep(0.5)
+        ensure_dir_exists(new)
 
         # ensure config files exist:
         self.security_dir = os.path.join(new, self.security_dir_name)

IPython/html/nbextensions.py

@@ -1,12 +1,8
 # coding: utf-8
 """Utilities for installing Javascript extensions for the notebook"""
 
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2014 The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
@@ -24,7 +20,7 except ImportError:
     from urlparse import urlparse
     from urllib import urlretrieve
 
-from IPython.utils.path import get_ipython_dir
+from IPython.utils.path import get_ipython_dir, ensure_dir_exists
 from IPython.utils.py3compat import string_types, cast_unicode_py2
 from IPython.utils.tempdir import TemporaryDirectory
 
@@ -109,8 +105,7 def install_nbextension(files, overwrite=False, symlink=False, ipython_dir=None,
     ipython_dir = ipython_dir or get_ipython_dir()
     nbext = pjoin(ipython_dir, u'nbextensions')
     # make sure nbextensions dir exists
-    if not os.path.exists(nbext):
-        os.makedirs(nbext)
+    ensure_dir_exists(nbext)
     
     if isinstance(files, string_types):
         # one file given, turn it into a list

IPython/html/services/notebooks/filenbmanager.py

@@ -1,21 +1,7
-"""A notebook manager that uses the local file system for storage.
+"""A notebook manager that uses the local file system for storage."""
 
-Authors:
-
-* Brian Granger
-* Zach Sailer
-"""
-
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 import io
 import os
@@ -26,6 +12,7 from tornado import web
 
 from .nbmanager import NotebookManager
 from IPython.nbformat import current
+from IPython.utils.path import ensure_dir_exists
 from IPython.utils.traitlets import Unicode, Bool, TraitError
 from IPython.utils.py3compat import getcwd
 from IPython.utils import tz
@@ -402,8 +389,7 class FileNotebookManager(NotebookManager):
         )
         os_path = self._get_os_path(path=path)
         cp_dir = os.path.join(os_path, self.checkpoint_dir)
-        if not os.path.exists(cp_dir):
-            os.mkdir(cp_dir)
+        ensure_dir_exists(cp_dir)
         cp_path = os.path.join(cp_dir, filename)
         return cp_path
 
@@ -429,8 +415,6 class FileNotebookManager(NotebookManager):
         checkpoint_id = u"checkpoint"
         cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
         self.log.debug("creating checkpoint for notebook %s", name)
-        if not os.path.exists(self.checkpoint_dir):
-            os.mkdir(self.checkpoint_dir)
         self._copy(nb_path, cp_path)
         
         # return the checkpoint info

IPython/nbconvert/writers/files.py

@@ -8,7 +8,7 import os
 import glob
 
 from IPython.utils.traitlets import Unicode
-from IPython.utils.path import link_or_copy
+from IPython.utils.path import link_or_copy, ensure_dir_exists
 from IPython.utils.py3compat import unicode_type
 
 from .base import WriterBase
@@ -28,8 +28,8 class FilesWriter(WriterBase):
 
     # Make sure that the output directory exists.
     def _build_directory_changed(self, name, old, new):
-        if new and not os.path.isdir(new):
-            os.makedirs(new)
+        if new:
+            ensure_dir_exists(new)
 
 
     def __init__(self, **kw):
@@ -39,9 +39,9 class FilesWriter(WriterBase):
     
     def _makedir(self, path):
         """Make a directory if it doesn't already exist"""
-        if path and not os.path.isdir(path):
+        if path:
             self.log.info("Making directory %s", path)
-            os.makedirs(path)
+            ensure_dir_exists(path)
 
     def write(self, output, resources, notebook_name=None, **kw):
             """

IPython/parallel/apps/launcher.py

@@ -1,23 +1,8
 # encoding: utf-8
-"""
-Facilities for launching IPython processes asynchronously.
+"""Facilities for launching IPython processes asynchronously."""
 
-Authors:
-
-* Brian Granger
-* MinRK
-"""
-
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 import copy
 import logging
@@ -62,7 +47,7 from IPython.utils.traitlets import (
     Any, Integer, CFloat, List, Unicode, Dict, Instance, HasTraits, CRegExp
 )
 from IPython.utils.encoding import DEFAULT_ENCODING
-from IPython.utils.path import get_home_dir
+from IPython.utils.path import get_home_dir, ensure_dir_exists
 from IPython.utils.process import find_cmd, FindCmdError
 from IPython.utils.py3compat import iteritems, itervalues
 
@@ -629,8 +614,7 class SSHLauncher(LocalProcessLauncher):
             elif check == u'yes':
                 break
         local_dir = os.path.dirname(local)
-        if not os.path.exists(local_dir):
-            os.makedirs(local_dir, 775)
+        ensure_dir_exists(local_dir, 775)
         check_output(self.scp_cmd + [full_remote, local])
     
     def fetch_files(self):

IPython/qt/console/rich_ipython_widget.py

@@ -1,20 +1,13
-#-----------------------------------------------------------------------------
-# Copyright (c) 2010, IPython Development Team.
-#
+# Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
 
-# Standard libary imports.
 from base64 import decodestring
 import os
 import re
 
-# System libary imports.
 from IPython.external.qt import QtCore, QtGui
 
-# Local imports
+from IPython.utils.path import ensure_dir_exists
 from IPython.utils.traitlets import Bool
 from IPython.qt.svg import save_svg, svg_to_clipboard, svg_to_image
 from .ipython_widget import IPythonWidget
@@ -233,8 +226,7 class RichIPythonWidget(IPythonWidget):
                 return "<b>Couldn't find image %s</b>" % match.group("name")
 
             if path is not None:
-                if not os.path.exists(path):
-                    os.mkdir(path)
+                ensure_dir_exists(path)
                 relpath = os.path.basename(path)
                 if image.save("%s/qt_img%s.%s" % (path, match.group("name"), format),
                               "PNG"):

IPython/utils/path.py

@@ -3,16 +3,8
 Utilities for path handling.
 """
 
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 import os
 import sys
@@ -324,7 +316,7 def get_ipython_cache_dir():
         return get_ipython_dir()
     ipdir = os.path.join(xdgdir, "ipython")
     if not os.path.exists(ipdir) and _writable_dir(xdgdir):
-        os.makedirs(ipdir)
+        ensure_dir_exists(ipdir)
     elif not _writable_dir(xdgdir):
         return get_ipython_dir()
 
@@ -572,3 +564,17 def link_or_copy(src, dst):
         # Either link isn't supported, or the filesystem doesn't support
         # linking, or 'src' and 'dst' are on different filesystems.
         shutil.copy(src, dst)
+
+def ensure_dir_exists(path, mode=0o777):
+    """ensure that a directory exists
+    
+    If it doesn't exist, try to create it and protect against a race condition
+    if another process is doing the same.
+    """
+    if not os.path.exists(path):
+        try:
+            os.makedirs(path, mode=mode)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+    
  No newline at end of file