clusterdir.py
460 lines
| 17.1 KiB
| text/x-python
|
PythonLexer
Brian Granger
|
r2301 | #!/usr/bin/env python | |
# encoding: utf-8 | |||
""" | |||
The IPython cluster directory | |||
""" | |||
#----------------------------------------------------------------------------- | |||
# Copyright (C) 2008-2009 The IPython Development Team | |||
# | |||
# Distributed under the terms of the BSD License. The full license is in | |||
# the file COPYING, distributed as part of this software. | |||
#----------------------------------------------------------------------------- | |||
#----------------------------------------------------------------------------- | |||
# Imports | |||
#----------------------------------------------------------------------------- | |||
Brian Granger
|
r2313 | from __future__ import with_statement | |
Brian Granger
|
r2301 | import os | |
import shutil | |||
Brian Granger
|
r2306 | import sys | |
from twisted.python import log | |||
Brian Granger
|
r2301 | ||
from IPython.core import release | |||
from IPython.config.loader import PyFileConfigLoader | |||
from IPython.core.application import Application | |||
from IPython.core.component import Component | |||
from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault | |||
Brian Granger
|
r2303 | from IPython.utils.traitlets import Unicode, Bool | |
Brian Granger
|
r2301 | ||
#----------------------------------------------------------------------------- | |||
# Imports | |||
#----------------------------------------------------------------------------- | |||
Brian Granger
|
r2303 | class ClusterDirError(Exception): | |
pass | |||
Brian Granger
|
r2313 | class PIDFileError(Exception): | |
pass | |||
Brian Granger
|
r2301 | class ClusterDir(Component): | |
"""An object to manage the cluster directory and its resources. | |||
The cluster directory is used by :command:`ipcontroller`, | |||
:command:`ipcontroller` and :command:`ipcontroller` to manage the | |||
configuration, logging and security of these applications. | |||
This object knows how to find, create and manage these directories. This | |||
should be used by any code that want's to handle cluster directories. | |||
""" | |||
security_dir_name = Unicode('security') | |||
log_dir_name = Unicode('log') | |||
Brian Granger
|
r2313 | pid_dir_name = Unicode('pid') | |
security_dir = Unicode(u'') | |||
log_dir = Unicode(u'') | |||
pid_dir = Unicode(u'') | |||
location = Unicode(u'') | |||
Brian Granger
|
r2301 | ||
def __init__(self, location): | |||
super(ClusterDir, self).__init__(None) | |||
self.location = location | |||
def _location_changed(self, name, old, new): | |||
if not os.path.isdir(new): | |||
os.makedirs(new, mode=0777) | |||
else: | |||
os.chmod(new, 0777) | |||
self.security_dir = os.path.join(new, self.security_dir_name) | |||
self.log_dir = os.path.join(new, self.log_dir_name) | |||
Brian Granger
|
r2313 | self.pid_dir = os.path.join(new, self.pid_dir_name) | |
Brian Granger
|
r2302 | self.check_dirs() | |
Brian Granger
|
r2301 | ||
def _log_dir_changed(self, name, old, new): | |||
Brian Granger
|
r2302 | self.check_log_dir() | |
def check_log_dir(self): | |||
if not os.path.isdir(self.log_dir): | |||
os.mkdir(self.log_dir, 0777) | |||
Brian Granger
|
r2301 | else: | |
Brian Granger
|
r2302 | os.chmod(self.log_dir, 0777) | |
Brian Granger
|
r2301 | ||
def _security_dir_changed(self, name, old, new): | |||
Brian Granger
|
r2302 | self.check_security_dir() | |
def check_security_dir(self): | |||
if not os.path.isdir(self.security_dir): | |||
os.mkdir(self.security_dir, 0700) | |||
Brian Granger
|
r2301 | else: | |
Brian Granger
|
r2302 | os.chmod(self.security_dir, 0700) | |
Brian Granger
|
r2313 | def _pid_dir_changed(self, name, old, new): | |
self.check_pid_dir() | |||
def check_pid_dir(self): | |||
if not os.path.isdir(self.pid_dir): | |||
os.mkdir(self.pid_dir, 0700) | |||
else: | |||
os.chmod(self.pid_dir, 0700) | |||
Brian Granger
|
r2302 | def check_dirs(self): | |
self.check_security_dir() | |||
self.check_log_dir() | |||
Brian Granger
|
r2313 | self.check_pid_dir() | |
Brian Granger
|
r2301 | ||
def load_config_file(self, filename): | |||
"""Load a config file from the top level of the cluster dir. | |||
Parameters | |||
---------- | |||
filename : unicode or str | |||
The filename only of the config file that must be located in | |||
the top-level of the cluster directory. | |||
""" | |||
loader = PyFileConfigLoader(filename, self.location) | |||
return loader.load_config() | |||
Brian Granger
|
r2302 | def copy_config_file(self, config_file, path=None, overwrite=False): | |
Brian Granger
|
r2301 | """Copy a default config file into the active cluster directory. | |
Default configuration files are kept in :mod:`IPython.config.default`. | |||
This function moves these from that location to the working cluster | |||
directory. | |||
""" | |||
if path is None: | |||
import IPython.config.default | |||
path = IPython.config.default.__file__.split(os.path.sep)[:-1] | |||
path = os.path.sep.join(path) | |||
src = os.path.join(path, config_file) | |||
dst = os.path.join(self.location, config_file) | |||
Brian Granger
|
r2302 | if not os.path.isfile(dst) or overwrite: | |
shutil.copy(src, dst) | |||
Brian Granger
|
r2301 | ||
Brian Granger
|
r2302 | def copy_all_config_files(self, path=None, overwrite=False): | |
Brian Granger
|
r2301 | """Copy all config files into the active cluster directory.""" | |
Brian Granger
|
r2302 | for f in ['ipcontroller_config.py', 'ipengine_config.py', | |
'ipcluster_config.py']: | |||
self.copy_config_file(f, path=path, overwrite=overwrite) | |||
Brian Granger
|
r2301 | ||
@classmethod | |||
Brian Granger
|
r2303 | def create_cluster_dir(csl, cluster_dir): | |
"""Create a new cluster directory given a full path. | |||
Brian Granger
|
r2301 | ||
Brian Granger
|
r2303 | Parameters | |
---------- | |||
cluster_dir : str | |||
The full path to the cluster directory. If it does exist, it will | |||
be used. If not, it will be created. | |||
""" | |||
return ClusterDir(cluster_dir) | |||
@classmethod | |||
def create_cluster_dir_by_profile(cls, path, profile='default'): | |||
"""Create a cluster dir by profile name and path. | |||
Parameters | |||
---------- | |||
path : str | |||
The path (directory) to put the cluster directory in. | |||
profile : str | |||
The name of the profile. The name of the cluster directory will | |||
be "cluster_<profile>". | |||
""" | |||
if not os.path.isdir(path): | |||
raise ClusterDirError('Directory not found: %s' % path) | |||
cluster_dir = os.path.join(path, 'cluster_' + profile) | |||
return ClusterDir(cluster_dir) | |||
@classmethod | |||
def find_cluster_dir_by_profile(cls, ipythondir, profile='default'): | |||
"""Find an existing cluster dir by profile name, return its ClusterDir. | |||
This searches through a sequence of paths for a cluster dir. If it | |||
is not found, a :class:`ClusterDirError` exception will be raised. | |||
The search path algorithm is: | |||
1. ``os.getcwd()`` | |||
2. ``ipythondir`` | |||
3. The directories found in the ":" separated | |||
:env:`IPCLUSTERDIR_PATH` environment variable. | |||
Brian Granger
|
r2301 | ||
Parameters | |||
---------- | |||
Brian Granger
|
r2303 | ipythondir : unicode or str | |
The IPython directory to use. | |||
Brian Granger
|
r2301 | profile : unicode or str | |
The name of the profile. The name of the cluster directory | |||
will be "cluster_<profile>". | |||
""" | |||
dirname = 'cluster_' + profile | |||
Brian Granger
|
r2303 | cluster_dir_paths = os.environ.get('IPCLUSTERDIR_PATH','') | |
if cluster_dir_paths: | |||
cluster_dir_paths = cluster_dir_paths.split(':') | |||
Brian Granger
|
r2301 | else: | |
Brian Granger
|
r2303 | cluster_dir_paths = [] | |
paths = [os.getcwd(), ipythondir] + cluster_dir_paths | |||
for p in paths: | |||
cluster_dir = os.path.join(p, dirname) | |||
if os.path.isdir(cluster_dir): | |||
return ClusterDir(cluster_dir) | |||
else: | |||
raise ClusterDirError('Cluster directory not found in paths: %s' % dirname) | |||
Brian Granger
|
r2301 | ||
@classmethod | |||
def find_cluster_dir(cls, cluster_dir): | |||
"""Find/create a cluster dir and return its ClusterDir. | |||
This will create the cluster directory if it doesn't exist. | |||
Parameters | |||
---------- | |||
cluster_dir : unicode or str | |||
The path of the cluster directory. This is expanded using | |||
:func:`os.path.expandvars` and :func:`os.path.expanduser`. | |||
""" | |||
cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir)) | |||
Brian Granger
|
r2303 | if not os.path.isdir(cluster_dir): | |
raise ClusterDirError('Cluster directory not found: %s' % cluster_dir) | |||
Brian Granger
|
r2301 | return ClusterDir(cluster_dir) | |
class AppWithClusterDirArgParseConfigLoader(ArgParseConfigLoader): | |||
"""Default command line options for IPython cluster applications.""" | |||
def _add_other_arguments(self): | |||
Brian Granger
|
r2314 | self.parser.add_argument('--ipython-dir', | |
Brian Granger
|
r2301 | dest='Global.ipythondir',type=str, | |
help='Set to override default location of Global.ipythondir.', | |||
default=NoConfigDefault, | |||
Brian Granger
|
r2306 | metavar='Global.ipythondir' | |
) | |||
Brian Granger
|
r2314 | self.parser.add_argument('-p', '--profile', | |
Brian Granger
|
r2301 | dest='Global.profile',type=str, | |
help='The string name of the profile to be used. This determines ' | |||
'the name of the cluster dir as: cluster_<profile>. The default profile ' | |||
'is named "default". The cluster directory is resolve this way ' | |||
'if the --cluster-dir option is not used.', | |||
default=NoConfigDefault, | |||
Brian Granger
|
r2306 | metavar='Global.profile' | |
) | |||
Brian Granger
|
r2314 | self.parser.add_argument('--log-level', | |
Brian Granger
|
r2301 | dest="Global.log_level",type=int, | |
help='Set the log level (0,10,20,30,40,50). Default is 30.', | |||
Brian Granger
|
r2303 | default=NoConfigDefault, | |
Brian Granger
|
r2306 | metavar="Global.log_level" | |
) | |||
Brian Granger
|
r2314 | self.parser.add_argument('--cluster-dir', | |
Brian Granger
|
r2301 | dest='Global.cluster_dir',type=str, | |
help='Set the cluster dir. This overrides the logic used by the ' | |||
'--profile option.', | |||
default=NoConfigDefault, | |||
Brian Granger
|
r2306 | metavar='Global.cluster_dir' | |
) | |||
Brian Granger
|
r2314 | self.parser.add_argument('--clean-logs', | |
Brian Granger
|
r2306 | dest='Global.clean_logs', action='store_true', | |
help='Delete old log flies before starting.', | |||
default=NoConfigDefault | |||
) | |||
Brian Granger
|
r2314 | self.parser.add_argument('--no-clean-logs', | |
Brian Granger
|
r2306 | dest='Global.clean_logs', action='store_false', | |
help="Don't Delete old log flies before starting.", | |||
default=NoConfigDefault | |||
) | |||
Brian Granger
|
r2301 | ||
class ApplicationWithClusterDir(Application): | |||
"""An application that puts everything into a cluster directory. | |||
Instead of looking for things in the ipythondir, this type of application | |||
will use its own private directory called the "cluster directory" | |||
for things like config files, log files, etc. | |||
The cluster directory is resolved as follows: | |||
* If the ``--cluster-dir`` option is given, it is used. | |||
* If ``--cluster-dir`` is not given, the application directory is | |||
resolve using the profile name as ``cluster_<profile>``. The search | |||
path for this directory is then i) cwd if it is found there | |||
and ii) in ipythondir otherwise. | |||
The config file for the application is to be put in the cluster | |||
dir and named the value of the ``config_file_name`` class attribute. | |||
""" | |||
Brian Granger
|
r2303 | auto_create_cluster_dir = True | |
Brian Granger
|
r2301 | def create_default_config(self): | |
super(ApplicationWithClusterDir, self).create_default_config() | |||
self.default_config.Global.profile = 'default' | |||
self.default_config.Global.cluster_dir = '' | |||
Brian Granger
|
r2306 | self.default_config.Global.log_to_file = False | |
self.default_config.Global.clean_logs = False | |||
Brian Granger
|
r2301 | ||
def create_command_line_config(self): | |||
"""Create and return a command line config loader.""" | |||
return AppWithClusterDirArgParseConfigLoader( | |||
description=self.description, | |||
version=release.version | |||
) | |||
Brian Granger
|
r2303 | def find_resources(self): | |
"""This resolves the cluster directory. | |||
Brian Granger
|
r2301 | ||
Brian Granger
|
r2303 | This tries to find the cluster directory and if successful, it will | |
have done: | |||
* Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for | |||
the application. | |||
* Sets ``self.cluster_dir`` attribute of the application and config | |||
Brian Granger
|
r2301 | objects. | |
Brian Granger
|
r2303 | ||
The algorithm used for this is as follows: | |||
1. Try ``Global.cluster_dir``. | |||
2. Try using ``Global.profile``. | |||
3. If both of these fail and ``self.auto_create_cluster_dir`` is | |||
``True``, then create the new cluster dir in the IPython directory. | |||
4. If all fails, then raise :class:`ClusterDirError`. | |||
Brian Granger
|
r2301 | """ | |
try: | |||
cluster_dir = self.command_line_config.Global.cluster_dir | |||
except AttributeError: | |||
cluster_dir = self.default_config.Global.cluster_dir | |||
cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir)) | |||
Brian Granger
|
r2303 | try: | |
Brian Granger
|
r2301 | self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir) | |
Brian Granger
|
r2303 | except ClusterDirError: | |
pass | |||
Brian Granger
|
r2301 | else: | |
Brian Granger
|
r2303 | self.log.info('Using existing cluster dir: %s' % \ | |
self.cluster_dir_obj.location | |||
) | |||
self.finish_cluster_dir() | |||
return | |||
try: | |||
self.profile = self.command_line_config.Global.profile | |||
except AttributeError: | |||
self.profile = self.default_config.Global.profile | |||
try: | |||
Brian Granger
|
r2301 | self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile( | |
self.ipythondir, self.profile) | |||
Brian Granger
|
r2303 | except ClusterDirError: | |
pass | |||
else: | |||
self.log.info('Using existing cluster dir: %s' % \ | |||
self.cluster_dir_obj.location | |||
) | |||
self.finish_cluster_dir() | |||
return | |||
if self.auto_create_cluster_dir: | |||
self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile( | |||
self.ipythondir, self.profile | |||
) | |||
self.log.info('Creating new cluster dir: %s' % \ | |||
self.cluster_dir_obj.location | |||
) | |||
self.finish_cluster_dir() | |||
else: | |||
raise ClusterDirError('Could not find a valid cluster directory.') | |||
Brian Granger
|
r2301 | ||
Brian Granger
|
r2303 | def finish_cluster_dir(self): | |
Brian Granger
|
r2301 | # Set the cluster directory | |
self.cluster_dir = self.cluster_dir_obj.location | |||
# These have to be set because they could be different from the one | |||
# that we just computed. Because command line has the highest | |||
# priority, this will always end up in the master_config. | |||
self.default_config.Global.cluster_dir = self.cluster_dir | |||
self.command_line_config.Global.cluster_dir = self.cluster_dir | |||
# Set the search path to the cluster directory | |||
self.config_file_paths = (self.cluster_dir,) | |||
Brian Granger
|
r2303 | ||
def find_config_file_name(self): | |||
"""Find the config file name for this application.""" | |||
# For this type of Application it should be set as a class attribute. | |||
if not hasattr(self, 'config_file_name'): | |||
self.log.critical("No config filename found") | |||
def find_config_file_paths(self): | |||
# Set the search path to the cluster directory | |||
self.config_file_paths = (self.cluster_dir,) | |||
Brian Granger
|
r2306 | def pre_construct(self): | |
# The log and security dirs were set earlier, but here we put them | |||
# into the config and log them. | |||
config = self.master_config | |||
sdir = self.cluster_dir_obj.security_dir | |||
self.security_dir = config.Global.security_dir = sdir | |||
ldir = self.cluster_dir_obj.log_dir | |||
self.log_dir = config.Global.log_dir = ldir | |||
Brian Granger
|
r2313 | pdir = self.cluster_dir_obj.pid_dir | |
self.pid_dir = config.Global.pid_dir = pdir | |||
Brian Granger
|
r2306 | self.log.info("Cluster directory set to: %s" % self.cluster_dir) | |
def start_logging(self): | |||
# Remove old log files | |||
if self.master_config.Global.clean_logs: | |||
log_dir = self.master_config.Global.log_dir | |||
for f in os.listdir(log_dir): | |||
if f.startswith(self.name + '-') and f.endswith('.log'): | |||
os.remove(os.path.join(log_dir, f)) | |||
# Start logging to the new log file | |||
if self.master_config.Global.log_to_file: | |||
log_filename = self.name + '-' + str(os.getpid()) + '.log' | |||
logfile = os.path.join(self.log_dir, log_filename) | |||
open_log_file = open(logfile, 'w') | |||
else: | |||
open_log_file = sys.stdout | |||
log.startLogging(open_log_file) | |||
Brian Granger
|
r2313 | ||
bgranger
|
r2318 | def write_pid_file(self, overwrite=False): | |
Brian Granger
|
r2313 | """Create a .pid file in the pid_dir with my pid. | |
This must be called after pre_construct, which sets `self.pid_dir`. | |||
This raises :exc:`PIDFileError` if the pid file exists already. | |||
""" | |||
pid_file = os.path.join(self.pid_dir, self.name + '.pid') | |||
if os.path.isfile(pid_file): | |||
pid = self.get_pid_from_file() | |||
bgranger
|
r2318 | if not overwrite: | |
raise PIDFileError( | |||
'The pid file [%s] already exists. \nThis could mean that this ' | |||
'server is already running with [pid=%s].' % (pid_file, pid) | |||
) | |||
Brian Granger
|
r2313 | with open(pid_file, 'w') as f: | |
self.log.info("Creating pid file: %s" % pid_file) | |||
f.write(repr(os.getpid())+'\n') | |||
def remove_pid_file(self): | |||
"""Remove the pid file. | |||
This should be called at shutdown by registering a callback with | |||
:func:`reactor.addSystemEventTrigger`. | |||
""" | |||
pid_file = os.path.join(self.pid_dir, self.name + '.pid') | |||
if os.path.isfile(pid_file): | |||
try: | |||
self.log.info("Removing pid file: %s" % pid_file) | |||
os.remove(pid_file) | |||
except: | |||
bgranger
|
r2318 | self.log.warn("Error removing the pid file: %s" % pid_file) | |
raise | |||
Brian Granger
|
r2313 | ||
def get_pid_from_file(self): | |||
"""Get the pid from the pid file. | |||
If the pid file doesn't exist a :exc:`PIDFileError` is raised. | |||
""" | |||
pid_file = os.path.join(self.pid_dir, self.name + '.pid') | |||
if os.path.isfile(pid_file): | |||
with open(pid_file, 'r') as f: | |||
pid = int(f.read().strip()) | |||
return pid | |||
else: | |||
raise PIDFileError('pid file not found: %s' % pid_file) |