Make logging unicode-aware

Closes gh-1777

IPython/core/logger.py

@@ -1,217 +1,220 b''
 """Logger class for IPython's logging facilities.
 """
 
 #*****************************************************************************
 #       Copyright (C) 2001 Janko Hauser <[e-mail]> and
 #       Copyright (C) 2001-2006 Fernando Perez <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 #****************************************************************************
 # Modules and globals
 
 # Python standard modules
 import glob
+import io
 import os
 import time
 
+from IPython.utils.py3compat import str_to_unicode
+
 #****************************************************************************
 # FIXME: This class isn't a mixin anymore, but it still needs attributes from
 # ipython and does input cache management.  Finish cleanup later...
 
 class Logger(object):
     """A Logfile class with different policies for file creation"""
 
-    def __init__(self, home_dir, logfname='Logger.log', loghead='',
+    def __init__(self, home_dir, logfname='Logger.log', loghead=u'',
                  logmode='over'):
 
         # this is the full ipython instance, we need some attributes from it
         # which won't exist until later. What a mess, clean up later...
         self.home_dir = home_dir
 
         self.logfname = logfname
         self.loghead = loghead
         self.logmode = logmode
         self.logfile = None
 
         # Whether to log raw or processed input
         self.log_raw_input = False
 
         # whether to also log output
         self.log_output = False
 
         # whether to put timestamps before each log entry
         self.timestamp = False
 
         # activity control flags
         self.log_active = False
 
     # logmode is a validated property
     def _set_mode(self,mode):
         if mode not in ['append','backup','global','over','rotate']:
             raise ValueError,'invalid log mode %s given' % mode
         self._logmode = mode
 
     def _get_mode(self):
         return self._logmode
 
     logmode = property(_get_mode,_set_mode)
 
     def logstart(self,logfname=None,loghead=None,logmode=None,
                  log_output=False,timestamp=False,log_raw_input=False):
         """Generate a new log-file with a default header.
 
         Raises RuntimeError if the log has already been started"""
 
         if self.logfile is not None:
             raise RuntimeError('Log file is already active: %s' %
                                self.logfname)
 
         # The parameters can override constructor defaults
         if logfname is not None: self.logfname = logfname
         if loghead is not None: self.loghead = loghead
         if logmode is not None: self.logmode = logmode
 
         # Parameters not part of the constructor
         self.timestamp = timestamp
         self.log_output = log_output
         self.log_raw_input = log_raw_input
 
         # init depending on the log mode requested
         isfile = os.path.isfile
         logmode = self.logmode
 
         if logmode == 'append':
-            self.logfile = open(self.logfname,'a')
+            self.logfile = io.open(self.logfname, 'a', encoding='utf-8')
 
         elif logmode == 'backup':
             if isfile(self.logfname):
                 backup_logname = self.logfname+'~'
                 # Manually remove any old backup, since os.rename may fail
                 # under Windows.
                 if isfile(backup_logname):
                     os.remove(backup_logname)
                 os.rename(self.logfname,backup_logname)
-            self.logfile = open(self.logfname,'w')
+            self.logfile = io.open(self.logfname, 'w', encoding='utf-8')
 
         elif logmode == 'global':
             self.logfname = os.path.join(self.home_dir,self.logfname)
-            self.logfile = open(self.logfname, 'a')
+            self.logfile = io.open(self.logfname, 'a', encoding='utf-8')
 
         elif logmode == 'over':
             if isfile(self.logfname):
                 os.remove(self.logfname)
-            self.logfile = open(self.logfname,'w')
+            self.logfile = io.open(self.logfname,'w', encoding='utf-8')
 
         elif logmode == 'rotate':
             if isfile(self.logfname):
                 if isfile(self.logfname+'.001~'):
                     old = glob.glob(self.logfname+'.*~')
                     old.sort()
                     old.reverse()
                     for f in old:
                         root, ext = os.path.splitext(f)
                         num = int(ext[1:-1])+1
                         os.rename(f, root+'.'+`num`.zfill(3)+'~')
                 os.rename(self.logfname, self.logfname+'.001~')
-            self.logfile = open(self.logfname,'w')
+            self.logfile = io.open(self.logfname, 'w', encoding='utf-8')
 
         if logmode != 'append':
             self.logfile.write(self.loghead)
 
         self.logfile.flush()
         self.log_active = True
 
     def switch_log(self,val):
         """Switch logging on/off. val should be ONLY a boolean."""
 
         if val not in [False,True,0,1]:
             raise ValueError, \
                   'Call switch_log ONLY with a boolean argument, not with:',val
 
         label = {0:'OFF',1:'ON',False:'OFF',True:'ON'}
 
         if self.logfile is None:
             print """
 Logging hasn't been started yet (use logstart for that).
 
 %logon/%logoff are for temporarily starting and stopping logging for a logfile
 which already exists. But you must first start the logging process with
 %logstart (optionally giving a logfile name)."""
 
         else:
             if self.log_active == val:
                 print 'Logging is already',label[val]
             else:
                 print 'Switching logging',label[val]
                 self.log_active = not self.log_active
                 self.log_active_out = self.log_active
 
     def logstate(self):
         """Print a status message about the logger."""
         if self.logfile is None:
             print 'Logging has not been activated.'
         else:
             state = self.log_active and 'active' or 'temporarily suspended'
             print 'Filename       :',self.logfname
             print 'Mode           :',self.logmode
             print 'Output logging :',self.log_output
             print 'Raw input log  :',self.log_raw_input
             print 'Timestamping   :',self.timestamp
             print 'State          :',state
 
     def log(self, line_mod, line_ori):
         """Write the sources to a log.
 
         Inputs:
 
         - line_mod: possibly modified input, such as the transformations made
         by input prefilters or input handlers of various kinds.  This should
         always be valid Python.
 
         - line_ori: unmodified input line from the user.  This is not
         necessarily valid Python.
         """
 
         # Write the log line, but decide which one according to the
         # log_raw_input flag, set when the log is started.
         if self.log_raw_input:
             self.log_write(line_ori)
         else:
             self.log_write(line_mod)
 
     def log_write(self, data, kind='input'):
         """Write data to the log file, if active"""
 
         #print 'data: %r' % data # dbg
         if self.log_active and data:
             write = self.logfile.write
             if kind=='input':
                 if self.timestamp:
-                    write(time.strftime('# %a, %d %b %Y %H:%M:%S\n',
-                                        time.localtime()))
+                    write(str_to_unicode(time.strftime('# %a, %d %b %Y %H:%M:%S\n',
+                                        time.localtime())))
                 write(data)
             elif kind=='output' and self.log_output:
-                odata = '\n'.join(['#[Out]# %s' % s
+                odata = u'\n'.join([u'#[Out]# %s' % s
                                    for s in data.splitlines()])
-                write('%s\n' % odata)
+                write(u'%s\n' % odata)
             self.logfile.flush()
 
     def logstop(self):
         """Fully stop logging and close log file.
 
         In order to start logging again, a new logstart() call needs to be
         made, possibly (though not necessarily) with a new filename, mode and
         other options."""
 
         if self.logfile is not None:
             self.logfile.close()
             self.logfile = None
         else:
             print "Logging hadn't been started."
         self.log_active = False
 
     # For backwards compatibility, in case anyone was using this.
     close_log = logstop

IPython/core/magics/logging.py

@@ -1,169 +1,170 b''
 """Implementation of magic functions for IPython's own logging.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012 The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import os
 import sys
 
 # Our own packages
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.utils.warn import warn
+from IPython.utils.py3compat import str_to_unicode
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 @magics_class
 class LoggingMagics(Magics):
     """Magics related to all logging machinery."""
 
     @line_magic
     def logstart(self, parameter_s=''):
         """Start logging anywhere in a session.
 
         %logstart [-o|-r|-t] [log_name [log_mode]]
 
         If no name is given, it defaults to a file named 'ipython_log.py' in your
         current directory, in 'rotate' mode (see below).
 
         '%logstart name' saves to file 'name' in 'backup' mode.  It saves your
         history up to that point and then continues logging.
 
         %logstart takes a second optional parameter: logging mode. This can be one
         of (note that the modes are given unquoted):\\
           append: well, that says it.\\
           backup: rename (if exists) to name~ and start name.\\
           global: single logfile in your home dir, appended to.\\
           over  : overwrite existing log.\\
           rotate: create rotating logs name.1~, name.2~, etc.
 
         Options:
 
           -o: log also IPython's output.  In this mode, all commands which
           generate an Out[NN] prompt are recorded to the logfile, right after
           their corresponding input line.  The output lines are always
           prepended with a '#[Out]# ' marker, so that the log remains valid
           Python code.
 
           Since this marker is always the same, filtering only the output from
           a log is very easy, using for example a simple awk call::
 
             awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
 
           -r: log 'raw' input.  Normally, IPython's logs contain the processed
           input, so that user lines are logged in their final form, converted
           into valid Python.  For example, %Exit is logged as
           _ip.magic("Exit").  If the -r flag is given, all input is logged
           exactly as typed, with no transformations applied.
 
           -t: put timestamps before each input line logged (these are put in
           comments)."""
 
         opts,par = self.parse_options(parameter_s,'ort')
         log_output = 'o' in opts
         log_raw_input = 'r' in opts
         timestamp = 't' in opts
 
         logger = self.shell.logger
 
         # if no args are given, the defaults set in the logger constructor by
         # ipython remain valid
         if par:
             try:
                 logfname,logmode = par.split()
             except:
                 logfname = par
                 logmode = 'backup'
         else:
             logfname = logger.logfname
             logmode = logger.logmode
         # put logfname into rc struct as if it had been called on the command
         # line, so it ends up saved in the log header Save it in case we need
         # to restore it...
         old_logfile = self.shell.logfile
         if logfname:
             logfname = os.path.expanduser(logfname)
         self.shell.logfile = logfname
 
-        loghead = '# IPython log file\n\n'
+        loghead = u'# IPython log file\n\n'
         try:
             logger.logstart(logfname, loghead, logmode, log_output, timestamp,
                             log_raw_input)
         except:
             self.shell.logfile = old_logfile
             warn("Couldn't start log: %s" % sys.exc_info()[1])
         else:
             # log input history up to this point, optionally interleaving
             # output if requested
 
             if timestamp:
                 # disable timestamping for the previous history, since we've
                 # lost those already (no time machine here).
                 logger.timestamp = False
 
             if log_raw_input:
                 input_hist = self.shell.history_manager.input_hist_raw
             else:
                 input_hist = self.shell.history_manager.input_hist_parsed
 
             if log_output:
                 log_write = logger.log_write
                 output_hist = self.shell.history_manager.output_hist
                 for n in range(1,len(input_hist)-1):
-                    log_write(input_hist[n].rstrip() + '\n')
+                    log_write(input_hist[n].rstrip() + u'\n')
                     if n in output_hist:
-                        log_write(repr(output_hist[n]),'output')
+                        log_write(str_to_unicode(repr(output_hist[n])),'output')
             else:
-                logger.log_write('\n'.join(input_hist[1:]))
-                logger.log_write('\n')
+                logger.log_write(u'\n'.join(input_hist[1:]))
+                logger.log_write(u'\n')
             if timestamp:
                 # re-enable timestamping
                 logger.timestamp = True
 
             print ('Activating auto-logging. '
                    'Current session state plus future input saved.')
             logger.logstate()
 
     @line_magic
     def logstop(self, parameter_s=''):
         """Fully stop logging and close log file.
 
         In order to start logging again, a new %logstart call needs to be made,
         possibly (though not necessarily) with a new filename, mode and other
         options."""
-        self.logger.logstop()
+        self.shell.logger.logstop()
 
     @line_magic
     def logoff(self, parameter_s=''):
         """Temporarily stop logging.
 
         You must have previously started logging."""
         self.shell.logger.switch_log(0)
 
     @line_magic
     def logon(self, parameter_s=''):
         """Restart logging.
 
         This function is for restarting logging which you've temporarily
         stopped with %logoff. For starting logging for the first time, you
         must use the %logstart function, which allows you to specify an
         optional log filename."""
 
         self.shell.logger.switch_log(1)
 
     @line_magic
     def logstate(self, parameter_s=''):
         """Print the status of the logging system."""
 
         self.shell.logger.logstate()