py3: replace os.environ with encoding.environ (part 5 of 5)

hgext/color.py

@@ -1,717 +1,719 b''
 # color.py color output for Mercurial commands
 #
 # Copyright (C) 2007 Kevin Christen <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 '''colorize output from some commands
 
 The color extension colorizes output from several Mercurial commands.
 For example, the diff command shows additions in green and deletions
 in red, while the status command shows modified files in magenta. Many
 other commands have analogous colors. It is possible to customize
 these colors.
 
 Effects
 -------
 
 Other effects in addition to color, like bold and underlined text, are
 also available. By default, the terminfo database is used to find the
 terminal codes used to change color and effect.  If terminfo is not
 available, then effects are rendered with the ECMA-48 SGR control
 function (aka ANSI escape codes).
 
 The available effects in terminfo mode are 'blink', 'bold', 'dim',
 'inverse', 'invisible', 'italic', 'standout', and 'underline'; in
 ECMA-48 mode, the options are 'bold', 'inverse', 'italic', and
 'underline'.  How each is rendered depends on the terminal emulator.
 Some may not be available for a given terminal type, and will be
 silently ignored.
 
 If the terminfo entry for your terminal is missing codes for an effect
 or has the wrong codes, you can add or override those codes in your
 configuration::
 
   [color]
   terminfo.dim = \E[2m
 
 where '\E' is substituted with an escape character.
 
 Labels
 ------
 
 Text receives color effects depending on the labels that it has. Many
 default Mercurial commands emit labelled text. You can also define
 your own labels in templates using the label function, see :hg:`help
 templates`. A single portion of text may have more than one label. In
 that case, effects given to the last label will override any other
 effects. This includes the special "none" effect, which nullifies
 other effects.
 
 Labels are normally invisible. In order to see these labels and their
 position in the text, use the global --color=debug option. The same
 anchor text may be associated to multiple labels, e.g.
 
   [log.changeset changeset.secret|changeset:   22611:6f0a53c8f587]
 
 The following are the default effects for some default labels. Default
 effects may be overridden from your configuration file::
 
   [color]
   status.modified = blue bold underline red_background
   status.added = green bold
   status.removed = red bold blue_background
   status.deleted = cyan bold underline
   status.unknown = magenta bold underline
   status.ignored = black bold
 
   # 'none' turns off all effects
   status.clean = none
   status.copied = none
 
   qseries.applied = blue bold underline
   qseries.unapplied = black bold
   qseries.missing = red bold
 
   diff.diffline = bold
   diff.extended = cyan bold
   diff.file_a = red bold
   diff.file_b = green bold
   diff.hunk = magenta
   diff.deleted = red
   diff.inserted = green
   diff.changed = white
   diff.tab =
   diff.trailingwhitespace = bold red_background
 
   # Blank so it inherits the style of the surrounding label
   changeset.public =
   changeset.draft =
   changeset.secret =
 
   resolve.unresolved = red bold
   resolve.resolved = green bold
 
   bookmarks.active = green
 
   branches.active = none
   branches.closed = black bold
   branches.current = green
   branches.inactive = none
 
   tags.normal = green
   tags.local = black bold
 
   rebase.rebased = blue
   rebase.remaining = red bold
 
   shelve.age = cyan
   shelve.newest = green bold
   shelve.name = blue bold
 
   histedit.remaining = red bold
 
 Custom colors
 -------------
 
 Because there are only eight standard colors, this module allows you
 to define color names for other color slots which might be available
 for your terminal type, assuming terminfo mode.  For instance::
 
   color.brightblue = 12
   color.pink = 207
   color.orange = 202
 
 to set 'brightblue' to color slot 12 (useful for 16 color terminals
 that have brighter colors defined in the upper eight) and, 'pink' and
 'orange' to colors in 256-color xterm's default color cube.  These
 defined colors may then be used as any of the pre-defined eight,
 including appending '_background' to set the background to that color.
 
 Modes
 -----
 
 By default, the color extension will use ANSI mode (or win32 mode on
 Windows) if it detects a terminal. To override auto mode (to enable
 terminfo mode, for example), set the following configuration option::
 
   [color]
   mode = terminfo
 
 Any value other than 'ansi', 'win32', 'terminfo', or 'auto' will
 disable color.
 
 Note that on some systems, terminfo mode may cause problems when using
 color with the pager extension and less -R. less with the -R option
 will only display ECMA-48 color codes, and terminfo mode may sometimes
 emit codes that less doesn't understand. You can work around this by
 either using ansi mode (or auto mode), or by using less -r (which will
 pass through all terminal control codes, not just color control
 codes).
 
 On some systems (such as MSYS in Windows), the terminal may support
 a different color mode than the pager (activated via the "pager"
 extension). It is possible to define separate modes depending on whether
 the pager is active::
 
   [color]
   mode = auto
   pagermode = ansi
 
 If ``pagermode`` is not defined, the ``mode`` will be used.
 '''
 
 from __future__ import absolute_import
 
 import os
 
 from mercurial.i18n import _
 from mercurial import (
     cmdutil,
     commands,
     dispatch,
+    encoding,
     extensions,
     subrepo,
     ui as uimod,
     util,
 )
 
 cmdtable = {}
 command = cmdutil.command(cmdtable)
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 # start and stop parameters for effects
 _effects = {'none': 0, 'black': 30, 'red': 31, 'green': 32, 'yellow': 33,
             'blue': 34, 'magenta': 35, 'cyan': 36, 'white': 37, 'bold': 1,
             'italic': 3, 'underline': 4, 'inverse': 7, 'dim': 2,
             'black_background': 40, 'red_background': 41,
             'green_background': 42, 'yellow_background': 43,
             'blue_background': 44, 'purple_background': 45,
             'cyan_background': 46, 'white_background': 47}
 
 def _terminfosetup(ui, mode):
     '''Initialize terminfo data and the terminal if we're in terminfo mode.'''
 
     global _terminfo_params
     # If we failed to load curses, we go ahead and return.
     if not _terminfo_params:
         return
     # Otherwise, see what the config file says.
     if mode not in ('auto', 'terminfo'):
         return
 
     _terminfo_params.update((key[6:], (False, int(val), ''))
         for key, val in ui.configitems('color')
         if key.startswith('color.'))
     _terminfo_params.update((key[9:], (True, '', val.replace('\\E', '\x1b')))
         for key, val in ui.configitems('color')
         if key.startswith('terminfo.'))
 
     try:
         curses.setupterm()
     except curses.error as e:
         _terminfo_params = {}
         return
 
     for key, (b, e, c) in _terminfo_params.items():
         if not b:
             continue
         if not c and not curses.tigetstr(e):
             # Most terminals don't support dim, invis, etc, so don't be
             # noisy and use ui.debug().
             ui.debug("no terminfo entry for %s\n" % e)
             del _terminfo_params[key]
     if not curses.tigetstr('setaf') or not curses.tigetstr('setab'):
         # Only warn about missing terminfo entries if we explicitly asked for
         # terminfo mode.
         if mode == "terminfo":
             ui.warn(_("no terminfo entry for setab/setaf: reverting to "
               "ECMA-48 color\n"))
         _terminfo_params = {}
 
 def _modesetup(ui, coloropt):
     global _terminfo_params
 
     if coloropt == 'debug':
         return 'debug'
 
     auto = (coloropt == 'auto')
     always = not auto and util.parsebool(coloropt)
     if not always and not auto:
         return None
 
-    formatted = always or (os.environ.get('TERM') != 'dumb' and ui.formatted())
+    formatted = (always or (encoding.environ.get('TERM') != 'dumb'
+                 and ui.formatted()))
 
     mode = ui.config('color', 'mode', 'auto')
 
     # If pager is active, color.pagermode overrides color.mode.
     if getattr(ui, 'pageractive', False):
         mode = ui.config('color', 'pagermode', mode)
 
     realmode = mode
     if mode == 'auto':
         if os.name == 'nt':
-            term = os.environ.get('TERM')
+            term = encoding.environ.get('TERM')
             # TERM won't be defined in a vanilla cmd.exe environment.
 
             # UNIX-like environments on Windows such as Cygwin and MSYS will
             # set TERM. They appear to make a best effort attempt at setting it
             # to something appropriate. However, not all environments with TERM
             # defined support ANSI. Since "ansi" could result in terminal
             # gibberish, we error on the side of selecting "win32". However, if
             # w32effects is not defined, we almost certainly don't support
             # "win32", so don't even try.
             if (term and 'xterm' in term) or not w32effects:
                 realmode = 'ansi'
             else:
                 realmode = 'win32'
         else:
             realmode = 'ansi'
 
     def modewarn():
         # only warn if color.mode was explicitly set and we're in
         # a formatted terminal
         if mode == realmode and ui.formatted():
             ui.warn(_('warning: failed to set color mode to %s\n') % mode)
 
     if realmode == 'win32':
         _terminfo_params = {}
         if not w32effects:
             modewarn()
             return None
         _effects.update(w32effects)
     elif realmode == 'ansi':
         _terminfo_params = {}
     elif realmode == 'terminfo':
         _terminfosetup(ui, mode)
         if not _terminfo_params:
             ## FIXME Shouldn't we return None in this case too?
             modewarn()
             realmode = 'ansi'
     else:
         return None
 
     if always or (auto and formatted):
         return realmode
     return None
 
 try:
     import curses
     # Mapping from effect name to terminfo attribute name (or raw code) or
     # color number.  This will also force-load the curses module.
     _terminfo_params = {'none': (True, 'sgr0', ''),
                         'standout': (True, 'smso', ''),
                         'underline': (True, 'smul', ''),
                         'reverse': (True, 'rev', ''),
                         'inverse': (True, 'rev', ''),
                         'blink': (True, 'blink', ''),
                         'dim': (True, 'dim', ''),
                         'bold': (True, 'bold', ''),
                         'invisible': (True, 'invis', ''),
                         'italic': (True, 'sitm', ''),
                         'black': (False, curses.COLOR_BLACK, ''),
                         'red': (False, curses.COLOR_RED, ''),
                         'green': (False, curses.COLOR_GREEN, ''),
                         'yellow': (False, curses.COLOR_YELLOW, ''),
                         'blue': (False, curses.COLOR_BLUE, ''),
                         'magenta': (False, curses.COLOR_MAGENTA, ''),
                         'cyan': (False, curses.COLOR_CYAN, ''),
                         'white': (False, curses.COLOR_WHITE, '')}
 except ImportError:
     _terminfo_params = {}
 
 _styles = {'grep.match': 'red bold',
            'grep.linenumber': 'green',
            'grep.rev': 'green',
            'grep.change': 'green',
            'grep.sep': 'cyan',
            'grep.filename': 'magenta',
            'grep.user': 'magenta',
            'grep.date': 'magenta',
            'bookmarks.active': 'green',
            'branches.active': 'none',
            'branches.closed': 'black bold',
            'branches.current': 'green',
            'branches.inactive': 'none',
            'diff.changed': 'white',
            'diff.deleted': 'red',
            'diff.diffline': 'bold',
            'diff.extended': 'cyan bold',
            'diff.file_a': 'red bold',
            'diff.file_b': 'green bold',
            'diff.hunk': 'magenta',
            'diff.inserted': 'green',
            'diff.tab': '',
            'diff.trailingwhitespace': 'bold red_background',
            'changeset.public' : '',
            'changeset.draft' : '',
            'changeset.secret' : '',
            'diffstat.deleted': 'red',
            'diffstat.inserted': 'green',
            'histedit.remaining': 'red bold',
            'ui.prompt': 'yellow',
            'log.changeset': 'yellow',
            'patchbomb.finalsummary': '',
            'patchbomb.from': 'magenta',
            'patchbomb.to': 'cyan',
            'patchbomb.subject': 'green',
            'patchbomb.diffstats': '',
            'rebase.rebased': 'blue',
            'rebase.remaining': 'red bold',
            'resolve.resolved': 'green bold',
            'resolve.unresolved': 'red bold',
            'shelve.age': 'cyan',
            'shelve.newest': 'green bold',
            'shelve.name': 'blue bold',
            'status.added': 'green bold',
            'status.clean': 'none',
            'status.copied': 'none',
            'status.deleted': 'cyan bold underline',
            'status.ignored': 'black bold',
            'status.modified': 'blue bold',
            'status.removed': 'red bold',
            'status.unknown': 'magenta bold underline',
            'tags.normal': 'green',
            'tags.local': 'black bold'}
 
 
 def _effect_str(effect):
     '''Helper function for render_effects().'''
 
     bg = False
     if effect.endswith('_background'):
         bg = True
         effect = effect[:-11]
     try:
         attr, val, termcode = _terminfo_params[effect]
     except KeyError:
         return ''
     if attr:
         if termcode:
             return termcode
         else:
             return curses.tigetstr(val)
     elif bg:
         return curses.tparm(curses.tigetstr('setab'), val)
     else:
         return curses.tparm(curses.tigetstr('setaf'), val)
 
 def render_effects(text, effects):
     'Wrap text in commands to turn on each effect.'
     if not text:
         return text
     if not _terminfo_params:
         start = [str(_effects[e]) for e in ['none'] + effects.split()]
         start = '\033[' + ';'.join(start) + 'm'
         stop = '\033[' + str(_effects['none']) + 'm'
     else:
         start = ''.join(_effect_str(effect)
                         for effect in ['none'] + effects.split())
         stop = _effect_str('none')
     return ''.join([start, text, stop])
 
 def extstyles():
     for name, ext in extensions.extensions():
         _styles.update(getattr(ext, 'colortable', {}))
 
 def valideffect(effect):
     'Determine if the effect is valid or not.'
     good = False
     if not _terminfo_params and effect in _effects:
         good = True
     elif effect in _terminfo_params or effect[:-11] in _terminfo_params:
         good = True
     return good
 
 def configstyles(ui):
     for status, cfgeffects in ui.configitems('color'):
         if '.' not in status or status.startswith(('color.', 'terminfo.')):
             continue
         cfgeffects = ui.configlist('color', status)
         if cfgeffects:
             good = []
             for e in cfgeffects:
                 if valideffect(e):
                     good.append(e)
                 else:
                     ui.warn(_("ignoring unknown color/effect %r "
                               "(configured in color.%s)\n")
                             % (e, status))
             _styles[status] = ' '.join(good)
 
 class colorui(uimod.ui):
     _colormode = 'ansi'
     def write(self, *args, **opts):
         if self._colormode is None:
             return super(colorui, self).write(*args, **opts)
 
         label = opts.get('label', '')
         if self._buffers and not opts.get('prompt', False):
             if self._bufferapplylabels:
                 self._buffers[-1].extend(self.label(a, label) for a in args)
             else:
                 self._buffers[-1].extend(args)
         elif self._colormode == 'win32':
             for a in args:
                 win32print(a, super(colorui, self).write, **opts)
         else:
             return super(colorui, self).write(
                 *[self.label(a, label) for a in args], **opts)
 
     def write_err(self, *args, **opts):
         if self._colormode is None:
             return super(colorui, self).write_err(*args, **opts)
 
         label = opts.get('label', '')
         if self._bufferstates and self._bufferstates[-1][0]:
             return self.write(*args, **opts)
         if self._colormode == 'win32':
             for a in args:
                 win32print(a, super(colorui, self).write_err, **opts)
         else:
             return super(colorui, self).write_err(
                 *[self.label(a, label) for a in args], **opts)
 
     def showlabel(self, msg, label):
         if label and msg:
             if msg[-1] == '\n':
                 return "[%s|%s]\n" % (label, msg[:-1])
             else:
                 return "[%s|%s]" % (label, msg)
         else:
             return msg
 
     def label(self, msg, label):
         if self._colormode is None:
             return super(colorui, self).label(msg, label)
 
         if self._colormode == 'debug':
             return self.showlabel(msg, label)
 
         effects = []
         for l in label.split():
             s = _styles.get(l, '')
             if s:
                 effects.append(s)
             elif valideffect(l):
                 effects.append(l)
         effects = ' '.join(effects)
         if effects:
             return '\n'.join([render_effects(line, effects)
                               for line in msg.split('\n')])
         return msg
 
 def uisetup(ui):
     if ui.plain():
         return
     if not isinstance(ui, colorui):
         colorui.__bases__ = (ui.__class__,)
         ui.__class__ = colorui
     def colorcmd(orig, ui_, opts, cmd, cmdfunc):
         mode = _modesetup(ui_, opts['color'])
         colorui._colormode = mode
         if mode and mode != 'debug':
             extstyles()
             configstyles(ui_)
         return orig(ui_, opts, cmd, cmdfunc)
     def colorgit(orig, gitsub, commands, env=None, stream=False, cwd=None):
         if gitsub.ui._colormode and len(commands) and commands[0] == "diff":
                 # insert the argument in the front,
                 # the end of git diff arguments is used for paths
                 commands.insert(1, '--color')
         return orig(gitsub, commands, env, stream, cwd)
     extensions.wrapfunction(dispatch, '_runcommand', colorcmd)
     extensions.wrapfunction(subrepo.gitsubrepo, '_gitnodir', colorgit)
 
 def extsetup(ui):
     commands.globalopts.append(
         ('', 'color', 'auto',
          # i18n: 'always', 'auto', 'never', and 'debug' are keywords
          # and should not be translated
          _("when to colorize (boolean, always, auto, never, or debug)"),
          _('TYPE')))
 
 @command('debugcolor',
         [('', 'style', None, _('show all configured styles'))],
         'hg debugcolor')
 def debugcolor(ui, repo, **opts):
     """show available color, effects or style"""
     ui.write(('color mode: %s\n') % ui._colormode)
     if opts.get('style'):
         return _debugdisplaystyle(ui)
     else:
         return _debugdisplaycolor(ui)
 
 def _debugdisplaycolor(ui):
     global _styles
     oldstyle = _styles
     try:
         _styles = {}
         for effect in _effects.keys():
             _styles[effect] = effect
         if _terminfo_params:
             for k, v in ui.configitems('color'):
                 if k.startswith('color.'):
                     _styles[k] = k[6:]
                 elif k.startswith('terminfo.'):
                     _styles[k] = k[9:]
         ui.write(_('available colors:\n'))
         # sort label with a '_' after the other to group '_background' entry.
         items = sorted(_styles.items(),
                        key=lambda i: ('_' in i[0], i[0], i[1]))
         for colorname, label in items:
             ui.write(('%s\n') % colorname, label=label)
     finally:
         _styles = oldstyle
 
 def _debugdisplaystyle(ui):
     ui.write(_('available style:\n'))
     width = max(len(s) for s in _styles)
     for label, effects in sorted(_styles.items()):
         ui.write('%s' % label, label=label)
         if effects:
             # 50
             ui.write(': ')
             ui.write(' ' * (max(0, width - len(label))))
             ui.write(', '.join(ui.label(e, e) for e in effects.split()))
         ui.write('\n')
 
 if os.name != 'nt':
     w32effects = None
 else:
     import ctypes
     import re
 
     _kernel32 = ctypes.windll.kernel32
 
     _WORD = ctypes.c_ushort
 
     _INVALID_HANDLE_VALUE = -1
 
     class _COORD(ctypes.Structure):
         _fields_ = [('X', ctypes.c_short),
                     ('Y', ctypes.c_short)]
 
     class _SMALL_RECT(ctypes.Structure):
         _fields_ = [('Left', ctypes.c_short),
                     ('Top', ctypes.c_short),
                     ('Right', ctypes.c_short),
                     ('Bottom', ctypes.c_short)]
 
     class _CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure):
         _fields_ = [('dwSize', _COORD),
                     ('dwCursorPosition', _COORD),
                     ('wAttributes', _WORD),
                     ('srWindow', _SMALL_RECT),
                     ('dwMaximumWindowSize', _COORD)]
 
     _STD_OUTPUT_HANDLE = 0xfffffff5 # (DWORD)-11
     _STD_ERROR_HANDLE = 0xfffffff4  # (DWORD)-12
 
     _FOREGROUND_BLUE = 0x0001
     _FOREGROUND_GREEN = 0x0002
     _FOREGROUND_RED = 0x0004
     _FOREGROUND_INTENSITY = 0x0008
 
     _BACKGROUND_BLUE = 0x0010
     _BACKGROUND_GREEN = 0x0020
     _BACKGROUND_RED = 0x0040
     _BACKGROUND_INTENSITY = 0x0080
 
     _COMMON_LVB_REVERSE_VIDEO = 0x4000
     _COMMON_LVB_UNDERSCORE = 0x8000
 
     # http://msdn.microsoft.com/en-us/library/ms682088%28VS.85%29.aspx
     w32effects = {
         'none': -1,
         'black': 0,
         'red': _FOREGROUND_RED,
         'green': _FOREGROUND_GREEN,
         'yellow': _FOREGROUND_RED | _FOREGROUND_GREEN,
         'blue': _FOREGROUND_BLUE,
         'magenta': _FOREGROUND_BLUE | _FOREGROUND_RED,
         'cyan': _FOREGROUND_BLUE | _FOREGROUND_GREEN,
         'white': _FOREGROUND_RED | _FOREGROUND_GREEN | _FOREGROUND_BLUE,
         'bold': _FOREGROUND_INTENSITY,
         'black_background': 0x100,                  # unused value > 0x0f
         'red_background': _BACKGROUND_RED,
         'green_background': _BACKGROUND_GREEN,
         'yellow_background': _BACKGROUND_RED | _BACKGROUND_GREEN,
         'blue_background': _BACKGROUND_BLUE,
         'purple_background': _BACKGROUND_BLUE | _BACKGROUND_RED,
         'cyan_background': _BACKGROUND_BLUE | _BACKGROUND_GREEN,
         'white_background': (_BACKGROUND_RED | _BACKGROUND_GREEN |
                              _BACKGROUND_BLUE),
         'bold_background': _BACKGROUND_INTENSITY,
         'underline': _COMMON_LVB_UNDERSCORE,  # double-byte charsets only
         'inverse': _COMMON_LVB_REVERSE_VIDEO, # double-byte charsets only
     }
 
     passthrough = set([_FOREGROUND_INTENSITY,
                        _BACKGROUND_INTENSITY,
                        _COMMON_LVB_UNDERSCORE,
                        _COMMON_LVB_REVERSE_VIDEO])
 
     stdout = _kernel32.GetStdHandle(
                   _STD_OUTPUT_HANDLE)  # don't close the handle returned
     if stdout is None or stdout == _INVALID_HANDLE_VALUE:
         w32effects = None
     else:
         csbi = _CONSOLE_SCREEN_BUFFER_INFO()
         if not _kernel32.GetConsoleScreenBufferInfo(
                     stdout, ctypes.byref(csbi)):
             # stdout may not support GetConsoleScreenBufferInfo()
             # when called from subprocess or redirected
             w32effects = None
         else:
             origattr = csbi.wAttributes
             ansire = re.compile('\033\[([^m]*)m([^\033]*)(.*)',
                                 re.MULTILINE | re.DOTALL)
 
     def win32print(text, orig, **opts):
         label = opts.get('label', '')
         attr = origattr
 
         def mapcolor(val, attr):
             if val == -1:
                 return origattr
             elif val in passthrough:
                 return attr | val
             elif val > 0x0f:
                 return (val & 0x70) | (attr & 0x8f)
             else:
                 return (val & 0x07) | (attr & 0xf8)
 
         # determine console attributes based on labels
         for l in label.split():
             style = _styles.get(l, '')
             for effect in style.split():
                 try:
                     attr = mapcolor(w32effects[effect], attr)
                 except KeyError:
                     # w32effects could not have certain attributes so we skip
                     # them if not found
                     pass
         # hack to ensure regexp finds data
         if not text.startswith('\033['):
             text = '\033[m' + text
 
         # Look for ANSI-like codes embedded in text
         m = re.match(ansire, text)
 
         try:
             while m:
                 for sattr in m.group(1).split(';'):
                     if sattr:
                         attr = mapcolor(int(sattr), attr)
                 _kernel32.SetConsoleTextAttribute(stdout, attr)
                 orig(m.group(2), **opts)
                 m = re.match(ansire, m.group(3))
         finally:
             # Explicitly reset original attributes
             _kernel32.SetConsoleTextAttribute(stdout, origattr)

hgext/convert/cvs.py

@@ -1,297 +1,297 b''
 # cvs.py: CVS conversion code inspired by hg-cvs-import and git-cvsimport
 #
 #  Copyright 2005-2009 Matt Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import errno
 import os
 import re
 import socket
 
 from mercurial.i18n import _
 from mercurial import (
     encoding,
     error,
     pycompat,
     util,
 )
 
 from . import (
     common,
     cvsps,
 )
 
 stringio = util.stringio
 checktool = common.checktool
 commit = common.commit
 converter_source = common.converter_source
 makedatetimestamp = common.makedatetimestamp
 NoRepo = common.NoRepo
 
 class convert_cvs(converter_source):
     def __init__(self, ui, path, revs=None):
         super(convert_cvs, self).__init__(ui, path, revs=revs)
 
         cvs = os.path.join(path, "CVS")
         if not os.path.exists(cvs):
             raise NoRepo(_("%s does not look like a CVS checkout") % path)
 
         checktool('cvs')
 
         self.changeset = None
         self.files = {}
         self.tags = {}
         self.lastbranch = {}
         self.socket = None
         self.cvsroot = open(os.path.join(cvs, "Root")).read()[:-1]
         self.cvsrepo = open(os.path.join(cvs, "Repository")).read()[:-1]
         self.encoding = encoding.encoding
 
         self._connect()
 
     def _parse(self):
         if self.changeset is not None:
             return
         self.changeset = {}
 
         maxrev = 0
         if self.revs:
             if len(self.revs) > 1:
                 raise error.Abort(_('cvs source does not support specifying '
                                    'multiple revs'))
             # TODO: handle tags
             try:
                 # patchset number?
                 maxrev = int(self.revs[0])
             except ValueError:
                 raise error.Abort(_('revision %s is not a patchset number')
                                  % self.revs[0])
 
         d = pycompat.getcwd()
         try:
             os.chdir(self.path)
             id = None
 
             cache = 'update'
             if not self.ui.configbool('convert', 'cvsps.cache', True):
                 cache = None
             db = cvsps.createlog(self.ui, cache=cache)
             db = cvsps.createchangeset(self.ui, db,
                 fuzz=int(self.ui.config('convert', 'cvsps.fuzz', 60)),
                 mergeto=self.ui.config('convert', 'cvsps.mergeto', None),
                 mergefrom=self.ui.config('convert', 'cvsps.mergefrom', None))
 
             for cs in db:
                 if maxrev and cs.id > maxrev:
                     break
                 id = str(cs.id)
                 cs.author = self.recode(cs.author)
                 self.lastbranch[cs.branch] = id
                 cs.comment = self.recode(cs.comment)
                 if self.ui.configbool('convert', 'localtimezone'):
                     cs.date = makedatetimestamp(cs.date[0])
                 date = util.datestr(cs.date, '%Y-%m-%d %H:%M:%S %1%2')
                 self.tags.update(dict.fromkeys(cs.tags, id))
 
                 files = {}
                 for f in cs.entries:
                     files[f.file] = "%s%s" % ('.'.join([str(x)
                                                         for x in f.revision]),
                                               ['', '(DEAD)'][f.dead])
 
                 # add current commit to set
                 c = commit(author=cs.author, date=date,
                            parents=[str(p.id) for p in cs.parents],
                            desc=cs.comment, branch=cs.branch or '')
                 self.changeset[id] = c
                 self.files[id] = files
 
             self.heads = self.lastbranch.values()
         finally:
             os.chdir(d)
 
     def _connect(self):
         root = self.cvsroot
         conntype = None
         user, host = None, None
         cmd = ['cvs', 'server']
 
         self.ui.status(_("connecting to %s\n") % root)
 
         if root.startswith(":pserver:"):
             root = root[9:]
             m = re.match(r'(?:(.*?)(?::(.*?))?@)?([^:\/]*)(?::(\d*))?(.*)',
                          root)
             if m:
                 conntype = "pserver"
                 user, passw, serv, port, root = m.groups()
                 if not user:
                     user = "anonymous"
                 if not port:
                     port = 2401
                 else:
                     port = int(port)
                 format0 = ":pserver:%s@%s:%s" % (user, serv, root)
                 format1 = ":pserver:%s@%s:%d%s" % (user, serv, port, root)
 
                 if not passw:
                     passw = "A"
                     cvspass = os.path.expanduser("~/.cvspass")
                     try:
                         pf = open(cvspass)
                         for line in pf.read().splitlines():
                             part1, part2 = line.split(' ', 1)
                             # /1 :pserver:[e-mail]:2401/cvsroot/foo
                             # Ah<Z
                             if part1 == '/1':
                                 part1, part2 = part2.split(' ', 1)
                                 format = format1
                             # :pserver:[e-mail]:/cvsroot/foo Ah<Z
                             else:
                                 format = format0
                             if part1 == format:
                                 passw = part2
                                 break
                         pf.close()
                     except IOError as inst:
                         if inst.errno != errno.ENOENT:
                             if not getattr(inst, 'filename', None):
                                 inst.filename = cvspass
                             raise
 
                 sck = socket.socket()
                 sck.connect((serv, port))
                 sck.send("\n".join(["BEGIN AUTH REQUEST", root, user, passw,
                                     "END AUTH REQUEST", ""]))
                 if sck.recv(128) != "I LOVE YOU\n":
                     raise error.Abort(_("CVS pserver authentication failed"))
 
                 self.writep = self.readp = sck.makefile('r+')
 
         if not conntype and root.startswith(":local:"):
             conntype = "local"
             root = root[7:]
 
         if not conntype:
             # :ext:user@host/home/user/path/to/cvsroot
             if root.startswith(":ext:"):
                 root = root[5:]
             m = re.match(r'(?:([^@:/]+)@)?([^:/]+):?(.*)', root)
             # Do not take Windows path "c:\foo\bar" for a connection strings
             if os.path.isdir(root) or not m:
                 conntype = "local"
             else:
                 conntype = "rsh"
                 user, host, root = m.group(1), m.group(2), m.group(3)
 
         if conntype != "pserver":
             if conntype == "rsh":
-                rsh = os.environ.get("CVS_RSH") or "ssh"
+                rsh = encoding.environ.get("CVS_RSH") or "ssh"
                 if user:
                     cmd = [rsh, '-l', user, host] + cmd
                 else:
                     cmd = [rsh, host] + cmd
 
             # popen2 does not support argument lists under Windows
             cmd = [util.shellquote(arg) for arg in cmd]
             cmd = util.quotecommand(' '.join(cmd))
             self.writep, self.readp = util.popen2(cmd)
 
         self.realroot = root
 
         self.writep.write("Root %s\n" % root)
         self.writep.write("Valid-responses ok error Valid-requests Mode"
                           " M Mbinary E Checked-in Created Updated"
                           " Merged Removed\n")
         self.writep.write("valid-requests\n")
         self.writep.flush()
         r = self.readp.readline()
         if not r.startswith("Valid-requests"):
             raise error.Abort(_('unexpected response from CVS server '
                                '(expected "Valid-requests", but got %r)')
                              % r)
         if "UseUnchanged" in r:
             self.writep.write("UseUnchanged\n")
             self.writep.flush()
             r = self.readp.readline()
 
     def getheads(self):
         self._parse()
         return self.heads
 
     def getfile(self, name, rev):
 
         def chunkedread(fp, count):
             # file-objects returned by socket.makefile() do not handle
             # large read() requests very well.
             chunksize = 65536
             output = stringio()
             while count > 0:
                 data = fp.read(min(count, chunksize))
                 if not data:
                     raise error.Abort(_("%d bytes missing from remote file")
                                      % count)
                 count -= len(data)
                 output.write(data)
             return output.getvalue()
 
         self._parse()
         if rev.endswith("(DEAD)"):
             return None, None
 
         args = ("-N -P -kk -r %s --" % rev).split()
         args.append(self.cvsrepo + '/' + name)
         for x in args:
             self.writep.write("Argument %s\n" % x)
         self.writep.write("Directory .\n%s\nco\n" % self.realroot)
         self.writep.flush()
 
         data = ""
         mode = None
         while True:
             line = self.readp.readline()
             if line.startswith("Created ") or line.startswith("Updated "):
                 self.readp.readline() # path
                 self.readp.readline() # entries
                 mode = self.readp.readline()[:-1]
                 count = int(self.readp.readline()[:-1])
                 data = chunkedread(self.readp, count)
             elif line.startswith(" "):
                 data += line[1:]
             elif line.startswith("M "):
                 pass
             elif line.startswith("Mbinary "):
                 count = int(self.readp.readline()[:-1])
                 data = chunkedread(self.readp, count)
             else:
                 if line == "ok\n":
                     if mode is None:
                         raise error.Abort(_('malformed response from CVS'))
                     return (data, "x" in mode and "x" or "")
                 elif line.startswith("E "):
                     self.ui.warn(_("cvs server: %s\n") % line[2:])
                 elif line.startswith("Remove"):
                     self.readp.readline()
                 else:
                     raise error.Abort(_("unknown CVS response: %s") % line)
 
     def getchanges(self, rev, full):
         if full:
             raise error.Abort(_("convert from cvs does not support --full"))
         self._parse()
         return sorted(self.files[rev].iteritems()), {}, set()
 
     def getcommit(self, rev):
         self._parse()
         return self.changeset[rev]
 
     def gettags(self):
         self._parse()
         return self.tags
 
     def getchangedfiles(self, rev, i):
         self._parse()
         return sorted(self.files[rev])

hgext/convert/cvsps.py

@@ -1,920 +1,921 b''
 # Mercurial built-in replacement for cvsps.
 #
 # Copyright 2008, Frank Kingswood <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 import os
 import re
 
 from mercurial.i18n import _
 from mercurial import (
+    encoding,
     hook,
     pycompat,
     util,
 )
 
 pickle = util.pickle
 
 class logentry(object):
     '''Class logentry has the following attributes:
         .author    - author name as CVS knows it
         .branch    - name of branch this revision is on
         .branches  - revision tuple of branches starting at this revision
         .comment   - commit message
         .commitid  - CVS commitid or None
         .date      - the commit date as a (time, tz) tuple
         .dead      - true if file revision is dead
         .file      - Name of file
         .lines     - a tuple (+lines, -lines) or None
         .parent    - Previous revision of this entry
         .rcs       - name of file as returned from CVS
         .revision  - revision number as tuple
         .tags      - list of tags on the file
         .synthetic - is this a synthetic "file ... added on ..." revision?
         .mergepoint - the branch that has been merged from (if present in
                       rlog output) or None
         .branchpoints - the branches that start at the current entry or empty
     '''
     def __init__(self, **entries):
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         items = ("%s=%r"%(k, self.__dict__[k]) for k in sorted(self.__dict__))
         return "%s(%s)"%(type(self).__name__, ", ".join(items))
 
 class logerror(Exception):
     pass
 
 def getrepopath(cvspath):
     """Return the repository path from a CVS path.
 
     >>> getrepopath('/foo/bar')
     '/foo/bar'
     >>> getrepopath('c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:10/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:10c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:c:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:[e-mail]:/foo/bar')
     '/foo/bar'
     >>> getrepopath(':pserver:[e-mail]:c:/foo/bar')
     '/foo/bar'
     >>> getrepopath('user@server/path/to/repository')
     '/path/to/repository'
     """
     # According to CVS manual, CVS paths are expressed like:
     # [:method:][[user][:password]@]hostname[:[port]]/path/to/repository
     #
     # CVSpath is splitted into parts and then position of the first occurrence
     # of the '/' char after the '@' is located. The solution is the rest of the
     # string after that '/' sign including it
 
     parts = cvspath.split(':')
     atposition = parts[-1].find('@')
     start = 0
 
     if atposition != -1:
         start = atposition
 
     repopath = parts[-1][parts[-1].find('/', start):]
     return repopath
 
 def createlog(ui, directory=None, root="", rlog=True, cache=None):
     '''Collect the CVS rlog'''
 
     # Because we store many duplicate commit log messages, reusing strings
     # saves a lot of memory and pickle storage space.
     _scache = {}
     def scache(s):
         "return a shared version of a string"
         return _scache.setdefault(s, s)
 
     ui.status(_('collecting CVS rlog\n'))
 
     log = []      # list of logentry objects containing the CVS state
 
     # patterns to match in CVS (r)log output, by state of use
     re_00 = re.compile('RCS file: (.+)$')
     re_01 = re.compile('cvs \\[r?log aborted\\]: (.+)$')
     re_02 = re.compile('cvs (r?log|server): (.+)\n$')
     re_03 = re.compile("(Cannot access.+CVSROOT)|"
                        "(can't create temporary directory.+)$")
     re_10 = re.compile('Working file: (.+)$')
     re_20 = re.compile('symbolic names:')
     re_30 = re.compile('\t(.+): ([\\d.]+)$')
     re_31 = re.compile('----------------------------$')
     re_32 = re.compile('======================================='
                        '======================================$')
     re_50 = re.compile('revision ([\\d.]+)(\s+locked by:\s+.+;)?$')
     re_60 = re.compile(r'date:\s+(.+);\s+author:\s+(.+);\s+state:\s+(.+?);'
                        r'(\s+lines:\s+(\+\d+)?\s+(-\d+)?;)?'
                        r'(\s+commitid:\s+([^;]+);)?'
                        r'(.*mergepoint:\s+([^;]+);)?')
     re_70 = re.compile('branches: (.+);$')
 
     file_added_re = re.compile(r'file [^/]+ was (initially )?added on branch')
 
     prefix = ''   # leading path to strip of what we get from CVS
 
     if directory is None:
         # Current working directory
 
         # Get the real directory in the repository
         try:
             prefix = open(os.path.join('CVS','Repository')).read().strip()
             directory = prefix
             if prefix == ".":
                 prefix = ""
         except IOError:
             raise logerror(_('not a CVS sandbox'))
 
         if prefix and not prefix.endswith(pycompat.ossep):
             prefix += pycompat.ossep
 
         # Use the Root file in the sandbox, if it exists
         try:
             root = open(os.path.join('CVS','Root')).read().strip()
         except IOError:
             pass
 
     if not root:
-        root = os.environ.get('CVSROOT', '')
+        root = encoding.environ.get('CVSROOT', '')
 
     # read log cache if one exists
     oldlog = []
     date = None
 
     if cache:
         cachedir = os.path.expanduser('~/.hg.cvsps')
         if not os.path.exists(cachedir):
             os.mkdir(cachedir)
 
         # The cvsps cache pickle needs a uniquified name, based on the
         # repository location. The address may have all sort of nasties
         # in it, slashes, colons and such. So here we take just the
         # alphanumeric characters, concatenated in a way that does not
         # mix up the various components, so that
         #    :pserver:user@server:/path
         # and
         #    /pserver/user/server/path
         # are mapped to different cache file names.
         cachefile = root.split(":") + [directory, "cache"]
         cachefile = ['-'.join(re.findall(r'\w+', s)) for s in cachefile if s]
         cachefile = os.path.join(cachedir,
                                  '.'.join([s for s in cachefile if s]))
 
     if cache == 'update':
         try:
             ui.note(_('reading cvs log cache %s\n') % cachefile)
             oldlog = pickle.load(open(cachefile))
             for e in oldlog:
                 if not (util.safehasattr(e, 'branchpoints') and
                         util.safehasattr(e, 'commitid') and
                         util.safehasattr(e, 'mergepoint')):
                     ui.status(_('ignoring old cache\n'))
                     oldlog = []
                     break
 
             ui.note(_('cache has %d log entries\n') % len(oldlog))
         except Exception as e:
             ui.note(_('error reading cache: %r\n') % e)
 
         if oldlog:
             date = oldlog[-1].date    # last commit date as a (time,tz) tuple
             date = util.datestr(date, '%Y/%m/%d %H:%M:%S %1%2')
 
     # build the CVS commandline
     cmd = ['cvs', '-q']
     if root:
         cmd.append('-d%s' % root)
         p = util.normpath(getrepopath(root))
         if not p.endswith('/'):
             p += '/'
         if prefix:
             # looks like normpath replaces "" by "."
             prefix = p + util.normpath(prefix)
         else:
             prefix = p
     cmd.append(['log', 'rlog'][rlog])
     if date:
         # no space between option and date string
         cmd.append('-d>%s' % date)
     cmd.append(directory)
 
     # state machine begins here
     tags = {}     # dictionary of revisions on current file with their tags
     branchmap = {} # mapping between branch names and revision numbers
     rcsmap = {}
     state = 0
     store = False # set when a new record can be appended
 
     cmd = [util.shellquote(arg) for arg in cmd]
     ui.note(_("running %s\n") % (' '.join(cmd)))
     ui.debug("prefix=%r directory=%r root=%r\n" % (prefix, directory, root))
 
     pfp = util.popen(' '.join(cmd))
     peek = pfp.readline()
     while True:
         line = peek
         if line == '':
             break
         peek = pfp.readline()
         if line.endswith('\n'):
             line = line[:-1]
         #ui.debug('state=%d line=%r\n' % (state, line))
 
         if state == 0:
             # initial state, consume input until we see 'RCS file'
             match = re_00.match(line)
             if match:
                 rcs = match.group(1)
                 tags = {}
                 if rlog:
                     filename = util.normpath(rcs[:-2])
                     if filename.startswith(prefix):
                         filename = filename[len(prefix):]
                     if filename.startswith('/'):
                         filename = filename[1:]
                     if filename.startswith('Attic/'):
                         filename = filename[6:]
                     else:
                         filename = filename.replace('/Attic/', '/')
                     state = 2
                     continue
                 state = 1
                 continue
             match = re_01.match(line)
             if match:
                 raise logerror(match.group(1))
             match = re_02.match(line)
             if match:
                 raise logerror(match.group(2))
             if re_03.match(line):
                 raise logerror(line)
 
         elif state == 1:
             # expect 'Working file' (only when using log instead of rlog)
             match = re_10.match(line)
             assert match, _('RCS file must be followed by working file')
             filename = util.normpath(match.group(1))
             state = 2
 
         elif state == 2:
             # expect 'symbolic names'
             if re_20.match(line):
                 branchmap = {}
                 state = 3
 
         elif state == 3:
             # read the symbolic names and store as tags
             match = re_30.match(line)
             if match:
                 rev = [int(x) for x in match.group(2).split('.')]
 
                 # Convert magic branch number to an odd-numbered one
                 revn = len(rev)
                 if revn > 3 and (revn % 2) == 0 and rev[-2] == 0:
                     rev = rev[:-2] + rev[-1:]
                 rev = tuple(rev)
 
                 if rev not in tags:
                     tags[rev] = []
                 tags[rev].append(match.group(1))
                 branchmap[match.group(1)] = match.group(2)
 
             elif re_31.match(line):
                 state = 5
             elif re_32.match(line):
                 state = 0
 
         elif state == 4:
             # expecting '------' separator before first revision
             if re_31.match(line):
                 state = 5
             else:
                 assert not re_32.match(line), _('must have at least '
                                                 'some revisions')
 
         elif state == 5:
             # expecting revision number and possibly (ignored) lock indication
             # we create the logentry here from values stored in states 0 to 4,
             # as this state is re-entered for subsequent revisions of a file.
             match = re_50.match(line)
             assert match, _('expected revision number')
             e = logentry(rcs=scache(rcs),
                          file=scache(filename),
                          revision=tuple([int(x) for x in
                                          match.group(1).split('.')]),
                          branches=[],
                          parent=None,
                          commitid=None,
                          mergepoint=None,
                          branchpoints=set())
 
             state = 6
 
         elif state == 6:
             # expecting date, author, state, lines changed
             match = re_60.match(line)
             assert match, _('revision must be followed by date line')
             d = match.group(1)
             if d[2] == '/':
                 # Y2K
                 d = '19' + d
 
             if len(d.split()) != 3:
                 # cvs log dates always in GMT
                 d = d + ' UTC'
             e.date = util.parsedate(d, ['%y/%m/%d %H:%M:%S',
                                         '%Y/%m/%d %H:%M:%S',
                                         '%Y-%m-%d %H:%M:%S'])
             e.author = scache(match.group(2))
             e.dead = match.group(3).lower() == 'dead'
 
             if match.group(5):
                 if match.group(6):
                     e.lines = (int(match.group(5)), int(match.group(6)))
                 else:
                     e.lines = (int(match.group(5)), 0)
             elif match.group(6):
                 e.lines = (0, int(match.group(6)))
             else:
                 e.lines = None
 
             if match.group(7): # cvs 1.12 commitid
                 e.commitid = match.group(8)
 
             if match.group(9): # cvsnt mergepoint
                 myrev = match.group(10).split('.')
                 if len(myrev) == 2: # head
                     e.mergepoint = 'HEAD'
                 else:
                     myrev = '.'.join(myrev[:-2] + ['0', myrev[-2]])
                     branches = [b for b in branchmap if branchmap[b] == myrev]
                     assert len(branches) == 1, ('unknown branch: %s'
                                                 % e.mergepoint)
                     e.mergepoint = branches[0]
 
             e.comment = []
             state = 7
 
         elif state == 7:
             # read the revision numbers of branches that start at this revision
             # or store the commit log message otherwise
             m = re_70.match(line)
             if m:
                 e.branches = [tuple([int(y) for y in x.strip().split('.')])
                                 for x in m.group(1).split(';')]
                 state = 8
             elif re_31.match(line) and re_50.match(peek):
                 state = 5
                 store = True
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         elif state == 8:
             # store commit log message
             if re_31.match(line):
                 cpeek = peek
                 if cpeek.endswith('\n'):
                     cpeek = cpeek[:-1]
                 if re_50.match(cpeek):
                     state = 5
                     store = True
                 else:
                     e.comment.append(line)
             elif re_32.match(line):
                 state = 0
                 store = True
             else:
                 e.comment.append(line)
 
         # When a file is added on a branch B1, CVS creates a synthetic
         # dead trunk revision 1.1 so that the branch has a root.
         # Likewise, if you merge such a file to a later branch B2 (one
         # that already existed when the file was added on B1), CVS
         # creates a synthetic dead revision 1.1.x.1 on B2.  Don't drop
         # these revisions now, but mark them synthetic so
         # createchangeset() can take care of them.
         if (store and
               e.dead and
               e.revision[-1] == 1 and      # 1.1 or 1.1.x.1
               len(e.comment) == 1 and
               file_added_re.match(e.comment[0])):
             ui.debug('found synthetic revision in %s: %r\n'
                      % (e.rcs, e.comment[0]))
             e.synthetic = True
 
         if store:
             # clean up the results and save in the log.
             store = False
             e.tags = sorted([scache(x) for x in tags.get(e.revision, [])])
             e.comment = scache('\n'.join(e.comment))
 
             revn = len(e.revision)
             if revn > 3 and (revn % 2) == 0:
                 e.branch = tags.get(e.revision[:-1], [None])[0]
             else:
                 e.branch = None
 
             # find the branches starting from this revision
             branchpoints = set()
             for branch, revision in branchmap.iteritems():
                 revparts = tuple([int(i) for i in revision.split('.')])
                 if len(revparts) < 2: # bad tags
                     continue
                 if revparts[-2] == 0 and revparts[-1] % 2 == 0:
                     # normal branch
                     if revparts[:-2] == e.revision:
                         branchpoints.add(branch)
                 elif revparts == (1, 1, 1): # vendor branch
                     if revparts in e.branches:
                         branchpoints.add(branch)
             e.branchpoints = branchpoints
 
             log.append(e)
 
             rcsmap[e.rcs.replace('/Attic/', '/')] = e.rcs
 
             if len(log) % 100 == 0:
                 ui.status(util.ellipsis('%d %s' % (len(log), e.file), 80)+'\n')
 
     log.sort(key=lambda x: (x.rcs, x.revision))
 
     # find parent revisions of individual files
     versions = {}
     for e in sorted(oldlog, key=lambda x: (x.rcs, x.revision)):
         rcs = e.rcs.replace('/Attic/', '/')
         if rcs in rcsmap:
             e.rcs = rcsmap[rcs]
         branch = e.revision[:-1]
         versions[(e.rcs, branch)] = e.revision
 
     for e in log:
         branch = e.revision[:-1]
         p = versions.get((e.rcs, branch), None)
         if p is None:
             p = e.revision[:-2]
         e.parent = p
         versions[(e.rcs, branch)] = e.revision
 
     # update the log cache
     if cache:
         if log:
             # join up the old and new logs
             log.sort(key=lambda x: x.date)
 
             if oldlog and oldlog[-1].date >= log[0].date:
                 raise logerror(_('log cache overlaps with new log entries,'
                                  ' re-run without cache.'))
 
             log = oldlog + log
 
             # write the new cachefile
             ui.note(_('writing cvs log cache %s\n') % cachefile)
             pickle.dump(log, open(cachefile, 'w'))
         else:
             log = oldlog
 
     ui.status(_('%d log entries\n') % len(log))
 
     hook.hook(ui, None, "cvslog", True, log=log)
 
     return log
 
 
 class changeset(object):
     '''Class changeset has the following attributes:
         .id        - integer identifying this changeset (list index)
         .author    - author name as CVS knows it
         .branch    - name of branch this changeset is on, or None
         .comment   - commit message
         .commitid  - CVS commitid or None
         .date      - the commit date as a (time,tz) tuple
         .entries   - list of logentry objects in this changeset
         .parents   - list of one or two parent changesets
         .tags      - list of tags on this changeset
         .synthetic - from synthetic revision "file ... added on branch ..."
         .mergepoint- the branch that has been merged from or None
         .branchpoints- the branches that start at the current entry or empty
     '''
     def __init__(self, **entries):
         self.id = None
         self.synthetic = False
         self.__dict__.update(entries)
 
     def __repr__(self):
         items = ("%s=%r"%(k, self.__dict__[k]) for k in sorted(self.__dict__))
         return "%s(%s)"%(type(self).__name__, ", ".join(items))
 
 def createchangeset(ui, log, fuzz=60, mergefrom=None, mergeto=None):
     '''Convert log into changesets.'''
 
     ui.status(_('creating changesets\n'))
 
     # try to order commitids by date
     mindate = {}
     for e in log:
         if e.commitid:
             mindate[e.commitid] = min(e.date, mindate.get(e.commitid))
 
     # Merge changesets
     log.sort(key=lambda x: (mindate.get(x.commitid), x.commitid, x.comment,
                             x.author, x.branch, x.date, x.branchpoints))
 
     changesets = []
     files = set()
     c = None
     for i, e in enumerate(log):
 
         # Check if log entry belongs to the current changeset or not.
 
         # Since CVS is file-centric, two different file revisions with
         # different branchpoints should be treated as belonging to two
         # different changesets (and the ordering is important and not
         # honoured by cvsps at this point).
         #
         # Consider the following case:
         # foo 1.1 branchpoints: [MYBRANCH]
         # bar 1.1 branchpoints: [MYBRANCH, MYBRANCH2]
         #
         # Here foo is part only of MYBRANCH, but not MYBRANCH2, e.g. a
         # later version of foo may be in MYBRANCH2, so foo should be the
         # first changeset and bar the next and MYBRANCH and MYBRANCH2
         # should both start off of the bar changeset. No provisions are
         # made to ensure that this is, in fact, what happens.
         if not (c and e.branchpoints == c.branchpoints and
                 (# cvs commitids
                  (e.commitid is not None and e.commitid == c.commitid) or
                  (# no commitids, use fuzzy commit detection
                   (e.commitid is None or c.commitid is None) and
                    e.comment == c.comment and
                    e.author == c.author and
                    e.branch == c.branch and
                    ((c.date[0] + c.date[1]) <=
                     (e.date[0] + e.date[1]) <=
                     (c.date[0] + c.date[1]) + fuzz) and
                    e.file not in files))):
             c = changeset(comment=e.comment, author=e.author,
                           branch=e.branch, date=e.date,
                           entries=[], mergepoint=e.mergepoint,
                           branchpoints=e.branchpoints, commitid=e.commitid)
             changesets.append(c)
 
             files = set()
             if len(changesets) % 100 == 0:
                 t = '%d %s' % (len(changesets), repr(e.comment)[1:-1])
                 ui.status(util.ellipsis(t, 80) + '\n')
 
         c.entries.append(e)
         files.add(e.file)
         c.date = e.date       # changeset date is date of latest commit in it
 
     # Mark synthetic changesets
 
     for c in changesets:
         # Synthetic revisions always get their own changeset, because
         # the log message includes the filename.  E.g. if you add file3
         # and file4 on a branch, you get four log entries and three
         # changesets:
         #   "File file3 was added on branch ..." (synthetic, 1 entry)
         #   "File file4 was added on branch ..." (synthetic, 1 entry)
         #   "Add file3 and file4 to fix ..."     (real, 2 entries)
         # Hence the check for 1 entry here.
         c.synthetic = len(c.entries) == 1 and c.entries[0].synthetic
 
     # Sort files in each changeset
 
     def entitycompare(l, r):
         'Mimic cvsps sorting order'
         l = l.file.split('/')
         r = r.file.split('/')
         nl = len(l)
         nr = len(r)
         n = min(nl, nr)
         for i in range(n):
             if i + 1 == nl and nl < nr:
                 return -1
             elif i + 1 == nr and nl > nr:
                 return +1
             elif l[i] < r[i]:
                 return -1
             elif l[i] > r[i]:
                 return +1
         return 0
 
     for c in changesets:
         c.entries.sort(entitycompare)
 
     # Sort changesets by date
 
     odd = set()
     def cscmp(l, r, odd=odd):
         d = sum(l.date) - sum(r.date)
         if d:
             return d
 
         # detect vendor branches and initial commits on a branch
         le = {}
         for e in l.entries:
             le[e.rcs] = e.revision
         re = {}
         for e in r.entries:
             re[e.rcs] = e.revision
 
         d = 0
         for e in l.entries:
             if re.get(e.rcs, None) == e.parent:
                 assert not d
                 d = 1
                 break
 
         for e in r.entries:
             if le.get(e.rcs, None) == e.parent:
                 if d:
                     odd.add((l, r))
                 d = -1
                 break
         # By this point, the changesets are sufficiently compared that
         # we don't really care about ordering. However, this leaves
         # some race conditions in the tests, so we compare on the
         # number of files modified, the files contained in each
         # changeset, and the branchpoints in the change to ensure test
         # output remains stable.
 
         # recommended replacement for cmp from
         # https://docs.python.org/3.0/whatsnew/3.0.html
         c = lambda x, y: (x > y) - (x < y)
         # Sort bigger changes first.
         if not d:
             d = c(len(l.entries), len(r.entries))
         # Try sorting by filename in the change.
         if not d:
             d = c([e.file for e in l.entries], [e.file for e in r.entries])
         # Try and put changes without a branch point before ones with
         # a branch point.
         if not d:
             d = c(len(l.branchpoints), len(r.branchpoints))
         return d
 
     changesets.sort(cscmp)
 
     # Collect tags
 
     globaltags = {}
     for c in changesets:
         for e in c.entries:
             for tag in e.tags:
                 # remember which is the latest changeset to have this tag
                 globaltags[tag] = c
 
     for c in changesets:
         tags = set()
         for e in c.entries:
             tags.update(e.tags)
         # remember tags only if this is the latest changeset to have it
         c.tags = sorted(tag for tag in tags if globaltags[tag] is c)
 
     # Find parent changesets, handle {{mergetobranch BRANCHNAME}}
     # by inserting dummy changesets with two parents, and handle
     # {{mergefrombranch BRANCHNAME}} by setting two parents.
 
     if mergeto is None:
         mergeto = r'{{mergetobranch ([-\w]+)}}'
     if mergeto:
         mergeto = re.compile(mergeto)
 
     if mergefrom is None:
         mergefrom = r'{{mergefrombranch ([-\w]+)}}'
     if mergefrom:
         mergefrom = re.compile(mergefrom)
 
     versions = {}    # changeset index where we saw any particular file version
     branches = {}    # changeset index where we saw a branch
     n = len(changesets)
     i = 0
     while i < n:
         c = changesets[i]
 
         for f in c.entries:
             versions[(f.rcs, f.revision)] = i
 
         p = None
         if c.branch in branches:
             p = branches[c.branch]
         else:
             # first changeset on a new branch
             # the parent is a changeset with the branch in its
             # branchpoints such that it is the latest possible
             # commit without any intervening, unrelated commits.
 
             for candidate in xrange(i):
                 if c.branch not in changesets[candidate].branchpoints:
                     if p is not None:
                         break
                     continue
                 p = candidate
 
         c.parents = []
         if p is not None:
             p = changesets[p]
 
             # Ensure no changeset has a synthetic changeset as a parent.
             while p.synthetic:
                 assert len(p.parents) <= 1, \
                        _('synthetic changeset cannot have multiple parents')
                 if p.parents:
                     p = p.parents[0]
                 else:
                     p = None
                     break
 
             if p is not None:
                 c.parents.append(p)
 
         if c.mergepoint:
             if c.mergepoint == 'HEAD':
                 c.mergepoint = None
             c.parents.append(changesets[branches[c.mergepoint]])
 
         if mergefrom:
             m = mergefrom.search(c.comment)
             if m:
                 m = m.group(1)
                 if m == 'HEAD':
                     m = None
                 try:
                     candidate = changesets[branches[m]]
                 except KeyError:
                     ui.warn(_("warning: CVS commit message references "
                               "non-existent branch %r:\n%s\n")
                             % (m, c.comment))
                 if m in branches and c.branch != m and not candidate.synthetic:
                     c.parents.append(candidate)
 
         if mergeto:
             m = mergeto.search(c.comment)
             if m:
                 if m.groups():
                     m = m.group(1)
                     if m == 'HEAD':
                         m = None
                 else:
                     m = None   # if no group found then merge to HEAD
                 if m in branches and c.branch != m:
                     # insert empty changeset for merge
                     cc = changeset(
                         author=c.author, branch=m, date=c.date,
                         comment='convert-repo: CVS merge from branch %s'
                         % c.branch,
                         entries=[], tags=[],
                         parents=[changesets[branches[m]], c])
                     changesets.insert(i + 1, cc)
                     branches[m] = i + 1
 
                     # adjust our loop counters now we have inserted a new entry
                     n += 1
                     i += 2
                     continue
 
         branches[c.branch] = i
         i += 1
 
     # Drop synthetic changesets (safe now that we have ensured no other
     # changesets can have them as parents).
     i = 0
     while i < len(changesets):
         if changesets[i].synthetic:
             del changesets[i]
         else:
             i += 1
 
     # Number changesets
 
     for i, c in enumerate(changesets):
         c.id = i + 1
 
     if odd:
         for l, r in odd:
             if l.id is not None and r.id is not None:
                 ui.warn(_('changeset %d is both before and after %d\n')
                         % (l.id, r.id))
 
     ui.status(_('%d changeset entries\n') % len(changesets))
 
     hook.hook(ui, None, "cvschangesets", True, changesets=changesets)
 
     return changesets
 
 
 def debugcvsps(ui, *args, **opts):
     '''Read CVS rlog for current directory or named path in
     repository, and convert the log to changesets based on matching
     commit log entries and dates.
     '''
     if opts["new_cache"]:
         cache = "write"
     elif opts["update_cache"]:
         cache = "update"
     else:
         cache = None
 
     revisions = opts["revisions"]
 
     try:
         if args:
             log = []
             for d in args:
                 log += createlog(ui, d, root=opts["root"], cache=cache)
         else:
             log = createlog(ui, root=opts["root"], cache=cache)
     except logerror as e:
         ui.write("%r\n"%e)
         return
 
     changesets = createchangeset(ui, log, opts["fuzz"])
     del log
 
     # Print changesets (optionally filtered)
 
     off = len(revisions)
     branches = {}    # latest version number in each branch
     ancestors = {}   # parent branch
     for cs in changesets:
 
         if opts["ancestors"]:
             if cs.branch not in branches and cs.parents and cs.parents[0].id:
                 ancestors[cs.branch] = (changesets[cs.parents[0].id - 1].branch,
                                         cs.parents[0].id)
             branches[cs.branch] = cs.id
 
         # limit by branches
         if opts["branches"] and (cs.branch or 'HEAD') not in opts["branches"]:
             continue
 
         if not off:
             # Note: trailing spaces on several lines here are needed to have
             #       bug-for-bug compatibility with cvsps.
             ui.write('---------------------\n')
             ui.write(('PatchSet %d \n' % cs.id))
             ui.write(('Date: %s\n' % util.datestr(cs.date,
                                                  '%Y/%m/%d %H:%M:%S %1%2')))
             ui.write(('Author: %s\n' % cs.author))
             ui.write(('Branch: %s\n' % (cs.branch or 'HEAD')))
             ui.write(('Tag%s: %s \n' % (['', 's'][len(cs.tags) > 1],
                                   ','.join(cs.tags) or '(none)')))
             if cs.branchpoints:
                 ui.write(('Branchpoints: %s \n') %
                          ', '.join(sorted(cs.branchpoints)))
             if opts["parents"] and cs.parents:
                 if len(cs.parents) > 1:
                     ui.write(('Parents: %s\n' %
                              (','.join([str(p.id) for p in cs.parents]))))
                 else:
                     ui.write(('Parent: %d\n' % cs.parents[0].id))
 
             if opts["ancestors"]:
                 b = cs.branch
                 r = []
                 while b:
                     b, c = ancestors[b]
                     r.append('%s:%d:%d' % (b or "HEAD", c, branches[b]))
                 if r:
                     ui.write(('Ancestors: %s\n' % (','.join(r))))
 
             ui.write(('Log:\n'))
             ui.write('%s\n\n' % cs.comment)
             ui.write(('Members: \n'))
             for f in cs.entries:
                 fn = f.file
                 if fn.startswith(opts["prefix"]):
                     fn = fn[len(opts["prefix"]):]
                 ui.write('\t%s:%s->%s%s \n' % (
                         fn, '.'.join([str(x) for x in f.parent]) or 'INITIAL',
                         '.'.join([str(x) for x in f.revision]),
                         ['', '(DEAD)'][f.dead]))
             ui.write('\n')
 
         # have we seen the start tag?
         if revisions and off:
             if revisions[0] == str(cs.id) or \
                 revisions[0] in cs.tags:
                 off = False
 
         # see if we reached the end tag
         if len(revisions) > 1 and not off:
             if revisions[1] == str(cs.id) or \
                 revisions[1] in cs.tags:
                 break

hgext/logtoprocess.py

@@ -1,129 +1,131 b''
 # logtoprocess.py - send ui.log() data to a subprocess
 #
 # Copyright 2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """Send ui.log() data to a subprocess (EXPERIMENTAL)
 
 This extension lets you specify a shell command per ui.log() event,
 sending all remaining arguments to as environment variables to that command.
 
 Each positional argument to the method results in a `MSG[N]` key in the
 environment, starting at 1 (so `MSG1`, `MSG2`, etc.). Each keyword argument
 is set as a `OPT_UPPERCASE_KEY` variable (so the key is uppercased, and
 prefixed with `OPT_`). The original event name is passed in the `EVENT`
 environment variable, and the process ID of mercurial is given in `HGPID`.
 
 So given a call `ui.log('foo', 'bar', 'baz', spam='eggs'), a script configured
 for the `foo` event can expect an environment with `MSG1=bar`, `MSG2=baz`, and
 `OPT_SPAM=eggs`.
 
 Scripts are configured in the `[logtoprocess]` section, each key an event name.
 For example::
 
   [logtoprocess]
   commandexception = echo "$MSG2$MSG3" > /var/log/mercurial_exceptions.log
 
 would log the warning message and traceback of any failed command dispatch.
 
 Scripts are run asynchronously as detached daemon processes; mercurial will
 not ensure that they exit cleanly.
 
 """
 
 from __future__ import absolute_import
 
 import itertools
 import os
 import platform
 import subprocess
 import sys
 
+from mercurial import encoding
+
 # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for
 # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should
 # be specifying the version(s) of Mercurial they are tested with, or
 # leave the attribute unspecified.
 testedwith = 'ships-with-hg-core'
 
 def uisetup(ui):
     if platform.system() == 'Windows':
         # no fork on Windows, but we can create a detached process
         # https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863.aspx
         # No stdlib constant exists for this value
         DETACHED_PROCESS = 0x00000008
         _creationflags = DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP
 
         def runshellcommand(script, env):
             # we can't use close_fds *and* redirect stdin. I'm not sure that we
             # need to because the detached process has no console connection.
             subprocess.Popen(
                 script, shell=True, env=env, close_fds=True,
                 creationflags=_creationflags)
     else:
         def runshellcommand(script, env):
             # double-fork to completely detach from the parent process
             # based on http://code.activestate.com/recipes/278731
             pid = os.fork()
             if pid:
                 # parent
                 return
             # subprocess.Popen() forks again, all we need to add is
             # flag the new process as a new session.
             if sys.version_info < (3, 2):
                 newsession = {'preexec_fn': os.setsid}
             else:
                 newsession = {'start_new_session': True}
             try:
                 # connect stdin to devnull to make sure the subprocess can't
                 # muck up that stream for mercurial.
                 subprocess.Popen(
                     script, shell=True, stdin=open(os.devnull, 'r'), env=env,
                     close_fds=True, **newsession)
             finally:
                 # mission accomplished, this child needs to exit and not
                 # continue the hg process here.
                 os._exit(0)
 
     class logtoprocessui(ui.__class__):
         def log(self, event, *msg, **opts):
             """Map log events to external commands
 
             Arguments are passed on as environment variables.
 
             """
             script = self.config('logtoprocess', event)
             if script:
                 if msg:
                     # try to format the log message given the remaining
                     # arguments
                     try:
                         # Python string formatting with % either uses a
                         # dictionary *or* tuple, but not both. If we have
                         # keyword options, assume we need a mapping.
                         formatted = msg[0] % (opts or msg[1:])
                     except (TypeError, KeyError):
                         # Failed to apply the arguments, ignore
                         formatted = msg[0]
                     messages = (formatted,) + msg[1:]
                 else:
                     messages = msg
                 # positional arguments are listed as MSG[N] keys in the
                 # environment
                 msgpairs = (
                     ('MSG{0:d}'.format(i), str(m))
                     for i, m in enumerate(messages, 1))
                 # keyword arguments get prefixed with OPT_ and uppercased
                 optpairs = (
                     ('OPT_{0}'.format(key.upper()), str(value))
                     for key, value in opts.iteritems())
-                env = dict(itertools.chain(os.environ.items(),
+                env = dict(itertools.chain(encoding.environ.items(),
                                            msgpairs, optpairs),
                            EVENT=event, HGPID=str(os.getpid()))
                 # Connect stdin to /dev/null to prevent child processes messing
                 # with mercurial's stdin.
                 runshellcommand(script, env)
             return super(logtoprocessui, self).log(event, *msg, **opts)
 
     # Replace the class for this instance and all clones created from it:
     ui.__class__ = logtoprocessui