py3kcompat: added a "compatibility layer" for py3k

This patch adds some ugly constructs. The first of them is bytesformatter, a
function that formats strings like when '%' is called. The main motivation for
this function is py3k's strange behavior:

>>> 'foo %s' % b'bar'
"foo b'bar'"
>>> b'foo %s' % b'bar'
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: unsupported operand type(s) for %: 'bytes' and 'bytes'
>>> b'foo %s' % 'bar'
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
TypeError: unsupported operand type(s) for %: 'bytes' and 'str'

In other words, if we can't format bytes with bytes, and recall that all
mercurial strings will be converted by a fixer, then things will break badly if
we don't take a similar approach.

The other addition with this patch is that the os.environ dictionary is
monkeypatched to have bytes items. Hopefully this won't be needed in the
future, as python 3.2 might get a os.environb dictionary that holds bytes
items.

mercurial/py3kcompat.py

@@ -0,0 +1,65 b''
+# py3kcompat.py - compatibility definitions for running hg in py3k
+#
+# Copyright 2010 Renato Cunha <[e-mail]>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+import os, builtins
+
+from numbers import Number
+
+def bytesformatter(format, args):
+    '''Custom implementation of a formatter for bytestrings.
+
+    This function currently relias on the string formatter to do the
+    formatting and always returns bytes objects.
+
+    >>> bytesformatter(20, 10)
+    0
+    >>> bytesformatter('unicode %s, %s!', ('string', 'foo'))
+    b'unicode string, foo!'
+    >>> bytesformatter(b'test %s', 'me')
+    b'test me'
+    >>> bytesformatter('test %s', 'me')
+    b'test me'
+    >>> bytesformatter(b'test %s', b'me')
+    b'test me'
+    >>> bytesformatter('test %s', b'me')
+    b'test me'
+    >>> bytesformatter('test %d: %s', (1, b'result'))
+    b'test 1: result'
+    '''
+    # The current implementation just converts from bytes to unicode, do
+    # what's needed and then convert the results back to bytes.
+    # Another alternative is to use the Python C API implementation.
+    if isinstance(format, Number):
+        # If the fixer erroneously passes a number remainder operation to
+        # bytesformatter, we just return the correct operation
+        return format % args
+    if isinstance(format, bytes):
+        format = format.decode('utf-8', 'surrogateescape')
+    if isinstance(args, bytes):
+        args = args.decode('utf-8', 'surrogateescape')
+    if isinstance(args, tuple):
+        newargs = []
+        for arg in args:
+            if isinstance(arg, bytes):
+                arg = arg.decode('utf-8', 'surrogateescape')
+            newargs.append(arg)
+        args = tuple(newargs)
+    ret = format % args
+    return ret.encode('utf-8', 'surrogateescape')
+builtins.bytesformatter = bytesformatter
+
+# Create bytes equivalents for os.environ values
+for key in list(os.environ.keys()):
+    # UTF-8 is fine for us
+    bkey = key.encode('utf-8', 'surrogateescape')
+    bvalue = os.environ[key].encode('utf-8', 'surrogateescape')
+    os.environ[bkey] = bvalue
+
+if __name__ == '__main__':
+    import doctest
+    doctest.testmod()
+

contrib/hgfixes/fix_bytes.py

@@ -1,96 +1,97 b''
 """Fixer that changes plain strings to bytes strings."""
 
 import re
 
 from lib2to3 import fixer_base
 from lib2to3.pgen2 import token
 from lib2to3.fixer_util import Name
 from lib2to3.pygram import python_symbols as syms
 
 _re = re.compile(r'[rR]?[\'\"]')
 
 # XXX: Implementing a blacklist in 2to3 turned out to be more troublesome than
 # blacklisting some modules inside the fixers. So, this is what I came with.
 
 blacklist = ['mercurial/demandimport.py',
+             'mercurial/py3kcompat.py', # valid python 3 already
              'mercurial/i18n.py',
             ]
 
 def isdocstring(node):
     def isclassorfunction(ancestor):
         symbols = (syms.funcdef, syms.classdef)
         # if the current node is a child of a function definition, a class
         # definition or a file, then it is a docstring
         if ancestor.type == syms.simple_stmt:
             try:
                 while True:
                     if ancestor.type in symbols:
                         return True
                     ancestor = ancestor.parent
             except AttributeError:
                 return False
         return False
 
     def ismodule(ancestor):
         # Our child is a docstring if we are a simple statement, and our
         # ancestor is file_input. In other words, our child is a lone string in
         # the source file.
         try:
             if (ancestor.type == syms.simple_stmt and
                 ancestor.parent.type == syms.file_input):
                     return True
         except AttributeError:
             return False
 
     def isdocassignment(ancestor):
         # Assigning to __doc__, definitely a string
         try:
             while True:
                 if (ancestor.type == syms.expr_stmt and
                     Name('__doc__') in ancestor.children):
                         return True
                 ancestor = ancestor.parent
         except AttributeError:
             return False
 
     if ismodule(node.parent) or \
        isdocassignment(node.parent) or \
        isclassorfunction(node.parent):
         return True
     return False
 
 def shouldtransform(node):
     specialnames = ['__main__']
 
     if node.value in specialnames:
         return False
 
     ggparent = node.parent.parent.parent
     sggparent = str(ggparent)
 
     if 'getattr' in sggparent or \
        'hasattr' in sggparent or \
        'setattr' in sggparent or \
        'encode' in sggparent or \
        'decode' in sggparent:
            return False
 
     return True
 
 class FixBytes(fixer_base.BaseFix):
 
     PATTERN = 'STRING'
 
     def transform(self, node, results):
         if self.filename in blacklist:
             return
         if node.type == token.STRING:
             if _re.match(node.value):
                 if isdocstring(node):
                     return
                 if not shouldtransform(node):
                     return
                 new = node.clone()
                 new.value = 'b' + new.value
                 return new