setup: use sysstr() on process output

Otherwise we get a str-bytes mismatch on Python 3 if
an error occurs.

Differential Revision: https://phab.mercurial-scm.org/D8469

setup.py

@@ -890,7 +890,8 b' class hgbuilddoc(Command):'
             )
             if res:
                 raise SystemExit(
-                    'error running gendoc.py: %s' % '\n'.join([out, err])
+                    'error running gendoc.py: %s'
+                    % '\n'.join([sysstr(out), sysstr(err)])
                 )
 
             with open(txt, 'wb') as fh:
@@ -907,7 +908,8 b' class hgbuilddoc(Command):'
             )
             if res:
                 raise SystemExit(
-                    'error running gendoc: %s' % '\n'.join([out, err])
+                    'error running gendoc: %s'
+                    % '\n'.join([sysstr(out), sysstr(err)])
                 )
 
             with open(gendoc, 'wb') as fh:
@@ -932,7 +934,8 b' class hgbuilddoc(Command):'
             )
             if res:
                 raise SystemExit(
-                    'error running runrst: %s' % '\n'.join([out, err])
+                    'error running runrst: %s'
+                    % '\n'.join([sysstr(out), sysstr(err)])
                 )
 
             normalizecrlf('doc/%s' % root)
@@ -957,7 +960,8 b' class hgbuilddoc(Command):'
             )
             if res:
                 raise SystemExit(
-                    'error running runrst: %s' % '\n'.join([out, err])
+                    'error running runrst: %s'
+                    % '\n'.join([sysstr(out), sysstr(err)])
                 )
 
             normalizecrlf('doc/%s.html' % root)