diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -890,7 +890,8 @@ class hgbuilddoc(Command):
             )
             if res:
                 raise SystemExit(
-                    'error running gendoc.py: %s' % '\n'.join([out, err])
+                    'error running gendoc.py: %s'
+                    % '\n'.join([sysstr(out), sysstr(err)])
                 )
 
             with open(txt, 'wb') as fh:
@@ -907,7 +908,8 @@ class hgbuilddoc(Command):
             )
             if res:
                 raise SystemExit(
-                    'error running gendoc: %s' % '\n'.join([out, err])
+                    'error running gendoc: %s'
+                    % '\n'.join([sysstr(out), sysstr(err)])
                 )
 
             with open(gendoc, 'wb') as fh:
@@ -932,7 +934,8 @@ class hgbuilddoc(Command):
             )
             if res:
                 raise SystemExit(
-                    'error running runrst: %s' % '\n'.join([out, err])
+                    'error running runrst: %s'
+                    % '\n'.join([sysstr(out), sysstr(err)])
                 )
 
             normalizecrlf('doc/%s' % root)
@@ -957,7 +960,8 @@ class hgbuilddoc(Command):
             )
             if res:
                 raise SystemExit(
-                    'error running runrst: %s' % '\n'.join([out, err])
+                    'error running runrst: %s'
+                    % '\n'.join([sysstr(out), sysstr(err)])
                 )
 
             normalizecrlf('doc/%s.html' % root)