diff --git a/mercurial/color.py b/mercurial/color.py
--- a/mercurial/color.py
+++ b/mercurial/color.py
@@ -7,6 +7,8 @@
 
 from __future__ import absolute_import
 
+import re
+
 from .i18n import _
 
 from . import (
@@ -327,6 +329,12 @@ def _render_effects(ui, text, effects):
         stop = '\033[' + str(_effects['none']) + 'm'
     return _mergeeffects(text, start, stop)
 
+_ansieffectre = re.compile(br'\x1b\[[0-9;]*m')
+
+def stripeffects(text):
+    """Strip ANSI control codes which could be inserted by colorlabel()"""
+    return _ansieffectre.sub('', text)
+
 def colorlabel(ui, msg, label):
     """add color control code according to the mode"""
     if ui._colormode == 'debug':
@@ -352,7 +360,6 @@ def colorlabel(ui, msg, label):
 w32effects = None
 if pycompat.osname == 'nt':
     import ctypes
-    import re
 
     _kernel32 = ctypes.windll.kernel32
 
diff --git a/mercurial/templater.py b/mercurial/templater.py
--- a/mercurial/templater.py
+++ b/mercurial/templater.py
@@ -13,6 +13,7 @@ import types
 
 from .i18n import _
 from . import (
+    color,
     config,
     encoding,
     error,
@@ -576,13 +577,13 @@ def pad(context, mapping, args):
     fillchar = ' '
     if len(args) > 2:
         fillchar = evalstring(context, mapping, args[2])
-        if len(fillchar) != 1:
+        if len(color.stripeffects(fillchar)) != 1:
             # i18n: "pad" is a keyword
             raise error.ParseError(_("pad() expects a single fill character"))
     if len(args) > 3:
         left = evalboolean(context, mapping, args[3])
 
-    fillwidth = width - encoding.colwidth(text)
+    fillwidth = width - encoding.colwidth(color.stripeffects(text))
     if fillwidth <= 0:
         return text
     if left:
diff --git a/tests/test-command-template.t b/tests/test-command-template.t
--- a/tests/test-command-template.t
+++ b/tests/test-command-template.t
@@ -3354,6 +3354,12 @@ color effects can be nested (issue5413)
   > '{label(red, "red{label(magenta, "ma{label(cyan, "cyan")}{label(yellow, "yellow")}genta")}")}\n'
   \x1b[0;31mred\x1b[0;35mma\x1b[0;36mcyan\x1b[0m\x1b[0;31m\x1b[0;35m\x1b[0;33myellow\x1b[0m\x1b[0;31m\x1b[0;35mgenta\x1b[0m (esc)
 
+pad() should interact well with color codes (issue5416)
+
+  $ hg debugtemplate --color=always \
+  > '{pad(label(red, "red"), 5, label(cyan, "-"))}\n'
+  \x1b[0;31mred\x1b[0m\x1b[0;36m-\x1b[0m\x1b[0;36m-\x1b[0m (esc)
+
 label should be no-op if color is disabled:
 
   $ hg log --color=never -l 1 --template '{label(red, "text\n")}'