extdiff: avoid unexpected quoting arguments for external tools (issue4463)

Before this patch, all command line arguments for external tools are
quoted by the combination of "shlex.split" and "util.shellquote". But
this causes some problems.

  - some problematic commands can't work correctly with quoted arguments

    For example, 'WinMerge /r ....' is OK, but 'WinMerge "/r" ....' is
    NG. See also below for detail about this problem.

        https://bitbucket.org/tortoisehg/thg/issue/3978/

  - quoting itself may change semantics of arguments

    For example, when the environment variable CONCAT="foo bar baz':

      - mydiff $CONCAT   => mydiff foo bar baz   (taking 3 arguments)
      - mydiff "$CONCAT" => mydiff "foo bar baz" (taking only 1 argument)

    For another example, single quoting (= "util.shellquote") on POSIX
    environment prevents shells from expanding environment variables,
    tilde, and so on:

      - mydiff "$HOME" => mydiff /home/foobar
      - mydiff '$HOME' => mydiff $HOME

  - "shlex.split" can't handle some special characters correctly

    It just splits specified command line by whitespaces.

    For example, "echo foo;echo bar" is split into ["echo",
    "foo;echo", "bar"].

On the other hand, if quoting itself is omitted, users can't specify
options including space characters with "--option" at runtime.

The root cause of this issue is that "shlex.split + util.shellquote"
combination loses whether users really want to quote each command line
elements or not, even though these can be quoted arbitrarily in
configurations.

To resolve this problem, this patch does:

  - prevent configurations from being processed by "shlex.split" and
    "util.shellquote"

    only (possibly) "findexe"-ed or "findexternaltool"-ed command path
    is "util.shellquote", because it may contain whitespaces.

  - quote options specified by "--option" via command line at runtime

This patch also makes "dodiff()" take only one "args" argument instead
of "diffcmd" and "diffopts. It also omits applying "util.shellquote"
on "args", because "args" should be already stringified in "extdiff()"
and "mydiff()".

The last hunk for "test-extdiff.t" replaces two whitespaces by single
whitespace, because change of "' '.join()" logic causes omitting
redundant whitespaces.

hgext/extdiff.py

@@ -109,7 +109,7 b' def snapshot(ui, repo, files, node, tmpr'
                                   os.lstat(dest).st_mtime))
     return dirname, fns_and_mtime
 
-def dodiff(ui, repo, diffcmd, diffopts, pats, opts):
+def dodiff(ui, repo, args, pats, opts):
     '''Do the actual diff:
 
     - copy to a temp structure if diffing 2 internal revisions
@@ -120,7 +120,6 b' def dodiff(ui, repo, diffcmd, diffopts, '
 
     revs = opts.get('rev')
     change = opts.get('change')
-    args = ' '.join(map(util.shellquote, diffopts))
     do3way = '$parent2' in args
 
     if revs and change:
@@ -222,8 +221,7 b' def dodiff(ui, repo, diffcmd, diffopts, '
         regex = '\$(parent2|parent1?|child|plabel1|plabel2|clabel|root)'
         if not do3way and not re.search(regex, args):
             args += ' $parent1 $child'
-        args = re.sub(regex, quote, args)
-        cmdline = util.shellquote(diffcmd) + ' ' + args
+        cmdline = re.sub(regex, quote, args)
 
         ui.debug('running %r in %s\n' % (cmdline, tmproot))
         ui.system(cmdline, cwd=tmproot)
@@ -271,7 +269,8 b' def extdiff(ui, repo, *pats, **opts):'
     if not program:
         program = 'diff'
         option = option or ['-Npru']
-    return dodiff(ui, repo, program, option, pats, opts)
+    cmdline = ' '.join(map(util.shellquote, [program] + option))
+    return dodiff(ui, repo, cmdline, pats, opts)
 
 def uisetup(ui):
     for cmd, path in ui.configitems('extdiff'):
@@ -281,29 +280,37 b' def uisetup(ui):'
                 path = util.findexe(cmd)
                 if path is None:
                     path = filemerge.findexternaltool(ui, cmd) or cmd
-            diffopts = shlex.split(ui.config('extdiff', 'opts.' + cmd, ''))
+            diffopts = ui.config('extdiff', 'opts.' + cmd, '')
+            cmdline = util.shellquote(path)
+            if diffopts:
+                cmdline += ' ' + diffopts
         elif cmd.startswith('opts.'):
             continue
         else:
-            # command = path opts
             if path:
-                diffopts = shlex.split(path)
-                path = diffopts.pop(0)
+                # case "cmd = path opts"
+                cmdline = path
+                diffopts = len(shlex.split(cmdline)) > 1
             else:
-                path, diffopts = util.findexe(cmd), []
+                # case "cmd ="
+                path = util.findexe(cmd)
                 if path is None:
                     path = filemerge.findexternaltool(ui, cmd) or cmd
+                cmdline = util.shellquote(path)
+                diffopts = False
         # look for diff arguments in [diff-tools] then [merge-tools]
-        if diffopts == []:
+        if not diffopts:
             args = ui.config('diff-tools', cmd+'.diffargs') or \
                    ui.config('merge-tools', cmd+'.diffargs')
             if args:
-                diffopts = shlex.split(args)
-        def save(cmd, path, diffopts):
+                cmdline += ' ' + args
+        def save(cmdline):
             '''use closure to save diff command to use'''
             def mydiff(ui, repo, *pats, **opts):
-                return dodiff(ui, repo, path, diffopts + opts['option'],
-                              pats, opts)
+                options = ' '.join(map(util.shellquote, opts['option']))
+                if options:
+                    options = ' ' + options
+                return dodiff(ui, repo, cmdline + options, pats, opts)
             doc = _('''\
 use %(path)s to diff repository (or selected files)
 
@@ -325,6 +332,6 b' use %(path)s to diff repository (or sele'
             # right encoding) prevents that.
             mydiff.__doc__ = doc.decode(encoding.encoding)
             return mydiff
-        cmdtable[cmd] = (save(cmd, path, diffopts),
+        cmdtable[cmd] = (save(cmdline),
                          cmdtable['extdiff'][1][1:],
                          _('hg %s [OPTION]... [FILE]...') % cmd)

tests/test-extdiff.t

@@ -94,6 +94,72 b' Check diff are made from the first paren'
   diffing */extdiff.*/a.2a13a4d2da36/a a.46c0e4daeb72/a (glob)
   diff-like tools yield a non-zero exit code
 
+issue4463: usage of command line configuration without additional quoting
+
+  $ cat <<EOF >> $HGRCPATH
+  > [extdiff]
+  > cmd.4463a = echo
+  > opts.4463a = a-naked 'single quoted' "double quoted"
+  > 4463b = echo b-naked 'single quoted' "double quoted"
+  > echo =
+  > EOF
+  $ hg update -q -C 0
+  $ echo a >> a
+#if windows
+  $ hg --debug 4463a | grep '^running'
+  running '"echo" a-naked \'single quoted\' "double quoted" "*\\a" "*\\a"' in */extdiff.* (glob)
+  $ hg --debug 4463b | grep '^running'
+  running 'echo b-naked \'single quoted\' "double quoted" "*\\a" "*\\a"' in */extdiff.* (glob)
+  $ hg --debug echo | grep '^running'
+  running '"*echo*" "*\\a" "*\\a"' in */extdiff.* (glob)
+#else
+  $ hg --debug 4463a | grep '^running'
+  running '\'echo\' a-naked \'single quoted\' "double quoted" \'*/a\' \'$TESTTMP/a/a\'' in */extdiff.* (glob)
+  $ hg --debug 4463b | grep '^running'
+  running 'echo b-naked \'single quoted\' "double quoted" \'*/a\' \'$TESTTMP/a/a\'' in */extdiff.* (glob)
+  $ hg --debug echo | grep '^running'
+  running "'*echo*' '*/a' '$TESTTMP/a/a'" in */extdiff.* (glob)
+#endif
+
+(getting options from other than extdiff section)
+
+  $ cat <<EOF >> $HGRCPATH
+  > [extdiff]
+  > # using diff-tools diffargs
+  > 4463b2 = echo
+  > # using merge-tools diffargs
+  > 4463b3 = echo
+  > # no diffargs
+  > 4463b4 = echo
+  > [diff-tools]
+  > 4463b2.diffargs = b2-naked 'single quoted' "double quoted"
+  > [merge-tools]
+  > 4463b3.diffargs = b3-naked 'single quoted' "double quoted"
+  > EOF
+#if windows
+  $ hg --debug 4463b2 | grep '^running'
+  running 'echo b2-naked \'single quoted\' "double quoted" "*\\a" "*\\a"' in */extdiff.* (glob)
+  $ hg --debug 4463b3 | grep '^running'
+  running 'echo b3-naked \'single quoted\' "double quoted" "*\\a" "*\\a"' in */extdiff.* (glob)
+  $ hg --debug 4463b4 | grep '^running'
+  running 'echo "*\\a" "*\\a"' in */extdiff.* (glob)
+  $ hg --debug 4463b4 --option 'being quoted' | grep '^running'
+  running 'echo "being quoted" "*\\a" "*\\a"' in */extdiff.* (glob)
+  $ hg --debug extdiff -p echo --option 'being quoted' | grep '^running'
+  running '"echo" "being quoted" "*\\a" "*\\a"' in */extdiff.* (glob)
+#else
+  $ hg --debug 4463b2 | grep '^running'
+  running 'echo b2-naked \'single quoted\' "double quoted" \'*/a\' \'$TESTTMP/a/a\'' in */extdiff.* (glob)
+  $ hg --debug 4463b3 | grep '^running'
+  running 'echo b3-naked \'single quoted\' "double quoted" \'*/a\' \'$TESTTMP/a/a\'' in */extdiff.* (glob)
+  $ hg --debug 4463b4 | grep '^running'
+  running "echo '*/a' '$TESTTMP/a/a'" in */extdiff.* (glob)
+  $ hg --debug 4463b4 --option 'being quoted' | grep '^running'
+  running "echo 'being quoted' '*/a' '$TESTTMP/a/a'" in */extdiff.* (glob)
+  $ hg --debug extdiff -p echo --option 'being quoted' | grep '^running'
+  running "'echo' 'being quoted' '*/a' '$TESTTMP/a/a'" in */extdiff.* (glob)
+#endif
+
 #if execbit
 
 Test extdiff of multiple files in tmp dir:
@@ -207,7 +273,7 b' Fallback to merge-tools.tool.executable|'
   making snapshot of 2 files from working directory
     a
     b
-  running "'$TESTTMP/a/dir/tool.sh'  'a.*' 'a'" in */extdiff.* (glob)
+  running "'$TESTTMP/a/dir/tool.sh' 'a.*' 'a'" in */extdiff.* (glob)
   ** custom diff **
   cleaning up temp directory
   [1]