patch: do not break up multibyte character when highlighting word

This changes {\W} to {\W - any 8bit characters} so that multibyte sequences
are taken as words. Since we don't know the encoding of user content, this
is the most sensible definition of a non-word.

mercurial/patch.py

@@ -46,6 +46,7 b' stringio = util.stringio'
 
 gitre = re.compile(br'diff --git a/(.*) b/(.*)')
 tabsplitter = re.compile(br'(\t+|[^\t]+)')
+_nonwordre = re.compile(br'([^a-zA-Z0-9_\x80-\xff])')
 
 PatchError = error.PatchError
 
@@ -2578,7 +2579,7 b' def _inlinediff(s1, s2, operation):'
         raise error.ProgrammingError("Case not expected, operation = %s" %
                                      operation)
 
-    s = difflib.ndiff(re.split(br'(\W)', s2), re.split(br'(\W)', s1))
+    s = difflib.ndiff(_nonwordre.split(s2), _nonwordre.split(s1))
     for part in s:
         if part[0] in operation_skip or len(part) == 2:
             continue

tests/test-diff-color.t

@@ -370,3 +370,23 b' test inline color diff'
   [diff.deleted|-(to see if it works)]
   [diff.inserted|+three of those lines ][diff.inserted.highlight|have]
   [diff.inserted|+][diff.inserted.highlight|collapsed][diff.inserted| onto one]
+
+multibyte character shouldn't be broken up in word diff:
+
+  $ $PYTHON <<'EOF'
+  > with open("utf8", "wb") as f:
+  >     f.write(b"blah \xe3\x82\xa2 blah\n")
+  > EOF
+  $ hg ci -Am 'add utf8 char' utf8
+  $ $PYTHON <<'EOF'
+  > with open("utf8", "wb") as f:
+  >     f.write(b"blah \xe3\x82\xa4 blah\n")
+  > EOF
+  $ hg ci -m 'slightly change utf8 char' utf8
+  $ hg diff --config experimental.worddiff=True --color=debug -c.
+  [diff.diffline|diff --git a/utf8 b/utf8]
+  [diff.file_a|--- a/utf8]
+  [diff.file_b|+++ b/utf8]
+  [diff.hunk|@@ -1,1 +1,1 @@]
+  [diff.deleted|-blah ][diff.deleted.highlight|\xe3\x82\xa2][diff.deleted| blah] (esc)
+  [diff.inserted|+blah ][diff.inserted.highlight|\xe3\x82\xa4][diff.inserted| blah] (esc)