templatefilters: make json filter be byte-transparent (BC) (issue4926)

This is necessary to preserve filename encoding over JSON. Instead, this
patch inserts "|utf8" where non-ascii local-encoding texts can be passed
to "|json".

See also the commit that introduced "utf8" filter.

mercurial/hgweb/webcommands.py

@@ -1196,7 +1196,8 b' def graph(web, req, tmpl):'
                 canvaswidth=(cols + 1) * bg_height,
                 truecanvasheight=rows * bg_height,
                 canvasheight=canvasheight, bg_height=bg_height,
-                jsdata=lambda **x: graphdata(True, str),
+                # {jsdata} will be passed to |json, so it must be in utf-8
+                jsdata=lambda **x: graphdata(True, encoding.fromlocal),
                 nodes=lambda **x: graphdata(False, str),
                 node=ctx.hex(), changenav=changenav)
 

mercurial/templatefilters.py

@@ -197,15 +197,8 b' def json(obj):'
         return {None: 'null', False: 'false', True: 'true'}[obj]
     elif isinstance(obj, int) or isinstance(obj, float):
         return str(obj)
-    elif isinstance(obj, encoding.localstr):
-        u = encoding.fromlocal(obj).decode('utf-8')  # can round-trip
-        return '"%s"' % jsonescape(u)
     elif isinstance(obj, str):
-        # no encoding.fromlocal() because it may abort if obj can't be decoded
-        u = unicode(obj, encoding.encoding, 'replace')
-        return '"%s"' % jsonescape(u)
-    elif isinstance(obj, unicode):
-        return '"%s"' % jsonescape(obj)
+        return '"%s"' % encoding.jsonescape(obj, paranoid=True)
     elif util.safehasattr(obj, 'keys'):
         out = []
         for k, v in sorted(obj.iteritems()):

mercurial/templates/json/map

@@ -8,26 +8,26 b' shortlog = changelist.tmpl'
 changelistentry = '\{
   "node": {node|json},
   "date": {date|json},
-  "desc": {desc|json},
+  "desc": {desc|utf8|json},
   "bookmarks": [{join(bookmarks%changelistentryname, ", ")}],
   "tags": [{join(tags%changelistentryname, ", ")}],
-  "user": {author|json}
+  "user": {author|utf8|json}
   }'
-changelistentryname = '{name|json}'
+changelistentryname = '{name|utf8|json}'
 changeset = '\{
   "node": {node|json},
   "date": {date|json},
-  "desc": {desc|json},
+  "desc": {desc|utf8|json},
   "branch": {if(branch, branch%changesetbranch, "default"|json)},
   "bookmarks": [{join(changesetbookmark, ", ")}],
   "tags": [{join(changesettag, ", ")}],
-  "user": {author|json},
+  "user": {author|utf8|json},
   "parents": [{join(parent%changesetparent, ", ")}],
   "phase": {phase|json}
   }'
-changesetbranch = '{name|json}'
-changesetbookmark = '{bookmark|json}'
-changesettag = '{tag|json}'
+changesetbranch = '{name|utf8|json}'
+changesetbookmark = '{bookmark|utf8|json}'
+changesettag = '{tag|utf8|json}'
 changesetparent = '{node|json}'
 manifest = '\{
   "node": {node|json},
@@ -37,7 +37,7 b" manifest = '\\{"
   "bookmarks": [{join(bookmarks%name, ", ")}],
   "tags": [{join(tags%name, ", ")}]
   }'
-name = '{name|json}'
+name = '{name|utf8|json}'
 direntry = '\{
   "abspath": {path|json},
   "basename": {basename|json},
@@ -55,7 +55,7 b" tags = '\\{"
   "tags": [{join(entriesnotip%tagentry, ", ")}]
   }'
 tagentry = '\{
-  "tag": {tag|json},
+  "tag": {tag|utf8|json},
   "node": {node|json},
   "date": {date|json}
   }'
@@ -64,7 +64,7 b" bookmarks = '\\{"
   "bookmarks": [{join(entries%bookmarkentry, ", ")}]
   }'
 bookmarkentry = '\{
-  "bookmark": {bookmark|json},
+  "bookmark": {bookmark|utf8|json},
   "node": {node|json},
   "date": {date|json}
   }'
@@ -72,7 +72,7 b" branches = '\\{"
   "branches": [{join(entries%branchentry, ", ")}]
   }'
 branchentry = '\{
-  "branch": {branch|json},
+  "branch": {branch|utf8|json},
   "node": {node|json},
   "date": {date|json},
   "status": {status|json}
@@ -82,8 +82,8 b" filediff = '\\{"
   "path": {file|json},
   "node": {node|json},
   "date": {date|json},
-  "desc": {desc|json},
-  "author": {author|json},
+  "desc": {desc|utf8|json},
+  "author": {author|utf8|json},
   "parents": [{join(parent%changesetparent, ", ")}],
   "children": [{join(child%changesetparent, ", ")}],
   "diff": [{join(diff%diffblock, ", ")}]
@@ -116,8 +116,8 b" filecomparison = '\\{"
   "path": {file|json},
   "node": {node|json},
   "date": {date|json},
-  "desc": {desc|json},
-  "author": {author|json},
+  "desc": {desc|utf8|json},
+  "author": {author|utf8|json},
   "parents": [{join(parent%changesetparent, ", ")}],
   "children": [{join(child%changesetparent, ", ")}],
   "leftnode": {leftnode|json},
@@ -137,9 +137,9 b" comparisonline = '\\{"
 fileannotate = '\{
   "abspath": {file|json},
   "node": {node|json},
-  "author": {author|json},
+  "author": {author|utf8|json},
   "date": {date|json},
-  "desc": {desc|json},
+  "desc": {desc|utf8|json},
   "parents": [{join(parent%changesetparent, ", ")}],
   "children": [{join(child%changesetparent, ", ")}],
   "permissions": {permissions|json},
@@ -147,8 +147,8 b" fileannotate = '\\{"
   }'
 fileannotation = '\{
   "node": {node|json},
-  "author": {author|json},
-  "desc": {desc|json},
+  "author": {author|utf8|json},
+  "desc": {desc|utf8|json},
   "abspath": {file|json},
   "targetline": {targetline|json},
   "line": {line|json},
@@ -163,12 +163,12 b" helptopics = '\\{"
   "othercommands": [{join(othercommands%helptopicentry, ", ")}]
   }'
 helptopicentry = '\{
-  "topic": {topic|json},
-  "summary": {summary|json}
+  "topic": {topic|utf8|json},
+  "summary": {summary|utf8|json}
   }'
 help = '\{
-  "topic": {topic|json},
-  "rawdoc": {doc|json}
+  "topic": {topic|utf8|json},
+  "rawdoc": {doc|utf8|json}
   }'
 filenodelink = ''
 filenolink = ''

tests/test-command-template.t

@@ -3542,6 +3542,11 b' Test broken string escapes:'
   hg: parse error: invalid \x escape
   [255]
 
+json filter should escape HTML tags so that the output can be embedded in hgweb:
+
+  $ hg log -T "{'<[e-mail]>'|json}\n" -R a -l1
+  "\[e-mail]\u003e"
+
 Set up repository for non-ascii encoding tests:
 
   $ hg init nonascii
@@ -3558,11 +3563,12 b' json filter should try round-trip conver'
   $ HGENCODING=ascii hg log -T "{branch|json}\n" -r0
   "\u00e9"
 
-json filter should not abort if it can't decode bytes:
-(not sure the current behavior is right; we might want to use utf-8b encoding?)
+json filter takes input as utf-8b:
 
   $ HGENCODING=ascii hg log -T "{'`cat utf-8`'|json}\n" -l1
-  "\ufffd\ufffd"
+  "\u00e9"
+  $ HGENCODING=ascii hg log -T "{'`cat latin1`'|json}\n" -l1
+  "\udce9"
 
 utf8 filter: