relnotes: various tweaks for release notes

Stop filtering out commits that are expected to be covered by releasenotes
extension: now we want two lists, one for WhatsNew and one for ReleaseX.Y.

Use `only(stoprev, startrev)` to make `relnotes -h` output be actually true
about what revisions are included.

More filter rules, mostly obvious.

More classifying rules to have less things in "unsorted".

Looks like nargs=1 was just making args.startrev and args.stoprev be lists for
no reason.

BC and API sections are renamed to what we're using on the WhatsNew page, and
also just skipped if empty.

contrib/relnotes

@@ -9,11 +9,6 b' import argparse'
 import re
 import subprocess
 
-# Regenerate this list with
-#   hg export 'grep("\.\. [a-z]+::")' | grep '^\.\.' | \
-#     sed 's/.. //;s/::.*//' | sort -u
-rnsections = ["api", "bc", "container", "feature", "fix", "note", "perf"]
-
 rules = {
     # keep
     r"\(issue": 100,
@@ -26,27 +21,37 b' rules = {'
     r"(mq|shelve|rebase):": 20,
     # newsy
     r": deprecate": 20,
-    r"(option|feature|command|support)": 10,
+    r"( ability|command|feature|option|support)": 10,
+    # experimental
+    r"hg-experimental": 20,
+    r"(from|graduate).*experimental": 15,
+    r"(hide|mark).*experimental": -10,
     # bug-like?
     r"(fix|don't break|improve)": 7,
+    r"(not|n't|avoid|fix|prevent).*crash": 10,
     # boring stuff, bump down
     r"^contrib": -5,
     r"debug": -5,
     r"help": -5,
-    r"(doc|bundle2|obsolete|obsmarker|rpm|setup|debug\S+:)": -15,
-    r"(check-code|check-commit|import-checker)": -20,
+    r"(doc|metavar|bundle2|obsolete|obsmarker|rpm|setup|debug\S+:)": -15,
+    r"(check-code|check-commit|check-config|import-checker)": -20,
+    r"(flake8|lintian|pyflakes|pylint)": -20,
     # cleanups and refactoring
-    r"(cleanup|whitespace|nesting|indent|spelling|comment)": -20,
-    r"(typo|hint|note|style:|correct doc)": -20,
+    r"(cleanup|white ?space|spelling|quoting)": -20,
+    r"(flatten|dedent|indent|nesting|unnest)": -20,
+    r"(typo|hint|note|comment|TODO|FIXME)": -20,
+    r"(style:|convention|one-?liner)": -20,
     r"_": -10,
     r"(argument|absolute_import|attribute|assignment|mutable)": -15,
-    r"(unused|useless|unnecessary|duplicate|deprecated|scope|True|False)": -10,
+    r"(scope|True|False)": -10,
+    r"(unused|useless|unnecessary|superfluous|duplicate|deprecated)": -10,
     r"(redundant|pointless|confusing|uninitialized|meaningless|dead)": -10,
-    r": (drop|remove|inherit|rename|simplify|naming|inline)": -10,
-    r"(docstring|document .* method)": -20,
-    r"(factor|extract|prepare|split|replace| import)": -20,
-    r": add.*(function|method|implementation|test|example)": -10,
-    r": (move|extract) .* (to|into|from)": -20,
+    r": (drop|remove|delete|rip out)": -10,
+    r": (inherit|rename|simplify|naming|inline)": -10,
+    r"(correct doc|docstring|document .* method)": -20,
+    r"(abstract|factor|extract|prepare|split|replace| import)": -20,
+    r": add.*(function|method|implementation|example)": -10,
+    r": (move|extract) .* (to|into|from|out of)": -20,
     r": implement ": -5,
     r": use .* implementation": -20,
     r"\S\S\S+\.\S\S\S\S+": -5,
@@ -55,15 +60,20 b' rules = {'
     # dumb keywords
     r"\S+/\S+:": -10,
     r"\S+\.\S+:": -10,
+    # python compatibility
+    r"[Pp]y(|thon) ?[23]": -20,
+    r"pycompat": -20,
+    r"(coerce|convert|encode) .*to (byte|sys|)(s|str|string)": -20,
+    # tests
+    r"^test(|s|ing|runner|-\S+):": -20,
+    r"^(f|hghave|run-tests):": -20,
+    r"add.* tests?": -20,
+    r"(buildbot|fuzz|mock|ratchet)": -10,
     # drop
     r"^i18n-": -50,
     r"^i18n:.*(hint|comment)": -50,
     r"perf:": -50,
-    r"check-code:": -50,
     r"Added.*for changeset": -50,
-    r"tests?:": -50,
-    r"test-": -50,
-    r"add.* tests": -50,
     r"^_": -50,
 }
 
@@ -72,9 +82,9 b' commits = []'
 
 groupings = [
     (r"util|parsers|repo|ctx|context|revlog|filelog|alias|cmdutil", "core"),
-    (r"revset|templater|ui|dirstate|hook|i18n|transaction|wire", "core"),
-    (r"color|pager", "core"),
-    (r"hgweb|paper|coal|gitweb", "hgweb"),
+    (r"revset|template|ui|dirstate|hook|i18n|transaction|wire|vfs", "core"),
+    (r"dispatch|exchange|localrepo|streamclone|color|pager", "core"),
+    (r"hgweb|paper|coal|gitweb|monoblue|spartan", "hgweb"),
     (r"pull|push|revert|resolve|annotate|bookmark|branch|clone", "commands"),
     (r"commands|commit|config|files|graft|import|log|merge|patch", "commands"),
     (r"phases|status|summary|amend|tag|help|verify", "commands"),
@@ -83,12 +93,12 b' groupings = ['
 ]
 
 def main():
-    ap = argparse.ArgumentParser()
+    desc = "example: %(prog)s 4.7.2 --stoprev 4.8rc0"
+    ap = argparse.ArgumentParser(description=desc)
     ap.add_argument(
         "startrev",
         metavar="REV",
         type=str,
-        nargs=1,
         help=(
             "Starting revision for the release notes. This revision "
             "won't be included, but later revisions will."
@@ -99,7 +109,6 b' def main():'
         metavar="REV",
         type=str,
         default="@",
-        nargs=1,
         help=(
             "Stop revision for release notes. This revision will be included,"
             " but no later revisions will. This revision needs to be "
@@ -114,7 +123,7 b' def main():'
             "extensions.releasenotes=",
             "releasenotes",
             "-r",
-            "%s::%s" % (args.startrev[0], args.stoprev[0]),
+            "only(%s, %s)" % (args.stoprev, args.startrev),
         ]
     ).decode("utf-8")
     # Find all release notes from un-relnotes-flagged commits.
@@ -124,8 +133,7 b' def main():'
                 "hg",
                 "log",
                 "-r",
-                r'%s::%s - merge() - grep("\n\.\. (%s)::")'
-                % (args.startrev[0], args.stoprev[0], "|".join(rnsections)),
+                "only(%s, %s) - merge()" % (args.stoprev, args.startrev),
                 "-T",
                 r"{desc|firstline}\n",
             ]
@@ -167,12 +175,14 b' def main():'
         for d in sorted(groups[g]):
             print(" * %s" % d)
 
-    print("\n=== BC ===\n")
+    if bcs:
+        print("\n=== Behavior Changes ===\n")
 
     for d in sorted(bcs):
         print(" * %s" % d)
 
-    print("\n=== API Changes ===\n")
+    if apis:
+        print("\n=== Internal API Changes ===\n")
 
     for d in sorted(apis):
         print(" * %s" % d)

tests/test-contrib-relnotes.t

@@ -132,28 +132,34 b''
    * bookmark: introduce a 'bookmarks' part
    * bookmark: introduce in advance a variant of the exchange test
    * bookmark: run 'pushkey' hooks after bookmark move, not 'prepushkey'
+   * bookmark: use the 'bookmarks' bundle2 part to push bookmark update (Bts:issue5165)
    * bookmarks: add bookmarks to hidden revs if directaccess config is set
    * bookmarks: calculate visibility exceptions only once
    * bookmarks: display the obsfate of hidden revision we create a bookmark on
    * bookmarks: fix pushkey compatibility mode (Bts:issue5777)
    * bookmarks: use context managers for lock and transaction in update()
    * bookmarks: use context managers for locks and transaction in pushbookmark()
+   * branch: add a --rev flag to change branch name of given revisions
    * branch: allow changing branch name to existing name if possible
    * clone: add support for storing remotenames while cloning
    * clone: use utility function to write hgrc
    * clonebundle: make it possible to retrieve the initial bundle through largefile
+   * commands: use the new API to access hidden changesets in various commands
    * commandserver: restore cwd in case of exception
    * commandserver: unblock SIGCHLD
+   * fileset: do not crash by unary negate operation
    * help: deprecate ui.slash in favor of slashpath template filter (Bts:issue5572)
    * log: allow matchfn to be non-null even if both --patch/--stat are off
    * log: build follow-log filematcher at once
    * log: don't expand aliases in revset built from command options
+   * log: follow file history across copies even with -rREV (BC) (Bts:issue4959)
    * log: make "slowpath" condition slightly more readable
    * log: make opt2revset table a module constant
    * log: merge getlogrevs() and getgraphlogrevs()
    * log: remove temporary variable 'date' used only once
    * log: resolve --follow thoroughly in getlogrevs()
    * log: resolve --follow with -rREV in cmdutil.getlogrevs()
+   * log: rewrite --follow-first -rREV like --follow for consistency (BC)
    * log: simplify 'x or ancestors(x)' expression
    * log: translate column labels at once (Bts:issue5750)
    * log: use revsetlang.formatspec() thoroughly
@@ -172,10 +178,23 b''
    * pull: store binary node in pullop.remotebookmarks
    * push: include a 'check:bookmarks' part when possible
    * push: restrict common discovery to the pushed set
+   * revert: do not reverse hunks in interactive when REV is not parent (Bts:issue5096)
    * revert: support reverting to hidden cset if directaccess config is set
   
   === core ===
+   * color: respect HGPLAINEXCEPT=color to allow colors while scripting (Bts:issue5749)
+   * dirstate: add explicit methods for querying directories (API)
+   * dispatch: abort if early boolean options can't be parsed
+   * dispatch: add HGPLAIN=+strictflags to restrict early parsing of global options
+   * dispatch: add option to not strip command args parsed by _earlygetopt()
+   * dispatch: alias --repo to --repository while parsing early options
+   * dispatch: fix early parsing of short option with value like -R=foo
+   * dispatch: handle IOError when writing to stderr
+   * dispatch: stop parsing of early boolean option at "--"
+   * dispatch: verify result of early command parsing
+   * exchange: return bundle info from getbundlechunks() (API)
    * filelog: add the ability to report the user facing name
+   * localrepo: specify optional callback parameter to pathauditor as a keyword
    * revlog: choose between ifh and dfh once for all
    * revlog: don't use slicing to return parents
    * revlog: group delta computation methods under _deltacomputer object
@@ -185,9 +204,21 b''
    * revlog: separate diff computation from the collection of other info
    * revset: evaluate filesets against each revision for 'file()' (Bts:issue5778)
    * revset: parse x^:: as (x^):: (Bts:issue5764)
+   * streamclone: add support for bundle2 based stream clone
+   * streamclone: add support for cloning non append-only file
+   * streamclone: also stream caches to the client
+   * streamclone: define first iteration of version 2 of stream format
+   * streamclone: move wire protocol status code from wireproto command
+   * streamclone: rework canperformstreamclone
+   * streamclone: tests phase exchange during stream clone
+   * streamclone: use readexactly when reading stream v2
+   * templater: fix crash by empty group expression
+   * templater: keep default resources per template engine (API)
    * templater: look up symbols/resources as if they were separated (Bts:issue5699)
    * transaction: register summary callbacks only at start of transaction (BC)
    * util: whitelist NTFS for hardlink creation (Bts:issue4580)
+   * vfs: drop text mode flag (API)
+   * wireproto: drop support for reader interface from streamres (API)
   
   === extensions ===
    * convert: restore the ability to use bzr < 2.6.0 (Bts:issue5733)
@@ -195,7 +226,6 b''
    * largefiles: add a 'debuglfput' command to put largefile into the store
    * largefiles: add support for 'largefiles://' url scheme
    * largefiles: allow to run 'debugupgraderepo' on repo with largefiles
-   * largefiles: convert EOL of hgrc before appending to bytes IO
    * largefiles: explicitly set the source and sink types to 'hg' for lfconvert
    * largefiles: modernize how capabilities are added to the wire protocol
    * largefiles: pay attention to dropped standin files when updating largefiles
@@ -208,6 +238,8 b''
    * rebase: don't run IMM if running rebase in a transaction
    * rebase: don't take out a dirstate guard for in-memory rebase
    * rebase: drop --style option
+   * rebase: enable multidest by default
+   * rebase: exclude descendants of obsoletes w/o a successor in dest (Bts:issue5300)
    * rebase: fix for hgsubversion
    * rebase: pass the wctx object (IMM or on-disk) to merge.update
    * rebase: pass wctx to rebasenode()
@@ -224,21 +256,14 b''
   === unsorted ===
    * archive: add support to specify hidden revs if directaccess config is set
    * atomicupdate: add an experimental option to use atomictemp when updating
+   * bundle2: don't use seekable bundle2 parts by default (Bts:issue5691)
    * bundle: allow bundlerepo to support alternative manifest implementations
    * changelog: introduce a 'tiprev' method
    * changelog: use 'tiprev()' in 'tip()'
    * completion: add support for new "amend" command
+   * crecord: fix revert -ir '.^' crash caused by 3649c3f2cd
    * debugssl: convert port number to int (Bts:issue5757)
    * diff: disable diff.noprefix option for diffstat (Bts:issue5759)
-   * dispatch: abort if early boolean options can't be parsed
-   * dispatch: add HGPLAIN=+strictflags to restrict early parsing of global options
-   * dispatch: add option to not strip command args parsed by _earlygetopt()
-   * dispatch: alias --repo to --repository while parsing early options
-   * dispatch: convert non-list option parsed by _earlygetopt() to string
-   * dispatch: fix early parsing of short option with value like -R=foo
-   * dispatch: handle IOError when writing to stderr
-   * dispatch: stop parsing of early boolean option at "--"
-   * dispatch: verify result of early command parsing
    * evolution: make reporting of new unstable changesets optional
    * extdata: abort if external command exits with non-zero status (BC)
    * fancyopts: add early-options parser compatible with getopt()
@@ -247,7 +272,6 b''
    * httppeer: add support for tracing all http request made by the peer
    * identify: document -r. explicitly how to disable wdir scanning (Bts:issue5622)
    * lfs: register config options
-   * localrepo: specify optional callback parameter to pathauditor as a keyword
    * match: do not weirdly include explicit files excluded by -X option
    * memfilectx: make changectx argument mandatory in constructor (API)
    * morestatus: don't crash with different drive letters for repo.root and CWD
@@ -256,36 +280,38 b''
    * rewriteutil: use precheck() in uncommit and amend commands
    * scmutil: don't try to delete origbackup symlinks to directories (Bts:issue5731)
    * sshpeer: add support for request tracing
-   * streamclone: add support for bundle2 based stream clone
-   * streamclone: add support for cloning non append-only file
-   * streamclone: also stream caches to the client
-   * streamclone: define first iteration of version 2 of stream format
-   * streamclone: move wire protocol status code from wireproto command
-   * streamclone: rework canperformstreamclone
-   * streamclone: tests phase exchange during stream clone
-   * streamclone: use readexactly when reading stream v2
    * subrepo: add config option to reject any subrepo operations (SEC)
    * subrepo: disable git and svn subrepos by default (BC) (SEC)
    * subrepo: extend config option to disable subrepos by type (SEC)
    * subrepo: handle 'C:' style paths on the command line (Bts:issue5770)
    * subrepo: use per-type config options to enable subrepos
    * svnsubrepo: check if subrepo is missing when checking dirty state (Bts:issue5657)
+   * test-bookmarks-pushpull: stabilize for Windows
+   * test-run-tests: stabilize the test (Bts:issue5735)
    * tr-summary: keep a weakref to the unfiltered repository
    * unamend: fix command summary line
    * uncommit: unify functions _uncommitdirstate and _unamenddirstate to one
+   * update: fix crash on bare update when directaccess is enabled
    * update: support updating to hidden cset if directaccess config is set
   
-  === BC ===
+  === Behavior Changes ===
   
    * extdata: abort if external command exits with non-zero status (BC)
    * graphlog: add another graph node type, unstable, using character "*" (BC)
    * hgweb: drop support of browsers that don't understand <canvas> (BC)
    * hgweb: only include graph-related data in jsdata variable on /graph pages (BC)
    * hgweb: stop adding strings to innerHTML of #graphnodes and #nodebgs (BC)
+   * log: follow file history across copies even with -rREV (BC) (Bts:issue4959)
+   * log: rewrite --follow-first -rREV like --follow for consistency (BC)
    * remove: print message for each file in verbose mode only while using '-A' (BC)
    * subrepo: disable git and svn subrepos by default (BC) (SEC)
    * transaction: register summary callbacks only at start of transaction (BC)
   
-  === API Changes ===
+  === Internal API Changes ===
   
+   * dirstate: add explicit methods for querying directories (API)
+   * exchange: return bundle info from getbundlechunks() (API)
    * memfilectx: make changectx argument mandatory in constructor (API)
+   * templater: keep default resources per template engine (API)
+   * vfs: drop text mode flag (API)
+   * wireproto: drop support for reader interface from streamres (API)