# HG changeset patch
# User Raphaël Gomès <rgomes@octobus.net>
# Date 2019-08-02 07:55:32
# Node ID b9a200477edfe9ccd2327e0be4c98d31cc56b6dc
# Parent  e9592e113c31dd4680523455d04ab5a59ed3f99e

byteify-strings: add support for ignore comments

Our simple token analysis is sometimes not clever enough, we need to be able
to turn off our script for parts of the code.

This change introduces three special comments:

    - `#no-py3-transform` to tell `byteify-strings` ignore the next line
    - `#py3-transform: off` to ignore everything until the end of the file
    - `#py3-transform: on` to stop ignoring

The last two can be particularly useful within Python 2/3 compatibility files.

diff --git a/contrib/byteify-strings.py b/contrib/byteify-strings.py
--- a/contrib/byteify-strings.py
+++ b/contrib/byteify-strings.py
@@ -95,6 +95,8 @@ def replacetokens(tokens, opts):
     coldelta = 0  # column increment for new opening parens
     coloffset = -1  # column offset for the current line (-1: TBD)
     parens = [(0, 0, 0)]  # stack of (line, end-column, column-offset)
+    ignorenextline = False  # don't transform the next line
+    insideignoreblock = False # don't transform until turned off
     for i, t in enumerate(tokens):
         # Compute the column offset for the current line, such that
         # the current line will be aligned to the last opening paren
@@ -113,6 +115,21 @@ def replacetokens(tokens, opts):
             yield adjusttokenpos(t, coloffset)
             coldelta = 0
             coloffset = -1
+            if not insideignoreblock:
+                ignorenextline = (
+                    tokens[i - 1].type == token.COMMENT
+                    and tokens[i - 1].string == "#no-py3-transform"
+                )
+            continue
+
+        if t.type == token.COMMENT:
+            if t.string == "#py3-transform: off":
+                insideignoreblock = True
+            if t.string == "#py3-transform: on":
+                insideignoreblock = False
+
+        if ignorenextline or insideignoreblock:
+            yield adjusttokenpos(t, coloffset)
             continue
 
         # Remember the last paren position.