check-code: support multiline matches like try/except/finally

- match one pattern at a time against entire file
- find line containing match
- sort matches by line number

contrib/check-code.py

@@ -1,386 +1,410 b''
 #!/usr/bin/env python
 #
 # check-code - a style and portability checker for Mercurial
 #
 # Copyright 2010 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import re, glob, os, sys
 import keyword
 import optparse
 
 def repquote(m):
     t = re.sub(r"\w", "x", m.group('text'))
-    t = re.sub(r"[^\sx]", "o", t)
+    t = re.sub(r"[^\s\nx]", "o", t)
     return m.group('quote') + t + m.group('quote')
 
 def reppython(m):
     comment = m.group('comment')
     if comment:
         return "#" * len(comment)
     return repquote(m)
 
 def repcomment(m):
     return m.group(1) + "#" * len(m.group(2))
 
 def repccomment(m):
     t = re.sub(r"((?<=\n) )|\S", "x", m.group(2))
     return m.group(1) + t + "*/"
 
 def repcallspaces(m):
     t = re.sub(r"\n\s+", "\n", m.group(2))
     return m.group(1) + t
 
 def repinclude(m):
     return m.group(1) + "<foo>"
 
 def rephere(m):
     t = re.sub(r"\S", "x", m.group(2))
     return m.group(1) + t
 
 
 testpats = [
   [
     (r'(pushd|popd)', "don't use 'pushd' or 'popd', use 'cd'"),
-    (r'\W\$?\(\([^\)]*\)\)', "don't use (()) or $(()), use 'expr'"),
+    (r'\W\$?\(\([^\)\n]*\)\)', "don't use (()) or $(()), use 'expr'"),
     (r'^function', "don't use 'function', use old style"),
     (r'grep.*-q', "don't use 'grep -q', redirect to /dev/null"),
     (r'echo.*\\n', "don't use 'echo \\n', use printf"),
     (r'echo -n', "don't use 'echo -n', use printf"),
     (r'^diff.*-\w*N', "don't use 'diff -N'"),
-    (r'(^| )wc[^|]*$', "filter wc output"),
+    (r'(^| )wc[^|\n]*$', "filter wc output"),
     (r'head -c', "don't use 'head -c', use 'dd'"),
     (r'ls.*-\w*R', "don't use 'ls -R', use 'find'"),
     (r'printf.*\\\d\d\d', "don't use 'printf \NNN', use Python"),
     (r'printf.*\\x', "don't use printf \\x, use Python"),
     (r'\$\(.*\)', "don't use $(expr), use `expr`"),
     (r'rm -rf \*', "don't use naked rm -rf, target a directory"),
-    (r'(^|\|\s*)grep (-\w\s+)*[^|]*[(|]\w',
+    (r'(^|\|\s*)grep (-\w\s+)*[^|\n]*[(|]\w',
      "use egrep for extended grep syntax"),
     (r'/bin/', "don't use explicit paths for tools"),
     (r'\$PWD', "don't use $PWD, use `pwd`"),
     (r'[^\n]\Z', "no trailing newline"),
     (r'export.*=', "don't export and assign at once"),
-    ('^([^"\']|("[^"]*")|(\'[^\']*\'))*\\^', "^ must be quoted"),
+    ('^([^"\'\n]|("[^"\n]*")|(\'[^\'\n]*\'))*\\^', "^ must be quoted"),
     (r'^source\b', "don't use 'source', use '.'"),
     (r'touch -d', "don't use 'touch -d', use 'touch -t' instead"),
-    (r'ls\s+[^|-]+\s+-', "options to 'ls' must come before filenames"),
-    (r'[^>]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"),
+    (r'ls\s+[^|\n-]+\s+-', "options to 'ls' must come before filenames"),
+    (r'[^>\n]>\s*\$HGRCPATH', "don't overwrite $HGRCPATH, append to it"),
     (r'stop\(\)', "don't use 'stop' as a shell function name"),
   ],
   # warnings
   []
 ]
 
 testfilters = [
     (r"( *)(#([^\n]*\S)?)", repcomment),
     (r"<<(\S+)((.|\n)*?\n\1)", rephere),
 ]
 
 uprefix = r"^  \$ "
 uprefixc = r"^  > "
 utestpats = [
   [
-    (r'^(\S|  $ ).*(\S\s+|^\s+)\n', "trailing whitespace on non-output"),
+    (r'^(\S|  $ ).*(\S[ \t]+|^[ \t]+)\n', "trailing whitespace on non-output"),
     (uprefix + r'.*\|\s*sed', "use regex test output patterns instead of sed"),
     (uprefix + r'(true|exit 0)', "explicit zero exit unnecessary"),
     (uprefix + r'.*\$\?', "explicit exit code checks unnecessary"),
     (uprefix + r'.*\|\| echo.*(fail|error)',
      "explicit exit code checks unnecessary"),
     (uprefix + r'set -e', "don't use set -e"),
     (uprefixc + r'( *)\t', "don't use tabs to indent"),
   ],
   # warnings
   []
 ]
 
 for i in [0, 1]:
     for p, m in testpats[i]:
         if p.startswith('^'):
             p = uprefix + p[1:]
         else:
             p = uprefix + p
         utestpats[i].append((p, m))
 
 utestfilters = [
     (r"( *)(#([^\n]*\S)?)", repcomment),
 ]
 
 pypats = [
   [
     (r'^\s*def\s*\w+\s*\(.*,\s*\(',
      "tuple parameter unpacking not available in Python 3+"),
     (r'lambda\s*\(.*,.*\)',
      "tuple parameter unpacking not available in Python 3+"),
     (r'(?<!def)\s+(cmp)\(', "cmp is not available in Python 3+"),
     (r'\breduce\s*\(.*', "reduce is not available in Python 3+"),
     (r'\.has_key\b', "dict.has_key is not available in Python 3+"),
     (r'^\s*\t', "don't use tabs"),
     (r'\S;\s*\n', "semicolon"),
     (r'\w,\w', "missing whitespace after ,"),
     (r'\w[+/*\-<>]\w', "missing whitespace in expression"),
-    (r'^\s+\w+=\w+[^,)]$', "missing whitespace in assignment"),
+    (r'^\s+\w+=\w+[^,)\n]$', "missing whitespace in assignment"),
+    (r'(?m)(\s+)try:\n((?:\n|\1\s.*\n)+?)\1except.*?:\n'
+     r'((?:\n|\1\s.*\n)+?)\1finally:', 'no try/except/finally in Py2.4'),
     (r'.{85}', "line too long"),
     (r'[^\n]\Z', "no trailing newline"),
-    (r'(\S\s+|^\s+)\n', "trailing whitespace"),
-#    (r'^\s+[^_ ][^_. ]+_[^_]+\s*=', "don't use underbars in identifiers"),
+    (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
+#    (r'^\s+[^_ \n][^_. \n]+_[^_\n]+\s*=', "don't use underbars in identifiers"),
 #    (r'\w*[a-z][A-Z]\w*\s*=', "don't use camelcase in identifiers"),
-    (r'^\s*(if|while|def|class|except|try)\s[^[]*:\s*[^\]#\s]+',
+    (r'^\s*(if|while|def|class|except|try)\s[^[\n]*:\s*[^\\n]#\s]+',
      "linebreak after :"),
-    (r'class\s[^( ]+:', "old-style class, use class foo(object)"),
-    (r'class\s[^( ]+\(\):',
+    (r'class\s[^( \n]+:', "old-style class, use class foo(object)"),
+    (r'class\s[^( \n]+\(\):',
      "class foo() not available in Python 2.4, use class foo(object)"),
     (r'\b(%s)\(' % '|'.join(keyword.kwlist),
      "Python keyword is not a function"),
     (r',]', "unneeded trailing ',' in list"),
 #    (r'class\s[A-Z][^\(]*\((?!Exception)',
 #     "don't capitalize non-exception classes"),
 #    (r'in range\(', "use xrange"),
 #    (r'^\s*print\s+', "avoid using print in core and extensions"),
     (r'[\x80-\xff]', "non-ASCII character literal"),
     (r'("\')\.format\(', "str.format() not available in Python 2.4"),
     (r'^\s*with\s+', "with not available in Python 2.4"),
     (r'\.isdisjoint\(', "set.isdisjoint not available in Python 2.4"),
     (r'^\s*except.* as .*:', "except as not available in Python 2.4"),
     (r'^\s*os\.path\.relpath', "relpath not available in Python 2.4"),
     (r'(?<!def)\s+(any|all|format)\(',
      "any/all/format not available in Python 2.4"),
     (r'(?<!def)\s+(callable)\(',
      "callable not available in Python 3, use getattr(f, '__call__', None)"),
     (r'if\s.*\selse', "if ... else form not available in Python 2.4"),
     (r'^\s*(%s)\s\s' % '|'.join(keyword.kwlist),
      "gratuitous whitespace after Python keyword"),
-    (r'([\(\[]\s\S)|(\S\s[\)\]])', "gratuitous whitespace in () or []"),
+    (r'([\(\[][ \t]\S)|(\S[ \t][\)\]])', "gratuitous whitespace in () or []"),
 #    (r'\s\s=', "gratuitous whitespace before ="),
     (r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=)\S',
      "missing whitespace around operator"),
     (r'[^>< ](\+=|-=|!=|<>|<=|>=|<<=|>>=)\s',
      "missing whitespace around operator"),
     (r'\s(\+=|-=|!=|<>|<=|>=|<<=|>>=)\S',
      "missing whitespace around operator"),
     (r'[^+=*/!<>&| -](\s=|=\s)[^= ]',
      "wrong whitespace around ="),
     (r'raise Exception', "don't raise generic exceptions"),
     (r' is\s+(not\s+)?["\'0-9-]', "object comparison with literal"),
     (r' [=!]=\s+(True|False|None)',
      "comparison with singleton, use 'is' or 'is not' instead"),
     (r'^\s*(while|if) [01]:',
      "use True/False for constant Boolean expression"),
     (r'(?<!def)\s+hasattr',
      'hasattr(foo, bar) is broken, use util.safehasattr(foo, bar) instead'),
     (r'opener\([^)]*\).read\(',
      "use opener.read() instead"),
     (r'opener\([^)]*\).write\(',
      "use opener.write() instead"),
     (r'[\s\(](open|file)\([^)]*\)\.read\(',
      "use util.readfile() instead"),
     (r'[\s\(](open|file)\([^)]*\)\.write\(',
      "use util.readfile() instead"),
     (r'^[\s\(]*(open(er)?|file)\([^)]*\)',
      "always assign an opened file to a variable, and close it afterwards"),
     (r'[\s\(](open|file)\([^)]*\)\.',
      "always assign an opened file to a variable, and close it afterwards"),
     (r'(?i)descendent', "the proper spelling is descendAnt"),
     (r'\.debug\(\_', "don't mark debug messages for translation"),
   ],
   # warnings
   [
     (r'.{81}', "warning: line over 80 characters"),
     (r'^\s*except:$', "warning: naked except clause"),
     (r'ui\.(status|progress|write|note|warn)\([\'\"]x',
      "warning: unwrapped ui message"),
   ]
 ]
 
 pyfilters = [
     (r"""(?msx)(?P<comment>\#.*?$)|
          ((?P<quote>('''|\"\"\"|(?<!')'(?!')|(?<!")"(?!")))
           (?P<text>(([^\\]|\\.)*?))
           (?P=quote))""", reppython),
 ]
 
 cpats = [
   [
     (r'//', "don't use //-style comments"),
     (r'^  ', "don't use spaces to indent"),
     (r'\S\t', "don't use tabs except for indent"),
-    (r'(\S\s+|^\s+)\n', "trailing whitespace"),
+    (r'(\S[ \t]+|^[ \t]+)\n', "trailing whitespace"),
     (r'.{85}', "line too long"),
     (r'(while|if|do|for)\(', "use space after while/if/do/for"),
     (r'return\(', "return is not a function"),
     (r' ;', "no space before ;"),
     (r'\w+\* \w+', "use int *foo, not int* foo"),
     (r'\([^\)]+\) \w+', "use (int)foo, not (int) foo"),
     (r'\S+ (\+\+|--)', "use foo++, not foo ++"),
     (r'\w,\w', "missing whitespace after ,"),
     (r'^[^#]\w[+/*]\w', "missing whitespace in expression"),
     (r'^#\s+\w', "use #foo, not # foo"),
     (r'[^\n]\Z', "no trailing newline"),
     (r'^\s*#import\b', "use only #include in standard C code"),
   ],
   # warnings
   []
 ]
 
 cfilters = [
     (r'(/\*)(((\*(?!/))|[^*])*)\*/', repccomment),
     (r'''(?P<quote>(?<!")")(?P<text>([^"]|\\")+)"(?!")''', repquote),
     (r'''(#\s*include\s+<)([^>]+)>''', repinclude),
     (r'(\()([^)]+\))', repcallspaces),
 ]
 
 inutilpats = [
   [
     (r'\bui\.', "don't use ui in util"),
   ],
   # warnings
   []
 ]
 
 inrevlogpats = [
   [
     (r'\brepo\.', "don't use repo in revlog"),
   ],
   # warnings
   []
 ]
 
 checks = [
     ('python', r'.*\.(py|cgi)$', pyfilters, pypats),
     ('test script', r'(.*/)?test-[^.~]*$', testfilters, testpats),
     ('c', r'.*\.c$', cfilters, cpats),
     ('unified test', r'.*\.t$', utestfilters, utestpats),
     ('layering violation repo in revlog', r'mercurial/revlog\.py', pyfilters,
      inrevlogpats),
     ('layering violation ui in util', r'mercurial/util\.py', pyfilters,
      inutilpats),
 ]
 
 class norepeatlogger(object):
     def __init__(self):
         self._lastseen = None
 
     def log(self, fname, lineno, line, msg, blame):
         """print error related a to given line of a given file.
 
         The faulty line will also be printed but only once in the case
         of multiple errors.
 
         :fname: filename
         :lineno: line number
         :line: actual content of the line
         :msg: error message
         """
         msgid = fname, lineno, line
         if msgid != self._lastseen:
             if blame:
                 print "%s:%d (%s):" % (fname, lineno, blame)
             else:
                 print "%s:%d:" % (fname, lineno)
             print " > %s" % line
             self._lastseen = msgid
         print " " + msg
 
 _defaultlogger = norepeatlogger()
 
 def getblame(f):
     lines = []
     for l in os.popen('hg annotate -un %s' % f):
         start, line = l.split(':', 1)
         user, rev = start.split()
         lines.append((line[1:-1], user, rev))
     return lines
 
 def checkfile(f, logfunc=_defaultlogger.log, maxerr=None, warnings=False,
               blame=False, debug=False):
     """checks style and portability of a given file
 
     :f: filepath
     :logfunc: function used to report error
               logfunc(filename, linenumber, linecontent, errormessage)
     :maxerr: number of error to display before arborting.
              Set to None (default) to report all errors
 
     return True if no error is found, False otherwise.
     """
     blamecache = None
     result = True
     for name, match, filters, pats in checks:
         if debug:
             print name, f
         fc = 0
         if not re.match(match, f):
             if debug:
                 print "Skipping %s for %s it doesn't match %s" % (
                        name, match, f)
             continue
         fp = open(f)
         pre = post = fp.read()
         fp.close()
         if "no-" + "check-code" in pre:
             if debug:
                 print "Skipping %s for %s it has no- and check-code" % (
                        name, f)
             break
         for p, r in filters:
             post = re.sub(p, r, post)
         if warnings:
             pats = pats[0] + pats[1]
         else:
             pats = pats[0]
         # print post # uncomment to show filtered version
-        z = enumerate(zip(pre.splitlines(), post.splitlines(True)))
+
         if debug:
             print "Checking %s for %s" % (name, f)
-        for n, l in z:
-            if "check-code" + "-ignore" in l[0]:
+
+        prelines = None
+        errors = []
+        for p, msg in pats:
+            pos = 0
+            n = 0
+            for m in re.finditer(p, post):
+                if prelines is None:
+                    prelines = pre.splitlines()
+                    postlines = post.splitlines(True)
+
+                start = m.start()
+                while n < len(postlines):
+                    step = len(postlines[n])
+                    if pos + step > start:
+                        break
+                    pos += step
+                    n += 1
+                l = prelines[n]
+
+                if "check-code" + "-ignore" in l:
                 if debug:
                     print "Skipping %s for %s:%s (check-code -ignore)" % (
                            name, f, n)
                 continue
-            for p, msg in pats:
-                if re.search(p, l[1]):
                     bd = ""
                     if blame:
                         bd = 'working directory'
                         if not blamecache:
                             blamecache = getblame(f)
                         if n < len(blamecache):
                             bl, bu, br = blamecache[n]
-                            if bl == l[0]:
+                        if bl == l:
                                 bd = '%s@%s' % (bu, br)
-                    logfunc(f, n + 1, l[0], msg, bd)
+                errors.append((f, n + 1, l, msg, bd))
+                result = False
+
+        errors.sort()
+        for e in errors:
+            logfunc(*e)
                     fc += 1
-                    result = False
             if maxerr is not None and fc >= maxerr:
                 print " (too many errors, giving up)"
                 break
+
     return result
 
 if __name__ == "__main__":
     parser = optparse.OptionParser("%prog [options] [files]")
     parser.add_option("-w", "--warnings", action="store_true",
                       help="include warning-level checks")
     parser.add_option("-p", "--per-file", type="int",
                       help="max warnings per file")
     parser.add_option("-b", "--blame", action="store_true",
                       help="use annotate to generate blame info")
     parser.add_option("", "--debug", action="store_true",
                       help="show debug information")
 
     parser.set_defaults(per_file=15, warnings=False, blame=False, debug=False)
     (options, args) = parser.parse_args()
 
     if len(args) == 0:
         check = glob.glob("*")
     else:
         check = args
 
     for f in check:
         ret = 0
         if not checkfile(f, maxerr=options.per_file, warnings=options.warnings,
                          blame=options.blame, debug=options.debug):
             ret = 1
     sys.exit(ret)

tests/test-check-code.t

@@ -1,114 +1,114 b''
   $ cat > correct.py <<EOF
   > def toto(arg1, arg2):
   >     del arg2
   >     return (5 + 6, 9)
   > EOF
   $ cat > wrong.py <<EOF
   > def toto( arg1, arg2):
   >     del(arg2)
   >     return ( 5+6, 9)
   > EOF
   $ cat > quote.py <<EOF
   > # let's use quote in comments
   > (''' ( 4x5 )
   > but """\\''' and finally''',
   > """let's fool checkpatch""", '1+2',
   > '"""', 42+1, """and
   > ( 4-1 ) """, "( 1+1 )\" and ")
   > a, '\\\\\\\\', "\\\\\\" x-2", "c-1"
   > EOF
   $ cat > non-py24.py <<EOF
   > # Using builtins that does not exist in Python 2.4
   > if any():
   >     x = all()
   >     y = format(x)
   > 
   > # Do not complain about our own definition
   > def any(x):
   >     pass
   > EOF
   $ cat > classstyle.py <<EOF
   > class newstyle_class(object):
   >     pass
   > 
   > class oldstyle_class:
   >     pass
   > 
   > class empty():
   >     pass
   > 
   > no_class = 1:
   >     pass
   > EOF
   $ check_code="$TESTDIR"/../contrib/check-code.py
   $ "$check_code" ./wrong.py ./correct.py ./quote.py ./non-py24.py ./classstyle.py
   ./wrong.py:1:
    > def toto( arg1, arg2):
    gratuitous whitespace in () or []
   ./wrong.py:2:
    >     del(arg2)
    Python keyword is not a function
   ./wrong.py:3:
    >     return ( 5+6, 9)
+   gratuitous whitespace in () or []
    missing whitespace in expression
-   gratuitous whitespace in () or []
   ./quote.py:5:
    > '"""', 42+1, """and
    missing whitespace in expression
   ./non-py24.py:2:
    > if any():
    any/all/format not available in Python 2.4
   ./non-py24.py:3:
    >     x = all()
    any/all/format not available in Python 2.4
   ./non-py24.py:4:
    >     y = format(x)
    any/all/format not available in Python 2.4
   ./classstyle.py:4:
    > class oldstyle_class:
    old-style class, use class foo(object)
   ./classstyle.py:7:
    > class empty():
    class foo() not available in Python 2.4, use class foo(object)
   [1]
 
   $ cat > is-op.py <<EOF
   > # is-operator comparing number or string literal
   > x = None
   > y = x is 'foo'
   > y = x is "foo"
   > y = x is 5346
   > y = x is -6
   > y = x is not 'foo'
   > y = x is not "foo"
   > y = x is not 5346
   > y = x is not -6
   > EOF
 
   $ "$check_code" ./is-op.py
   ./is-op.py:3:
    > y = x is 'foo'
    object comparison with literal
   ./is-op.py:4:
    > y = x is "foo"
    object comparison with literal
   ./is-op.py:5:
    > y = x is 5346
    object comparison with literal
   ./is-op.py:6:
    > y = x is -6
    object comparison with literal
   ./is-op.py:7:
    > y = x is not 'foo'
    object comparison with literal
   ./is-op.py:8:
    > y = x is not "foo"
    object comparison with literal
   ./is-op.py:9:
    > y = x is not 5346
    object comparison with literal
   ./is-op.py:10:
    > y = x is not -6
    object comparison with literal
   [1]