mdiff: carriage return (\r) is also ignorable whitespace

mercurial/mdiff.py

@@ -1,277 +1,277
 # mdiff.py - diff and patch routines for mercurial
 #
 # Copyright 2005, 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from i18n import _
 import bdiff, mpatch, util
 import re, struct
 
 def splitnewlines(text):
     '''like str.splitlines, but only split on newlines.'''
     lines = [l + '\n' for l in text.split('\n')]
     if lines:
         if lines[-1] == '\n':
             lines.pop()
         else:
             lines[-1] = lines[-1][:-1]
     return lines
 
 class diffopts(object):
     '''context is the number of context lines
     text treats all files as text
     showfunc enables diff -p output
     git enables the git extended patch format
     nodates removes dates from diff headers
     ignorews ignores all whitespace changes in the diff
     ignorewsamount ignores changes in the amount of whitespace
     ignoreblanklines ignores changes whose lines are all blank
     upgrade generates git diffs to avoid data loss
     '''
 
     defaults = {
         'context': 3,
         'text': False,
         'showfunc': False,
         'git': False,
         'nodates': False,
         'ignorews': False,
         'ignorewsamount': False,
         'ignoreblanklines': False,
         'upgrade': False,
         }
 
     __slots__ = defaults.keys()
 
     def __init__(self, **opts):
         for k in self.__slots__:
             v = opts.get(k)
             if v is None:
                 v = self.defaults[k]
             setattr(self, k, v)
 
         try:
             self.context = int(self.context)
         except ValueError:
             raise util.Abort(_('diff context lines count must be '
                                'an integer, not %r') % self.context)
 
     def copy(self, **kwargs):
         opts = dict((k, getattr(self, k)) for k in self.defaults)
         opts.update(kwargs)
         return diffopts(**opts)
 
 defaultopts = diffopts()
 
 def wsclean(opts, text, blank=True):
     if opts.ignorews:
-        text = re.sub('[ \t]+', '', text)
+        text = re.sub('[ \t\r]+', '', text)
     elif opts.ignorewsamount:
-        text = re.sub('[ \t]+', ' ', text)
-        text = re.sub('[ \t]+\n', '\n', text)
+        text = re.sub('[ \t\r]+', ' ', text)
+        text = text.replace(' \n', '\n')
     if blank and opts.ignoreblanklines:
         text = re.sub('\n+', '', text)
     return text
 
 def diffline(revs, a, b, opts):
     parts = ['diff']
     if opts.git:
         parts.append('--git')
     if revs and not opts.git:
         parts.append(' '.join(["-r %s" % rev for rev in revs]))
     if opts.git:
         parts.append('a/%s' % a)
         parts.append('b/%s' % b)
     else:
         parts.append(a)
     return ' '.join(parts) + '\n'
 
 def unidiff(a, ad, b, bd, fn1, fn2, r=None, opts=defaultopts):
     def datetag(date, addtab=True):
         if not opts.git and not opts.nodates:
             return '\t%s\n' % date
         if addtab and ' ' in fn1:
             return '\t\n'
         return '\n'
 
     if not a and not b:
         return ""
     epoch = util.datestr((0, 0))
 
     if not opts.text and (util.binary(a) or util.binary(b)):
         if a and b and len(a) == len(b) and a == b:
             return ""
         l = ['Binary file %s has changed\n' % fn1]
     elif not a:
         b = splitnewlines(b)
         if a is None:
             l1 = '--- /dev/null%s' % datetag(epoch, False)
         else:
             l1 = "--- %s%s" % ("a/" + fn1, datetag(ad))
         l2 = "+++ %s%s" % ("b/" + fn2, datetag(bd))
         l3 = "@@ -0,0 +1,%d @@\n" % len(b)
         l = [l1, l2, l3] + ["+" + e for e in b]
     elif not b:
         a = splitnewlines(a)
         l1 = "--- %s%s" % ("a/" + fn1, datetag(ad))
         if b is None:
             l2 = '+++ /dev/null%s' % datetag(epoch, False)
         else:
             l2 = "+++ %s%s" % ("b/" + fn2, datetag(bd))
         l3 = "@@ -1,%d +0,0 @@\n" % len(a)
         l = [l1, l2, l3] + ["-" + e for e in a]
     else:
         al = splitnewlines(a)
         bl = splitnewlines(b)
         l = list(_unidiff(a, b, al, bl, opts=opts))
         if not l:
             return ""
 
         l.insert(0, "--- a/%s%s" % (fn1, datetag(ad)))
         l.insert(1, "+++ b/%s%s" % (fn2, datetag(bd)))
 
     for ln in xrange(len(l)):
         if l[ln][-1] != '\n':
             l[ln] += "\n\ No newline at end of file\n"
 
     if r:
         l.insert(0, diffline(r, fn1, fn2, opts))
 
     return "".join(l)
 
 # creates a headerless unified diff
 # t1 and t2 are the text to be diffed
 # l1 and l2 are the text broken up into lines
 def _unidiff(t1, t2, l1, l2, opts=defaultopts):
     def contextend(l, len):
         ret = l + opts.context
         if ret > len:
             ret = len
         return ret
 
     def contextstart(l):
         ret = l - opts.context
         if ret < 0:
             return 0
         return ret
 
     def yieldhunk(hunk):
         (astart, a2, bstart, b2, delta) = hunk
         aend = contextend(a2, len(l1))
         alen = aend - astart
         blen = b2 - bstart + aend - a2
 
         func = ""
         if opts.showfunc:
             # walk backwards from the start of the context
             # to find a line starting with an alphanumeric char.
             for x in xrange(astart - 1, -1, -1):
                 t = l1[x].rstrip()
                 if funcre.match(t):
                     func = ' ' + t[:40]
                     break
 
         yield "@@ -%d,%d +%d,%d @@%s\n" % (astart + 1, alen,
                                            bstart + 1, blen, func)
         for x in delta:
             yield x
         for x in xrange(a2, aend):
             yield ' ' + l1[x]
 
     if opts.showfunc:
         funcre = re.compile('\w')
 
     # bdiff.blocks gives us the matching sequences in the files.  The loop
     # below finds the spaces between those matching sequences and translates
     # them into diff output.
     #
     if opts.ignorews or opts.ignorewsamount:
         t1 = wsclean(opts, t1, False)
         t2 = wsclean(opts, t2, False)
 
     diff = bdiff.blocks(t1, t2)
     hunk = None
     for i, s1 in enumerate(diff):
         # The first match is special.
         # we've either found a match starting at line 0 or a match later
         # in the file.  If it starts later, old and new below will both be
         # empty and we'll continue to the next match.
         if i > 0:
             s = diff[i - 1]
         else:
             s = [0, 0, 0, 0]
         delta = []
         a1 = s[1]
         a2 = s1[0]
         b1 = s[3]
         b2 = s1[2]
 
         old = l1[a1:a2]
         new = l2[b1:b2]
 
         # bdiff sometimes gives huge matches past eof, this check eats them,
         # and deals with the special first match case described above
         if not old and not new:
             continue
 
         if opts.ignoreblanklines:
             if wsclean(opts, "".join(old)) == wsclean(opts, "".join(new)):
                 continue
 
         astart = contextstart(a1)
         bstart = contextstart(b1)
         prev = None
         if hunk:
             # join with the previous hunk if it falls inside the context
             if astart < hunk[1] + opts.context + 1:
                 prev = hunk
                 astart = hunk[1]
                 bstart = hunk[3]
             else:
                 for x in yieldhunk(hunk):
                     yield x
         if prev:
             # we've joined the previous hunk, record the new ending points.
             hunk[1] = a2
             hunk[3] = b2
             delta = hunk[4]
         else:
             # create a new hunk
             hunk = [astart, a2, bstart, b2, delta]
 
         delta[len(delta):] = [' ' + x for x in l1[astart:a1]]
         delta[len(delta):] = ['-' + x for x in old]
         delta[len(delta):] = ['+' + x for x in new]
 
     if hunk:
         for x in yieldhunk(hunk):
             yield x
 
 def patchtext(bin):
     pos = 0
     t = []
     while pos < len(bin):
         p1, p2, l = struct.unpack(">lll", bin[pos:pos + 12])
         pos += 12
         t.append(bin[pos:pos + l])
         pos += l
     return "".join(t)
 
 def patch(a, bin):
     if len(a) == 0:
         # skip over trivial delta header
         return buffer(bin, 12)
     return mpatch.patches(a, [bin])
 
 # similar to difflib.SequenceMatcher.get_matching_blocks
 def get_matching_blocks(a, b):
     return [(d[0], d[2], d[1] - d[0]) for d in bdiff.blocks(a, b)]
 
 def trivialdiffheader(length):
     return struct.pack(">lll", 0, 0, length)
 
 patches = mpatch.patches
 patchedsize = mpatch.patchedsize
 textdiff = bdiff.bdiff

tests/test-diff-ignore-whitespace.t

@@ -1,392 +1,446
 GNU diff is the reference for all of these results.
 
 Prepare tests:
 
   $ echo '[alias]' >> $HGRCPATH
   $ echo 'ndiff = diff --nodates' >> $HGRCPATH
 
   $ hg init
   $ printf 'hello world\ngoodbye world\n' >foo
   $ hg ci -Amfoo -ufoo
   adding foo
 
 
 Test added blank lines:
 
   $ printf '\nhello world\n\ngoodbye world\n\n' >foo
 
 >>> two diffs showing three added lines <<<
 
   $ hg ndiff
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,5 @@
   +
    hello world
   +
    goodbye world
   +
   $ hg ndiff -b
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,5 @@
   +
    hello world
   +
    goodbye world
   +
 
 >>> no diffs <<<
 
   $ hg ndiff -B
   $ hg ndiff -Bb
 
 
 Test added horizontal space first on a line():
 
   $ printf '\t hello world\ngoodbye world\n' >foo
 
 >>> four diffs showing added space first on the first line <<<
 
   $ hg ndiff
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
   -hello world
   +	 hello world
    goodbye world
 
   $ hg ndiff -b
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
   -hello world
   +	 hello world
    goodbye world
 
   $ hg ndiff -B
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
   -hello world
   +	 hello world
    goodbye world
 
   $ hg ndiff -Bb
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
   -hello world
   +	 hello world
    goodbye world
 
 
 Test added horizontal space last on a line:
 
   $ printf 'hello world\t \ngoodbye world\n' >foo
 
 >>> two diffs showing space appended to the first line <<<
 
   $ hg ndiff
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
   -hello world
   +hello world	 
    goodbye world
 
   $ hg ndiff -B
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
   -hello world
   +hello world	 
    goodbye world
 
 >>> no diffs <<<
 
   $ hg ndiff -b
   $ hg ndiff -Bb
 
 
 Test added horizontal space in the middle of a word:
 
   $ printf 'hello world\ngood bye world\n' >foo
 
 >>> four diffs showing space inserted into "goodbye" <<<
 
   $ hg ndiff
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
    hello world
   -goodbye world
   +good bye world
 
   $ hg ndiff -B
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
    hello world
   -goodbye world
   +good bye world
 
   $ hg ndiff -b
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
    hello world
   -goodbye world
   +good bye world
 
   $ hg ndiff -Bb
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
    hello world
   -goodbye world
   +good bye world
 
 
 Test increased horizontal whitespace amount:
 
   $ printf 'hello world\ngoodbye\t\t  \tworld\n' >foo
 
 >>> two diffs showing changed whitespace amount in the last line <<<
 
   $ hg ndiff
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
    hello world
   -goodbye world
   +goodbye		  	world
 
   $ hg ndiff -B
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
    hello world
   -goodbye world
   +goodbye		  	world
 
 >>> no diffs <<<
 
   $ hg ndiff -b
   $ hg ndiff -Bb
 
 
 Test added blank line with horizontal whitespace:
 
   $ printf 'hello world\n \t\ngoodbye world\n' >foo
 
 >>> three diffs showing added blank line with horizontal space <<<
 
   $ hg ndiff
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,3 @@
    hello world
   + 	
    goodbye world
 
   $ hg ndiff -B
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,3 @@
    hello world
   + 	
    goodbye world
 
   $ hg ndiff -b
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,3 @@
    hello world
   + 	
    goodbye world
 
 >>> no diffs <<<
 
   $ hg ndiff -Bb
 
 
 Test added blank line with other whitespace:
 
   $ printf 'hello  world\n \t\ngoodbye world \n' >foo
 
 >>> three diffs showing added blank line with other space <<<
 
   $ hg ndiff
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,3 @@
   -hello world
   -goodbye world
   +hello  world
   + 	
   +goodbye world 
 
   $ hg ndiff -B
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,3 @@
   -hello world
   -goodbye world
   +hello  world
   + 	
   +goodbye world 
 
   $ hg ndiff -b
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,3 @@
    hello world
   + 	
    goodbye world
 
 >>> no diffs <<<
 
   $ hg ndiff -Bb
 
 
 Test whitespace changes:
 
   $ printf 'helloworld\ngoodbye\tworld \n' >foo
 
 >>> four diffs showing changed whitespace <<<
 
   $ hg ndiff
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
   -hello world
   -goodbye world
   +helloworld
   +goodbye	world 
 
   $ hg ndiff -B
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
   -hello world
   -goodbye world
   +helloworld
   +goodbye	world 
 
   $ hg ndiff -b
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
   -hello world
   +helloworld
    goodbye world
 
   $ hg ndiff -Bb
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,2 @@
   -hello world
   +helloworld
    goodbye world
 
 >>> no diffs <<<
 
   $ hg ndiff -w
 
 
 Test whitespace changes and blank lines:
 
   $ printf 'helloworld\n\n\n\ngoodbye\tworld \n' >foo
 
 >>> five diffs showing changed whitespace <<<
 
   $ hg ndiff
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,5 @@
   -hello world
   -goodbye world
   +helloworld
   +
   +
   +
   +goodbye	world 
 
   $ hg ndiff -B
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,5 @@
   -hello world
   -goodbye world
   +helloworld
   +
   +
   +
   +goodbye	world 
 
   $ hg ndiff -b
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,5 @@
   -hello world
   +helloworld
   +
   +
   +
    goodbye world
 
   $ hg ndiff -Bb
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,5 @@
   -hello world
   +helloworld
   +
   +
   +
    goodbye world
 
   $ hg ndiff -w
   diff -r 540c40a65b78 foo
   --- a/foo
   +++ b/foo
   @@ -1,2 +1,5 @@
    hello world
   +
   +
   +
    goodbye world
 
 >>> no diffs <<<
 
   $ hg ndiff -wB
 
+
+Test \r (carriage return) as used in "DOS" line endings:
+
+  $ printf 'hello world\r\n\r\ngoodbye\rworld\n' >foo
+
+  $ hg ndiff
+  diff -r 540c40a65b78 foo
+  --- a/foo
+  +++ b/foo
+  @@ -1,2 +1,3 @@
+  -hello world
+  -goodbye world
+  +hello world
+  +
+  +goodbye
+world
+world
+
+No completely blank lines to ignore:
+
+  $ hg ndiff --ignore-blank-lines
+  diff -r 540c40a65b78 foo
+  --- a/foo
+  +++ b/foo
+  @@ -1,2 +1,3 @@
+  -hello world
+  -goodbye world
+  +hello world
+  +
+  +goodbye
+world
+world
+
+Only new line noticed:
+
+  $ hg ndiff --ignore-space-change
+  diff -r 540c40a65b78 foo
+  --- a/foo
+  +++ b/foo
+  @@ -1,2 +1,3 @@
+   hello world
+  +
+   goodbye world
+
+  $ hg ndiff --ignore-all-space
+  diff -r 540c40a65b78 foo
+  --- a/foo
+  +++ b/foo
+  @@ -1,2 +1,3 @@
+   hello world
+  +
+   goodbye world
+
+New line not noticed when space change ignored: