upstream/mercurial-mirror Files · mercurial/cffi/bdiff.py

manifest: avoid corruption by dropping removed files with pure (issue5801)...

manifest: avoid corruption by dropping removed files with pure (issue5801) Previously, removed files would simply be marked by overwriting the first byte with NUL and dropping their entry in `self.position`. But no effort was made to ignore them when compacting the dictionary into text form. This allowed them to slip into the manifest revision, since the code seems to be trying to minimize the string operations by copying as large a chunk as possible. As part of this, compact() walks the existing text based on entries in the `positions` list, and consumed everything up to the next position entry. This typically resulted in a ValueError complaining about unsorted manifest entries. Sometimes it seems that files do get dropped in large repos- it seems to correspond to there being a new entry that would take the same slot. A much more trivial problem is that if the only changes were removals, `_compact()` didn't even run because `__delitem__` doesn't add anything to `self.extradata`. Now there's an explicit variable to flag this, both to allow `_compact()` to run, and to avoid searching the manifest in cases where there are no removals. In practice, this behavior was mostly obscured by the check in fastdelta() which takes a different path that explicitly drops removed files if there are fewer than 1000 changes. However, timeless has a repo where after rebasing tens of commits, a totally different path[1] is taken that bypasses the change count check and hits this problem. [1] https://www.mercurial-scm.org/repo/hg/file/2338bdea4474/mercurial/manifest.py#l1511

Kevin Bullock - - Load All Authors

File last commit:

r33573:857876eb 4.3-rc stable


                r42569:0546ead3

stable

Download file

             bdiff.py
        
                    76 lines
            
             | 2.1 KiB
            
                | text/x-python
            
             |
                PythonLexer
            
             / mercurial / cffi / bdiff.py
          
                    History
                
                 |
                  Source
                 | Raw
                 |Copy content
                 |Copy permalink

        Yuya Nishihara
    
cffi: split modules from pure...

              r32512
            
      # bdiff.py - CFFI implementation of bdiff.c

      #

      # Copyright 2016 Maciej Fijalkowski <fijall@gmail.com>

      #

      # This software may be used and distributed according to the terms of the

      # GNU General Public License version 2 or any later version.

      from __future__ import absolute_import

      import struct

      from ..pure.bdiff import *

      from . import _bdiff

      ffi = _bdiff.ffi

      lib = _bdiff.lib

        Yuya Nishihara
    
cffi: remove superfluous "if True" blocks

              r32513
            
      def blocks(sa, sb):

          a = ffi.new("struct bdiff_line**")

          b = ffi.new("struct bdiff_line**")

          ac = ffi.new("char[]", str(sa))

          bc = ffi.new("char[]", str(sb))

          l = ffi.new("struct bdiff_hunk*")

          try:

              an = lib.bdiff_splitlines(ac, len(sa), a)

              bn = lib.bdiff_splitlines(bc, len(sb), b)

              if not a[0] or not b[0]:

                  raise MemoryError

              count = lib.bdiff_diff(a[0], an, b[0], bn, l)

              if count < 0:

                  raise MemoryError

              rl = [None] * count

              h = l.next

              i = 0

              while h:

                  rl[i] = (h.a1, h.a2, h.b1, h.b2)

                  h = h.next

                  i += 1

          finally:

              lib.free(a[0])

              lib.free(b[0])

              lib.bdiff_freehunks(l.next)

          return rl

        Yuya Nishihara
    
cffi: split modules from pure...

              r32512
            
        Yuya Nishihara
    
cffi: remove superfluous "if True" blocks

              r32513
            
      def bdiff(sa, sb):

          a = ffi.new("struct bdiff_line**")

          b = ffi.new("struct bdiff_line**")

          ac = ffi.new("char[]", str(sa))

          bc = ffi.new("char[]", str(sb))

          l = ffi.new("struct bdiff_hunk*")

          try:

              an = lib.bdiff_splitlines(ac, len(sa), a)

              bn = lib.bdiff_splitlines(bc, len(sb), b)

              if not a[0] or not b[0]:

                  raise MemoryError

              count = lib.bdiff_diff(a[0], an, b[0], bn, l)

              if count < 0:

                  raise MemoryError

              rl = []

              h = l.next

              la = lb = 0

              while h:

                  if h.a1 != la or h.b1 != lb:

                      lgt = (b[0] + h.b1).l - (b[0] + lb).l

                      rl.append(struct.pack(">lll", (a[0] + la).l - a[0].l,

                                            (a[0] + h.a1).l - a[0].l, lgt))

                      rl.append(str(ffi.buffer((b[0] + lb).l, lgt)))

                  la = h.a2

                  lb = h.b2

                  h = h.next

        Yuya Nishihara
    
cffi: split modules from pure...

              r32512
            
        Yuya Nishihara
    
cffi: remove superfluous "if True" blocks

              r32513
            
          finally:

              lib.free(a[0])

              lib.free(b[0])

              lib.bdiff_freehunks(l.next)

          return "".join(rl)

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages

Yuya Nishihara cffi: split modules from pure...	r32512	# bdiff.py - CFFI implementation of bdiff.c
		#
		# Copyright 2016 Maciej Fijalkowski <fijall@gmail.com>
		#
		# This software may be used and distributed according to the terms of the
		# GNU General Public License version 2 or any later version.

		from __future__ import absolute_import

		import struct

		from ..pure.bdiff import *
		from . import _bdiff

		ffi = _bdiff.ffi
		lib = _bdiff.lib

Yuya Nishihara cffi: remove superfluous "if True" blocks	r32513	def blocks(sa, sb):
		a = ffi.new("struct bdiff_line**")
		b = ffi.new("struct bdiff_line**")
		ac = ffi.new("char[]", str(sa))
		bc = ffi.new("char[]", str(sb))
		l = ffi.new("struct bdiff_hunk*")
		try:
		an = lib.bdiff_splitlines(ac, len(sa), a)
		bn = lib.bdiff_splitlines(bc, len(sb), b)
		if not a[0] or not b[0]:
		raise MemoryError
		count = lib.bdiff_diff(a[0], an, b[0], bn, l)
		if count < 0:
		raise MemoryError
		rl = [None] * count
		h = l.next
		i = 0
		while h:
		rl[i] = (h.a1, h.a2, h.b1, h.b2)
		h = h.next
		i += 1
		finally:
		lib.free(a[0])
		lib.free(b[0])
		lib.bdiff_freehunks(l.next)
		return rl
Yuya Nishihara cffi: split modules from pure...	r32512
Yuya Nishihara cffi: remove superfluous "if True" blocks	r32513	def bdiff(sa, sb):
		a = ffi.new("struct bdiff_line**")
		b = ffi.new("struct bdiff_line**")
		ac = ffi.new("char[]", str(sa))
		bc = ffi.new("char[]", str(sb))
		l = ffi.new("struct bdiff_hunk*")
		try:
		an = lib.bdiff_splitlines(ac, len(sa), a)
		bn = lib.bdiff_splitlines(bc, len(sb), b)
		if not a[0] or not b[0]:
		raise MemoryError
		count = lib.bdiff_diff(a[0], an, b[0], bn, l)
		if count < 0:
		raise MemoryError
		rl = []
		h = l.next
		la = lb = 0
		while h:
		if h.a1 != la or h.b1 != lb:
		lgt = (b[0] + h.b1).l - (b[0] + lb).l
		rl.append(struct.pack(">lll", (a[0] + la).l - a[0].l,
		(a[0] + h.a1).l - a[0].l, lgt))
		rl.append(str(ffi.buffer((b[0] + lb).l, lgt)))
		la = h.a2
		lb = h.b2
		h = h.next
Yuya Nishihara cffi: split modules from pure...	r32512
Yuya Nishihara cffi: remove superfluous "if True" blocks	r32513	finally:
		lib.free(a[0])
		lib.free(b[0])
		lib.bdiff_freehunks(l.next)
		return "".join(rl)