##// END OF EJS Templates
util: allow lrucachedict to track cost of entries...
util: allow lrucachedict to track cost of entries Currently, lrucachedict allows tracking of arbitrary items with the only limit being the total number of items in the cache. Caches can be a lot more useful when they are bound by the size of the items in them rather than the number of elements in the cache. In preparation for teaching lrucachedict to enforce a max size of cached items, we teach lrucachedict to optionally associate a numeric cost value with each node. We purposefully let the caller define their own cost for nodes. This does introduce some overhead. Most of it comes from __setitem__, since that function now calls into insert(), thus introducing Python function call overhead. $ hg perflrucachedict --size 4 --gets 1000000 --sets 1000000 --mixed 1000000 ! gets ! wall 0.599552 comb 0.600000 user 0.600000 sys 0.000000 (best of 17) ! wall 0.614643 comb 0.610000 user 0.610000 sys 0.000000 (best of 17) ! inserts ! <not available> ! wall 0.655817 comb 0.650000 user 0.650000 sys 0.000000 (best of 16) ! sets ! wall 0.540448 comb 0.540000 user 0.540000 sys 0.000000 (best of 18) ! wall 0.805644 comb 0.810000 user 0.810000 sys 0.000000 (best of 13) ! mixed ! wall 0.651556 comb 0.660000 user 0.660000 sys 0.000000 (best of 15) ! wall 0.781357 comb 0.780000 user 0.780000 sys 0.000000 (best of 13) $ hg perflrucachedict --size 1000 --gets 1000000 --sets 1000000 --mixed 1000000 ! gets ! wall 0.621014 comb 0.620000 user 0.620000 sys 0.000000 (best of 16) ! wall 0.615146 comb 0.620000 user 0.620000 sys 0.000000 (best of 17) ! inserts ! <not available> ! wall 0.698115 comb 0.700000 user 0.700000 sys 0.000000 (best of 15) ! sets ! wall 0.560247 comb 0.560000 user 0.560000 sys 0.000000 (best of 18) ! wall 0.832495 comb 0.830000 user 0.830000 sys 0.000000 (best of 12) ! mixed ! wall 0.686172 comb 0.680000 user 0.680000 sys 0.000000 (best of 15) ! wall 0.841359 comb 0.840000 user 0.840000 sys 0.000000 (best of 12) We're still under 1us per insert, which seems like reasonable performance for a cache. If we comment out updating of self.totalcost during insert(), performance of insert() is identical to __setitem__ before. However, I don't want to make total cost evaluation lazy because it has significant performance implications for when we need to evaluate the total cost at mutation time (it requires a cache traversal, which could be expensive for large caches). Differential Revision: https://phab.mercurial-scm.org/D4502

File last commit:

r39603:ee087f0d default
r39603:ee087f0d default
Show More
test-lrucachedict.py
221 lines | 6.2 KiB | text/x-python | PythonLexer
/ tests / test-lrucachedict.py
Pulkit Goyal
tests: make test-lrucachedict use print_function
r28931 from __future__ import absolute_import, print_function
Pulkit Goyal
tests: make test-lrucachedict use absolute_import
r28930
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 import unittest
import silenttestrunner
Pulkit Goyal
tests: make test-lrucachedict use absolute_import
r28930 from mercurial import (
util,
)
Siddharth Agarwal
util: add an LRU cache dict...
r18603
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 class testlrucachedict(unittest.TestCase):
def testsimple(self):
d = util.lrucachedict(4)
Gregory Szorc
util: make capacity a public attribute on lrucachedict...
r39600 self.assertEqual(d.capacity, 4)
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 d.insert('a', 'va', cost=2)
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 d['b'] = 'vb'
d['c'] = 'vc'
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 d.insert('d', 'vd', cost=42)
Siddharth Agarwal
util: add an LRU cache dict...
r18603
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 self.assertEqual(d['a'], 'va')
self.assertEqual(d['b'], 'vb')
self.assertEqual(d['c'], 'vc')
self.assertEqual(d['d'], 'vd')
Siddharth Agarwal
util: add an LRU cache dict...
r18603
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 self.assertEqual(d.totalcost, 44)
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 # 'a' should be dropped because it was least recently used.
d['e'] = 've'
self.assertNotIn('a', d)
self.assertIsNone(d.get('a'))
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 self.assertEqual(d.totalcost, 42)
Siddharth Agarwal
util: add an LRU cache dict...
r18603
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 self.assertEqual(d['b'], 'vb')
self.assertEqual(d['c'], 'vc')
self.assertEqual(d['d'], 'vd')
self.assertEqual(d['e'], 've')
Siddharth Agarwal
util: add an LRU cache dict...
r18603
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 # Replacing item with different cost adjusts totalcost.
d.insert('e', 've', cost=4)
self.assertEqual(d.totalcost, 46)
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 # Touch entries in some order (both get and set).
d['e']
d['c'] = 'vc2'
d['d']
d['b'] = 'vb2'
Gregory Szorc
util: properly implement lrucachedict.get()...
r29828
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 # 'e' should be dropped now
d['f'] = 'vf'
self.assertNotIn('e', d)
self.assertEqual(d['b'], 'vb2')
self.assertEqual(d['c'], 'vc2')
self.assertEqual(d['d'], 'vd')
self.assertEqual(d['f'], 'vf')
Siddharth Agarwal
util: add an LRU cache dict...
r18603
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 d.clear()
for key in ('a', 'b', 'c', 'd', 'e', 'f'):
self.assertNotIn(key, d)
Siddharth Agarwal
util: add an LRU cache dict...
r18603
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 def testunfull(self):
d = util.lrucachedict(4)
d['a'] = 1
d['b'] = 2
d['a']
d['b']
for key in ('a', 'b'):
self.assertIn(key, d)
Siddharth Agarwal
lrucachedict: implement clear()
r19710
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 def testcopypartial(self):
d = util.lrucachedict(4)
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 d.insert('a', 'va', cost=4)
d.insert('b', 'vb', cost=2)
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598
dc = d.copy()
self.assertEqual(len(dc), 2)
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 self.assertEqual(dc.totalcost, 6)
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 for key in ('a', 'b'):
self.assertIn(key, dc)
self.assertEqual(dc[key], 'v%s' % key)
Gregory Szorc
util: reimplement lrucachedict...
r27371
Gregory Szorc
util: properly copy lrucachedict instances...
r39599 self.assertEqual(len(d), 2)
for key in ('a', 'b'):
self.assertIn(key, d)
self.assertEqual(d[key], 'v%s' % key)
d['c'] = 'vc'
del d['b']
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 self.assertEqual(d.totalcost, 4)
Gregory Szorc
util: properly copy lrucachedict instances...
r39599 dc = d.copy()
self.assertEqual(len(dc), 2)
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 self.assertEqual(dc.totalcost, 4)
Gregory Szorc
util: properly copy lrucachedict instances...
r39599 for key in ('a', 'c'):
self.assertIn(key, dc)
self.assertEqual(dc[key], 'v%s' % key)
def testcopyempty(self):
d = util.lrucachedict(4)
dc = d.copy()
self.assertEqual(len(dc), 0)
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 def testcopyfull(self):
d = util.lrucachedict(4)
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 d.insert('a', 'va', cost=42)
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 d['b'] = 'vb'
d['c'] = 'vc'
d['d'] = 'vd'
Eric Sumner
lrucachedict: add copy method...
r27576
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 dc = d.copy()
for key in ('a', 'b', 'c', 'd'):
self.assertIn(key, dc)
self.assertEqual(dc[key], 'v%s' % key)
Eric Sumner
lrucachedict: add copy method...
r27576
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 self.assertEqual(d.totalcost, 42)
self.assertEqual(dc.totalcost, 42)
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 # 'a' should be dropped because it was least recently used.
dc['e'] = 've'
self.assertNotIn('a', dc)
for key in ('b', 'c', 'd', 'e'):
self.assertIn(key, dc)
self.assertEqual(dc[key], 'v%s' % key)
Eric Sumner
lrucachedict: add copy method...
r27576
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 self.assertEqual(d.totalcost, 42)
self.assertEqual(dc.totalcost, 0)
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 # Contents and order of original dict should remain unchanged.
dc['b'] = 'vb_new'
Eric Sumner
lrucachedict: add copy method...
r27576
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 self.assertEqual(list(iter(d)), ['d', 'c', 'b', 'a'])
for key in ('a', 'b', 'c', 'd'):
self.assertEqual(d[key], 'v%s' % key)
Eric Sumner
lrucachedict: add copy method...
r27576
Gregory Szorc
util: ability to change capacity when copying lrucachedict...
r39601 def testcopydecreasecapacity(self):
d = util.lrucachedict(5)
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 d.insert('a', 'va', cost=4)
d.insert('b', 'vb', cost=2)
Gregory Szorc
util: ability to change capacity when copying lrucachedict...
r39601 d['c'] = 'vc'
d['d'] = 'vd'
dc = d.copy(2)
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 self.assertEqual(dc.totalcost, 0)
Gregory Szorc
util: ability to change capacity when copying lrucachedict...
r39601 for key in ('a', 'b'):
self.assertNotIn(key, dc)
for key in ('c', 'd'):
self.assertIn(key, dc)
self.assertEqual(dc[key], 'v%s' % key)
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 dc.insert('e', 've', cost=7)
self.assertEqual(dc.totalcost, 7)
Gregory Szorc
util: ability to change capacity when copying lrucachedict...
r39601 self.assertNotIn('c', dc)
for key in ('d', 'e'):
self.assertIn(key, dc)
self.assertEqual(dc[key], 'v%s' % key)
# Original should remain unchanged.
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 self.assertEqual(d.totalcost, 6)
Gregory Szorc
util: ability to change capacity when copying lrucachedict...
r39601 for key in ('a', 'b', 'c', 'd'):
self.assertIn(key, d)
self.assertEqual(d[key], 'v%s' % key)
def testcopyincreasecapacity(self):
d = util.lrucachedict(5)
d['a'] = 'va'
d['b'] = 'vb'
d['c'] = 'vc'
d['d'] = 'vd'
dc = d.copy(6)
for key in ('a', 'b', 'c', 'd'):
self.assertIn(key, dc)
self.assertEqual(dc[key], 'v%s' % key)
dc['e'] = 've'
dc['f'] = 'vf'
for key in ('a', 'b', 'c', 'd', 'e', 'f'):
self.assertIn(key, dc)
self.assertEqual(dc[key], 'v%s' % key)
dc['g'] = 'vg'
self.assertNotIn('a', dc)
for key in ('b', 'c', 'd', 'e', 'f', 'g'):
self.assertIn(key, dc)
self.assertEqual(dc[key], 'v%s' % key)
# Original should remain unchanged.
for key in ('a', 'b', 'c', 'd'):
self.assertIn(key, d)
self.assertEqual(d[key], 'v%s' % key)
Gregory Szorc
util: add a popoldest() method to lrucachedict...
r39602 def testpopoldest(self):
d = util.lrucachedict(4)
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 d.insert('a', 'va', cost=10)
d.insert('b', 'vb', cost=5)
Gregory Szorc
util: add a popoldest() method to lrucachedict...
r39602
self.assertEqual(len(d), 2)
self.assertEqual(d.popoldest(), ('a', 'va'))
self.assertEqual(len(d), 1)
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 self.assertEqual(d.totalcost, 5)
Gregory Szorc
util: add a popoldest() method to lrucachedict...
r39602 self.assertEqual(d.popoldest(), ('b', 'vb'))
self.assertEqual(len(d), 0)
Gregory Szorc
util: allow lrucachedict to track cost of entries...
r39603 self.assertEqual(d.totalcost, 0)
Gregory Szorc
util: add a popoldest() method to lrucachedict...
r39602 self.assertIsNone(d.popoldest())
d['a'] = 'va'
d['b'] = 'vb'
d['c'] = 'vc'
d['d'] = 'vd'
self.assertEqual(d.popoldest(), ('a', 'va'))
self.assertEqual(len(d), 3)
for key in ('b', 'c', 'd'):
self.assertEqual(d[key], 'v%s' % key)
d['a'] = 'va'
self.assertEqual(d.popoldest(), ('b', 'vb'))
Siddharth Agarwal
util: add an LRU cache dict...
r18603 if __name__ == '__main__':
Gregory Szorc
tests: rewrite test-lrucachedict.py to use unittest...
r39598 silenttestrunner.main(__name__)