summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTobias Henkel <tobias.henkel@bmw.de>2018-11-20 16:36:05 +0100
committerTobias Henkel <tobias.henkel@bmw.de>2018-11-26 20:04:04 +0100
commit35094dbb62f273bfdb64ed3c9f0ac2336c87f7f3 (patch)
tree247d1336561e815b890fceb579d55dbaf669af71
parent1d278ffaee965f04c3a24dd99afa85ac15d73cf2 (diff)
Add second level cache of nodes
An earlier change introduced zNode caching of Nodes. This first version still required frequent re-parsing of the node json data. This change introduces a second level cache that updates a dict of the current nodes based on the events emitted by the TreeCache. Using this we can reduce json parsing and make node caching mode effective. Change-Id: I4834a7ea722cf2ac7df79455ce077832ae966e63
Notes
Notes (review): Code-Review+2: James E. Blair <corvus@inaugust.com> Code-Review+2: David Shrewsbury <shrewsbury.dave@gmail.com> Workflow+1: David Shrewsbury <shrewsbury.dave@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Thu, 29 Nov 2018 09:59:39 +0000 Reviewed-on: https://review.openstack.org/619025 Project: openstack-infra/nodepool Branch: refs/heads/master
-rw-r--r--nodepool/tests/unit/test_commands.py31
-rwxr-xr-xnodepool/zk.py75
2 files changed, 80 insertions, 26 deletions
diff --git a/nodepool/tests/unit/test_commands.py b/nodepool/tests/unit/test_commands.py
index 9860527..373b52a 100644
--- a/nodepool/tests/unit/test_commands.py
+++ b/nodepool/tests/unit/test_commands.py
@@ -39,18 +39,25 @@ class TestNodepoolCMD(tests.DBTestCase):
39 def assert_listed(self, configfile, cmd, col, val, count, col_count=0): 39 def assert_listed(self, configfile, cmd, col, val, count, col_count=0):
40 log = logging.getLogger("tests.PrettyTableMock") 40 log = logging.getLogger("tests.PrettyTableMock")
41 self.patch_argv("-c", configfile, *cmd) 41 self.patch_argv("-c", configfile, *cmd)
42 with mock.patch('prettytable.PrettyTable.add_row') as m_add_row: 42 for _ in iterate_timeout(10, AssertionError, 'assert listed'):
43 nodepoolcmd.main() 43 try:
44 rows_with_val = 0 44 with mock.patch('prettytable.PrettyTable.add_row') as \
45 # Find add_rows with the status were looking for 45 m_add_row:
46 for args, kwargs in m_add_row.call_args_list: 46 nodepoolcmd.main()
47 row = args[0] 47 rows_with_val = 0
48 if col_count: 48 # Find add_rows with the status were looking for
49 self.assertEquals(len(row), col_count) 49 for args, kwargs in m_add_row.call_args_list:
50 log.debug(row) 50 row = args[0]
51 if row[col] == val: 51 if col_count:
52 rows_with_val += 1 52 self.assertEquals(len(row), col_count)
53 self.assertEquals(rows_with_val, count) 53 log.debug(row)
54 if row[col] == val:
55 rows_with_val += 1
56 self.assertEquals(rows_with_val, count)
57 break
58 except AssertionError:
59 # retry
60 pass
54 61
55 def assert_alien_images_listed(self, configfile, image_cnt, image_id): 62 def assert_alien_images_listed(self, configfile, image_cnt, image_id):
56 self.assert_listed(configfile, ['alien-image-list'], 2, image_id, 63 self.assert_listed(configfile, ['alien-image-list'], 2, image_id,
diff --git a/nodepool/zk.py b/nodepool/zk.py
index 66c2f38..6ce42e4 100755
--- a/nodepool/zk.py
+++ b/nodepool/zk.py
@@ -22,7 +22,7 @@ from kazoo.client import KazooClient, KazooState
22from kazoo import exceptions as kze 22from kazoo import exceptions as kze
23from kazoo.handlers.threading import KazooTimeoutError 23from kazoo.handlers.threading import KazooTimeoutError
24from kazoo.recipe.lock import Lock 24from kazoo.recipe.lock import Lock
25from kazoo.recipe.cache import TreeCache 25from kazoo.recipe.cache import TreeCache, TreeEvent
26 26
27from nodepool import exceptions as npe 27from nodepool import exceptions as npe
28 28
@@ -702,6 +702,7 @@ class ZooKeeper(object):
702 self._last_retry_log = 0 702 self._last_retry_log = 0
703 self._node_cache = None 703 self._node_cache = None
704 self._request_cache = None 704 self._request_cache = None
705 self._cached_nodes = {}
705 706
706 # ======================================================================= 707 # =======================================================================
707 # Private Methods 708 # Private Methods
@@ -893,6 +894,8 @@ class ZooKeeper(object):
893 self.logConnectionRetryEvent() 894 self.logConnectionRetryEvent()
894 895
895 self._node_cache = TreeCache(self.client, self.NODE_ROOT) 896 self._node_cache = TreeCache(self.client, self.NODE_ROOT)
897 self._node_cache.listen_fault(self.cacheFaultListener)
898 self._node_cache.listen(self.nodeCacheListener)
896 self._node_cache.start() 899 self._node_cache.start()
897 900
898 self._request_cache = TreeCache(self.client, self.REQUEST_ROOT) 901 self._request_cache = TreeCache(self.client, self.REQUEST_ROOT)
@@ -1780,25 +1783,21 @@ class ZooKeeper(object):
1780 1783
1781 :returns: The node data, or None if the node was not found. 1784 :returns: The node data, or None if the node was not found.
1782 ''' 1785 '''
1783 path = self._nodePath(node)
1784 data = None
1785 stat = None
1786 if cached: 1786 if cached:
1787 cached_data = self._node_cache.get_data(path) 1787 d = self._cached_nodes.get(node)
1788 if cached_data: 1788 if d:
1789 data = cached_data.data 1789 return d
1790 stat = cached_data.stat
1791 1790
1792 # If data is empty we either didn't use the cache or the cache didn't 1791 # We got here we either didn't use the cache or the cache didn't
1793 # have the node (yet). Note that even if we use caching we need to 1792 # have the node (yet). Note that even if we use caching we need to
1794 # do a real query if the cached data is empty because the node data 1793 # do a real query if the cached data is empty because the node data
1795 # might not be in the cache yet when it's listed by the get_children 1794 # might not be in the cache yet when it's listed by the get_children
1796 # call. 1795 # call.
1797 if not data: 1796 try:
1798 try: 1797 path = self._nodePath(node)
1799 data, stat = self.client.get(path) 1798 data, stat = self.client.get(path)
1800 except kze.NoNodeError: 1799 except kze.NoNodeError:
1801 return None 1800 return None
1802 1801
1803 if not data: 1802 if not data:
1804 return None 1803 return None
@@ -2060,3 +2059,51 @@ class ZooKeeper(object):
2060 ''' 2059 '''
2061 for node in provider_nodes: 2060 for node in provider_nodes:
2062 self.deleteNode(node) 2061 self.deleteNode(node)
2062
2063 def cacheFaultListener(self, e):
2064 self.log.exception(e)
2065
2066 def nodeCacheListener(self, event):
2067
2068 if hasattr(event.event_data, 'path'):
2069 # Ignore root node
2070 path = event.event_data.path
2071 if path == self.NODE_ROOT:
2072 return
2073
2074 # Ignore lock nodes
2075 if '/lock' in path:
2076 return
2077
2078 # Ignore any non-node related events such as connection events here
2079 if event.event_type not in (TreeEvent.NODE_ADDED,
2080 TreeEvent.NODE_UPDATED,
2081 TreeEvent.NODE_REMOVED):
2082 return
2083
2084 path = event.event_data.path
2085 node_id = path.rsplit('/', 1)[1]
2086
2087 if event.event_type in (TreeEvent.NODE_ADDED, TreeEvent.NODE_UPDATED):
2088 # Perform an in-place update of the already cached node if possible
2089 d = self._bytesToDict(event.event_data.data)
2090 old_node = self._cached_nodes.get(node_id)
2091 if old_node:
2092 if event.event_data.stat.version <= old_node.stat.version:
2093 # Don't update to older data
2094 return
2095 if old_node.lock:
2096 # Don't update a locked node
2097 return
2098 old_node.updateFromDict(d)
2099 old_node.stat = event.event_data.stat
2100 else:
2101 node = Node.fromDict(d, node_id)
2102 node.stat = event.event_data.stat
2103 self._cached_nodes[node_id] = node
2104 elif event.event_type == TreeEvent.NODE_REMOVED:
2105 try:
2106 del self._cached_nodes[node_id]
2107 except KeyError:
2108 # If it's already gone, don't care
2109 pass