summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTobias Henkel <tobias.henkel@bmw.de>2018-11-19 18:56:04 +0100
committerTobias Henkel <tobias.henkel@bmw.de>2018-11-26 19:38:10 +0100
commit36666800ffcd961a3474cffddb7463e0a3524bbd (patch)
treed8754b5079322fabc8b43414757f8e7b27a67670
parente4f245eb1310935f3acdbf06616e93e7855482f5 (diff)
Cache node request zNodes
When implementing dynamic priority queues in nodepool we need to cache the requests. This introduces a simple node request caching using the TreeCache supplied by kazoo. This currently only caches the raw zookeeper data and still requires json parsing when traversing the requests. In a later change we can introduce a second layer of caching that updates the NodeRequest objects in place so we can avoid excessive json parsing. Change-Id: I84b3474636ea1f14faa6bf7c8a6c6f83598c3741
Notes
Notes (review): Code-Review+2: James E. Blair <corvus@inaugust.com> Code-Review+2: David Shrewsbury <shrewsbury.dave@gmail.com> Workflow+1: David Shrewsbury <shrewsbury.dave@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Wed, 28 Nov 2018 18:30:46 +0000 Reviewed-on: https://review.openstack.org/618806 Project: openstack-infra/nodepool Branch: refs/heads/master
-rwxr-xr-xnodepool/zk.py37
1 files changed, 30 insertions, 7 deletions
diff --git a/nodepool/zk.py b/nodepool/zk.py
index 5e84dcb..abb0faf 100755
--- a/nodepool/zk.py
+++ b/nodepool/zk.py
@@ -697,6 +697,7 @@ class ZooKeeper(object):
697 self._became_lost = False 697 self._became_lost = False
698 self._last_retry_log = 0 698 self._last_retry_log = 0
699 self._node_cache = None 699 self._node_cache = None
700 self._request_cache = None
700 701
701 # ======================================================================= 702 # =======================================================================
702 # Private Methods 703 # Private Methods
@@ -890,6 +891,9 @@ class ZooKeeper(object):
890 self._node_cache = TreeCache(self.client, self.NODE_ROOT) 891 self._node_cache = TreeCache(self.client, self.NODE_ROOT)
891 self._node_cache.start() 892 self._node_cache.start()
892 893
894 self._request_cache = TreeCache(self.client, self.REQUEST_ROOT)
895 self._request_cache.start()
896
893 def disconnect(self): 897 def disconnect(self):
894 ''' 898 '''
895 Close the ZooKeeper cluster connection. 899 Close the ZooKeeper cluster connection.
@@ -902,6 +906,10 @@ class ZooKeeper(object):
902 self._node_cache.close() 906 self._node_cache.close()
903 self._node_cache = None 907 self._node_cache = None
904 908
909 if self._request_cache is not None:
910 self._request_cache.close()
911 self._request_cache = None
912
905 if self.client is not None and self.client.connected: 913 if self.client is not None and self.client.connected:
906 self.client.stop() 914 self.client.stop()
907 self.client.close() 915 self.client.close()
@@ -1545,19 +1553,34 @@ class ZooKeeper(object):
1545 except kze.NoNodeError: 1553 except kze.NoNodeError:
1546 pass 1554 pass
1547 1555
1548 def getNodeRequest(self, request): 1556 def getNodeRequest(self, request, cached=False):
1549 ''' 1557 '''
1550 Get the data for a specific node request. 1558 Get the data for a specific node request.
1551 1559
1552 :param str request: The request ID. 1560 :param str request: The request ID.
1561 :param cached: True if cached node requests should be returned.
1553 1562
1554 :returns: The request data, or None if the request was not found. 1563 :returns: The request data, or None if the request was not found.
1555 ''' 1564 '''
1556 path = self._requestPath(request) 1565 path = self._requestPath(request)
1557 try: 1566 data = None
1558 data, stat = self.client.get(path) 1567 stat = None
1559 except kze.NoNodeError: 1568 if cached:
1560 return None 1569 cached_data = self._request_cache.get_data(path)
1570 if cached_data:
1571 data = cached_data.data
1572 stat = cached_data.stat
1573
1574 # If data is empty we either didn't use the cache or the cache didn't
1575 # have the request (yet). Note that even if we use caching we need to
1576 # do a real query if the cached data is empty because the request data
1577 # might not be in the cache yet when it's listed by the get_children
1578 # call.
1579 if not data:
1580 try:
1581 data, stat = self.client.get(path)
1582 except kze.NoNodeError:
1583 return None
1561 1584
1562 d = NodeRequest.fromDict(self._bytesToDict(data), request) 1585 d = NodeRequest.fromDict(self._bytesToDict(data), request)
1563 d.stat = stat 1586 d.stat = stat
@@ -1931,12 +1954,12 @@ class ZooKeeper(object):
1931 if lock_stats: 1954 if lock_stats:
1932 yield lock_stats 1955 yield lock_stats
1933 1956
1934 def nodeRequestIterator(self): 1957 def nodeRequestIterator(self, cached=True):
1935 ''' 1958 '''
1936 Utility generator method for iterating through all nodes requests. 1959 Utility generator method for iterating through all nodes requests.
1937 ''' 1960 '''
1938 for req_id in self.getNodeRequests(): 1961 for req_id in self.getNodeRequests():
1939 req = self.getNodeRequest(req_id) 1962 req = self.getNodeRequest(req_id, cached=cached)
1940 if req: 1963 if req:
1941 yield req 1964 yield req
1942 1965