summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDavid Shrewsbury <shrewsbury.dave@gmail.com>2018-12-04 17:28:07 -0500
committerDavid Shrewsbury <shrewsbury.dave@gmail.com>2018-12-04 17:33:33 -0500
commit387b134a526af40362d16e9333d3d60d4fd4b859 (patch)
tree81dd2dba329a0def519f7175aa4faa7249cda1ca
parent1b5d416f36dc027474dcc3eccfd5665e30884cc7 (diff)
Add cleanup routine to delete empty nodes
We've discovered that our node deletion process has the possibility to leave empty (i.e., no data) node znodes in ZooKeeper. Although a fix for this has been merged, we need a way to remove this extraneous data. Change-Id: I6596060f5026088ce987e5d0d7c18b00a6b77c5a
Notes
Notes (review): Code-Review+2: James E. Blair <corvus@inaugust.com> Code-Review+2: Tobias Henkel <tobias.henkel@bmw.de> Workflow+1: Tobias Henkel <tobias.henkel@bmw.de> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Wed, 05 Dec 2018 05:47:39 +0000 Reviewed-on: https://review.openstack.org/622616 Project: openstack-infra/nodepool Branch: refs/heads/master
-rwxr-xr-xnodepool/launcher.py16
-rw-r--r--nodepool/tests/unit/test_launcher.py17
-rwxr-xr-xnodepool/zk.py20
3 files changed, 48 insertions, 5 deletions
diff --git a/nodepool/launcher.py b/nodepool/launcher.py
index e881ec2..8a2b734 100755
--- a/nodepool/launcher.py
+++ b/nodepool/launcher.py
@@ -410,6 +410,7 @@ class CleanupWorker(BaseCleanupWorker):
410 (self._cleanupLostRequests, 'lost request cleanup'), 410 (self._cleanupLostRequests, 'lost request cleanup'),
411 (self._cleanupMaxReadyAge, 'max ready age cleanup'), 411 (self._cleanupMaxReadyAge, 'max ready age cleanup'),
412 (self._cleanupMaxHoldAge, 'max hold age cleanup'), 412 (self._cleanupMaxHoldAge, 'max hold age cleanup'),
413 (self._cleanupEmptyNodes, 'empty node cleanup'),
413 ] 414 ]
414 415
415 def _resetLostRequest(self, zk_conn, req): 416 def _resetLostRequest(self, zk_conn, req):
@@ -616,6 +617,21 @@ class CleanupWorker(BaseCleanupWorker):
616 finally: 617 finally:
617 zk_conn.unlockNode(node) 618 zk_conn.unlockNode(node)
618 619
620 def _cleanupEmptyNodes(self):
621 '''
622 Remove any Node znodes that may be totally empty.
623 '''
624 self.log.debug('Cleaning up empty nodes...')
625 zk_conn = self._nodepool.getZK()
626
627 # We cannot use nodeIterator() here since that does not yield us
628 # empty nodes.
629 for node_id in zk_conn.getNodes():
630 node = zk_conn.getNode(node_id)
631 if node is None:
632 self.log.debug("Removing empty node %s", node_id)
633 zk_conn.deleteRawNode(node_id)
634
619 def _run(self): 635 def _run(self):
620 ''' 636 '''
621 Catch exceptions individually so that other cleanup routines may 637 Catch exceptions individually so that other cleanup routines may
diff --git a/nodepool/tests/unit/test_launcher.py b/nodepool/tests/unit/test_launcher.py
index 49bb688..2d8994f 100644
--- a/nodepool/tests/unit/test_launcher.py
+++ b/nodepool/tests/unit/test_launcher.py
@@ -1824,3 +1824,20 @@ class TestLauncher(tests.DBTestCase):
1824 1824
1825 self.assertTrue(req2.id > req1.id) 1825 self.assertTrue(req2.id > req1.id)
1826 self.assertTrue(req2.state_time < req1.state_time) 1826 self.assertTrue(req2.state_time < req1.state_time)
1827
1828 def test_empty_node_deleted(self):
1829 """Test that empty nodes are deleted by the cleanup thread"""
1830 configfile = self.setup_config('node.yaml')
1831
1832 # Create empty node
1833 path = "%s" % self.zk._nodePath("12345")
1834 self.log.debug("node path %s", path)
1835 self.zk.client.create(path, makepath=True)
1836 self.assertTrue(self.zk.client.exists(path))
1837
1838 pool = self.useNodepool(configfile, watermark_sleep=1)
1839 pool.cleanup_interval = .1
1840 pool.start()
1841
1842 while self.zk.client.exists(path):
1843 time.sleep(.1)
diff --git a/nodepool/zk.py b/nodepool/zk.py
index cfdb6df..e42f644 100755
--- a/nodepool/zk.py
+++ b/nodepool/zk.py
@@ -1882,6 +1882,20 @@ class ZooKeeper(object):
1882 path = self._nodePath(node.id) 1882 path = self._nodePath(node.id)
1883 self.client.set(path, node.serialize()) 1883 self.client.set(path, node.serialize())
1884 1884
1885 def deleteRawNode(self, node_id):
1886 '''
1887 Delete a znode for a Node.
1888
1889 This is used to forcefully delete a Node znode that has somehow
1890 ended up without any actual data. In most cases, you should be using
1891 deleteNode() instead.
1892 '''
1893 path = self._nodePath(node_id)
1894 try:
1895 self.client.delete(path, recursive=True)
1896 except kze.NoNodeError:
1897 pass
1898
1885 def deleteNode(self, node): 1899 def deleteNode(self, node):
1886 ''' 1900 '''
1887 Delete a node. 1901 Delete a node.
@@ -1898,11 +1912,7 @@ class ZooKeeper(object):
1898 # lock is removed before the node deletion occurs. 1912 # lock is removed before the node deletion occurs.
1899 node.state = DELETED 1913 node.state = DELETED
1900 self.client.set(path, node.serialize()) 1914 self.client.set(path, node.serialize())
1901 1915 self.deleteRawNode(node.id)
1902 try:
1903 self.client.delete(path, recursive=True)
1904 except kze.NoNodeError:
1905 pass
1906 1916
1907 def getReadyNodesOfTypes(self, labels, cached=True): 1917 def getReadyNodesOfTypes(self, labels, cached=True):
1908 ''' 1918 '''