Add cleanup routine to delete empty nodes

We've discovered that our node deletion process has the possibility
to leave empty (i.e., no data) node znodes in ZooKeeper. Although a
fix for this has been merged, we need a way to remove this extraneous
data.

Change-Id: I6596060f5026088ce987e5d0d7c18b00a6b77c5a
This commit is contained in:
David Shrewsbury 2018-12-04 17:28:07 -05:00
parent 1b5d416f36
commit 387b134a52
3 changed files with 48 additions and 5 deletions

View File

@ -410,6 +410,7 @@ class CleanupWorker(BaseCleanupWorker):
(self._cleanupLostRequests, 'lost request cleanup'),
(self._cleanupMaxReadyAge, 'max ready age cleanup'),
(self._cleanupMaxHoldAge, 'max hold age cleanup'),
(self._cleanupEmptyNodes, 'empty node cleanup'),
]
def _resetLostRequest(self, zk_conn, req):
@ -616,6 +617,21 @@ class CleanupWorker(BaseCleanupWorker):
finally:
zk_conn.unlockNode(node)
def _cleanupEmptyNodes(self):
'''
Remove any Node znodes that may be totally empty.
'''
self.log.debug('Cleaning up empty nodes...')
zk_conn = self._nodepool.getZK()
# We cannot use nodeIterator() here since that does not yield us
# empty nodes.
for node_id in zk_conn.getNodes():
node = zk_conn.getNode(node_id)
if node is None:
self.log.debug("Removing empty node %s", node_id)
zk_conn.deleteRawNode(node_id)
def _run(self):
'''
Catch exceptions individually so that other cleanup routines may

View File

@ -1824,3 +1824,20 @@ class TestLauncher(tests.DBTestCase):
self.assertTrue(req2.id > req1.id)
self.assertTrue(req2.state_time < req1.state_time)
def test_empty_node_deleted(self):
"""Test that empty nodes are deleted by the cleanup thread"""
configfile = self.setup_config('node.yaml')
# Create empty node
path = "%s" % self.zk._nodePath("12345")
self.log.debug("node path %s", path)
self.zk.client.create(path, makepath=True)
self.assertTrue(self.zk.client.exists(path))
pool = self.useNodepool(configfile, watermark_sleep=1)
pool.cleanup_interval = .1
pool.start()
while self.zk.client.exists(path):
time.sleep(.1)

View File

@ -1882,6 +1882,20 @@ class ZooKeeper(object):
path = self._nodePath(node.id)
self.client.set(path, node.serialize())
def deleteRawNode(self, node_id):
'''
Delete a znode for a Node.
This is used to forcefully delete a Node znode that has somehow
ended up without any actual data. In most cases, you should be using
deleteNode() instead.
'''
path = self._nodePath(node_id)
try:
self.client.delete(path, recursive=True)
except kze.NoNodeError:
pass
def deleteNode(self, node):
'''
Delete a node.
@ -1898,11 +1912,7 @@ class ZooKeeper(object):
# lock is removed before the node deletion occurs.
node.state = DELETED
self.client.set(path, node.serialize())
try:
self.client.delete(path, recursive=True)
except kze.NoNodeError:
pass
self.deleteRawNode(node.id)
def getReadyNodesOfTypes(self, labels, cached=True):
'''