summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2018-06-29 16:50:17 +0000
committerGerrit Code Review <review@openstack.org>2018-06-29 16:50:17 +0000
commitfe80343d905860faa85ffeca339461405b7f5efb (patch)
tree492d653e55e8eb3de30ffff4349420df0f3d39bd
parentc08eb291f77f204b0a3037b32b4946c7c237df64 (diff)
parent69f952b83e1db748e503efe6c3061f2c57c2c47a (diff)
Merge "zk: retry initial zookeeper connection attempts"3.1.0
-rwxr-xr-xnodepool/zk.py22
1 files changed, 21 insertions, 1 deletions
diff --git a/nodepool/zk.py b/nodepool/zk.py
index 6a5d18a..6ead919 100755
--- a/nodepool/zk.py
+++ b/nodepool/zk.py
@@ -20,6 +20,7 @@ import logging
20import time 20import time
21from kazoo.client import KazooClient, KazooState 21from kazoo.client import KazooClient, KazooState
22from kazoo import exceptions as kze 22from kazoo import exceptions as kze
23from kazoo.handlers.threading import KazooTimeoutError
23from kazoo.recipe.lock import Lock 24from kazoo.recipe.lock import Lock
24 25
25from nodepool import exceptions as npe 26from nodepool import exceptions as npe
@@ -655,12 +656,16 @@ class ZooKeeper(object):
655 REQUEST_ROOT = "/nodepool/requests" 656 REQUEST_ROOT = "/nodepool/requests"
656 REQUEST_LOCK_ROOT = "/nodepool/requests-lock" 657 REQUEST_LOCK_ROOT = "/nodepool/requests-lock"
657 658
659 # Log zookeeper retry every 10 seconds
660 retry_log_rate = 10
661
658 def __init__(self): 662 def __init__(self):
659 ''' 663 '''
660 Initialize the ZooKeeper object. 664 Initialize the ZooKeeper object.
661 ''' 665 '''
662 self.client = None 666 self.client = None
663 self._became_lost = False 667 self._became_lost = False
668 self._last_retry_log = 0
664 669
665 # ======================================================================= 670 # =======================================================================
666 # Private Methods 671 # Private Methods
@@ -788,6 +793,15 @@ class ZooKeeper(object):
788 else: 793 else:
789 self.log.debug("ZooKeeper connection: CONNECTED") 794 self.log.debug("ZooKeeper connection: CONNECTED")
790 795
796 def logConnectionRetryEvent(self):
797 '''
798 Kazoo retry callback
799 '''
800 now = time.monotonic()
801 if now - self._last_retry_log >= self.retry_log_rate:
802 self.log.warning("Retrying zookeeper connection")
803 self._last_retry_log = now
804
791 # ======================================================================= 805 # =======================================================================
792 # Public Methods and Properties 806 # Public Methods and Properties
793 # ======================================================================= 807 # =======================================================================
@@ -834,7 +848,13 @@ class ZooKeeper(object):
834 hosts = buildZooKeeperHosts(host_list) 848 hosts = buildZooKeeperHosts(host_list)
835 self.client = KazooClient(hosts=hosts, read_only=read_only) 849 self.client = KazooClient(hosts=hosts, read_only=read_only)
836 self.client.add_listener(self._connection_listener) 850 self.client.add_listener(self._connection_listener)
837 self.client.start() 851 # Manually retry initial connection attempt
852 while True:
853 try:
854 self.client.start(1)
855 break
856 except KazooTimeoutError:
857 self.logConnectionRetryEvent()
838 858
839 def disconnect(self): 859 def disconnect(self):
840 ''' 860 '''