Add command socket handler for full reconfiguration
In future we want to support different types of reconfigurations so only relying on signals won't scale. Thus we should make the full reconfiguration available via the command socket which will be extensible in the future. A later change will add a reconfiguration without clearing the cache to be able to quuickly add or remove projects from the tenant configuration without having too much impact into the system. Change-Id: I9748ecbcffa8c9b65f98d8768735bdf00e78cf25
This commit is contained in:
parent
08913074d3
commit
e20ebbe5cc
|
@ -284,9 +284,10 @@ PID which was saved in the pidfile specified in the configuration.
|
|||
Most of Zuul's configuration is automatically updated as changes to
|
||||
the repositories which contain it are merged. However, Zuul must be
|
||||
explicitly notified of changes to the tenant config file, since it is
|
||||
not read from a git repository. To do so, send the scheduler PID
|
||||
(saved in the pidfile specified in the configuration) a `SIGHUP`
|
||||
signal.
|
||||
not read from a git repository. To do so, run
|
||||
``zuul-scheduler full-reconfigure``. The signal based method by sending
|
||||
a `SIGHUP` signal to the scheduler PID is deprecated.
|
||||
|
||||
|
||||
Merger
|
||||
------
|
||||
|
|
|
@ -0,0 +1,10 @@
|
|||
---
|
||||
features:
|
||||
- |
|
||||
Zuul now supports triggering a full reconfiguration by using the command
|
||||
``zuul-scheduler full-reconfigure``.
|
||||
deprecations:
|
||||
- |
|
||||
Signal based triggering of a full reconfiguration via sending `SIGHUP` to
|
||||
the zuul-scheduler PID is deprecated. Use the command
|
||||
``zuul-scheduler full-reconfigure`` now.
|
|
@ -2397,6 +2397,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
gerritconnection.GerritEventConnector.delay = 0.0
|
||||
|
||||
self.sched = zuul.scheduler.Scheduler(self.config)
|
||||
self.sched.setZuulApp(self)
|
||||
self.sched._stats_interval = 1
|
||||
|
||||
self.event_queues = [
|
||||
|
@ -2447,6 +2448,12 @@ class ZuulTestCase(BaseTestCase):
|
|||
self.sched.reconfigure(self.config)
|
||||
self.sched.resume()
|
||||
|
||||
def fullReconfigure(self):
|
||||
try:
|
||||
self.sched.reconfigure(self.config)
|
||||
except Exception:
|
||||
self.log.exception("Reconfiguration failed:")
|
||||
|
||||
def configure_connections(self, source_only=False):
|
||||
# Set up gerrit related fakes
|
||||
# Set a changes database so multiple FakeGerrit's can report back to
|
||||
|
|
|
@ -18,6 +18,7 @@ import textwrap
|
|||
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import time
|
||||
from unittest import mock
|
||||
from unittest import skip
|
||||
|
@ -2884,6 +2885,32 @@ class TestScheduler(ZuulTestCase):
|
|||
self.assertEqual(A.data['status'], 'MERGED')
|
||||
self.assertEqual(A.reported, 2)
|
||||
|
||||
def test_live_reconfiguration_command_socket(self):
|
||||
"Test that live reconfiguration via command socket works"
|
||||
|
||||
# record previous tenant reconfiguration time, which may not be set
|
||||
old = self.sched.tenant_last_reconfigured.get('tenant-one', 0)
|
||||
time.sleep(1)
|
||||
self.waitUntilSettled()
|
||||
|
||||
command_socket = self.config.get('scheduler', 'command_socket')
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
|
||||
s.connect(command_socket)
|
||||
s.sendall('full-reconfigure\n'.encode('utf8'))
|
||||
|
||||
# Wait for full reconfiguration. Note that waitUntilSettled is not
|
||||
# reliable here because the reconfigure event may arrive in the
|
||||
# event queue after waitUntilSettled.
|
||||
start = time.time()
|
||||
while True:
|
||||
if time.time() - start > 15:
|
||||
raise Exception("Timeout waiting for full reconfiguration")
|
||||
new = self.sched.tenant_last_reconfigured.get('tenant-one', 0)
|
||||
if old < new:
|
||||
break
|
||||
else:
|
||||
time.sleep(0)
|
||||
|
||||
def test_live_reconfiguration_abort(self):
|
||||
# Raise an exception during reconfiguration and verify we
|
||||
# still function.
|
||||
|
|
|
@ -50,8 +50,7 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
|
|||
if self.args.command:
|
||||
self.args.nodaemon = True
|
||||
|
||||
def reconfigure_handler(self, signum, frame):
|
||||
signal.signal(signal.SIGHUP, signal.SIG_IGN)
|
||||
def fullReconfigure(self):
|
||||
self.log.debug("Reconfiguration triggered")
|
||||
self.readConfig()
|
||||
self.setup_logging('scheduler', 'log_config')
|
||||
|
@ -59,6 +58,10 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
|
|||
self.sched.reconfigure(self.config)
|
||||
except Exception:
|
||||
self.log.exception("Reconfiguration failed:")
|
||||
|
||||
def reconfigure_handler(self, signum, frame):
|
||||
signal.signal(signal.SIGHUP, signal.SIG_IGN)
|
||||
self.fullReconfigure()
|
||||
signal.signal(signal.SIGHUP, self.reconfigure_handler)
|
||||
|
||||
def exit_handler(self, signum, frame):
|
||||
|
@ -129,6 +132,7 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
|
|||
self.sched = zuul.scheduler.Scheduler(self.config)
|
||||
|
||||
gearman = zuul.executor.client.ExecutorClient(self.config, self.sched)
|
||||
self.sched.setZuulApp(self)
|
||||
merger = zuul.merger.client.MergeClient(self.config, self.sched)
|
||||
nodepool = zuul.nodepool.Nodepool(self.sched)
|
||||
|
||||
|
|
|
@ -36,7 +36,7 @@ from zuul.lib.config import get_default
|
|||
from zuul.lib.statsd import get_statsd
|
||||
import zuul.lib.queue
|
||||
|
||||
COMMANDS = ['stop']
|
||||
COMMANDS = ['full-reconfigure', 'stop']
|
||||
|
||||
|
||||
class ManagementEvent(object):
|
||||
|
@ -257,12 +257,14 @@ class Scheduler(threading.Thread):
|
|||
self.wake_event = threading.Event()
|
||||
self.layout_lock = threading.Lock()
|
||||
self.run_handler_lock = threading.Lock()
|
||||
self.command_map = dict(
|
||||
stop=self.stop,
|
||||
)
|
||||
self.command_map = {
|
||||
'stop': self.stop,
|
||||
'full-reconfigure': self.fullReconfigureCommandHandler,
|
||||
}
|
||||
self._pause = False
|
||||
self._exit = False
|
||||
self._stopped = False
|
||||
self._zuul_app = None
|
||||
self.executor = None
|
||||
self.merger = None
|
||||
self.connections = None
|
||||
|
@ -346,6 +348,9 @@ class Scheduler(threading.Thread):
|
|||
def stopConnections(self):
|
||||
self.connections.stop()
|
||||
|
||||
def setZuulApp(self, app):
|
||||
self._zuul_app = app
|
||||
|
||||
def setExecutor(self, executor):
|
||||
self.executor = executor
|
||||
|
||||
|
@ -478,6 +483,9 @@ class Scheduler(threading.Thread):
|
|||
self.management_event_queue.put(event)
|
||||
self.wake_event.set()
|
||||
|
||||
def fullReconfigureCommandHandler(self):
|
||||
self._zuul_app.fullReconfigure()
|
||||
|
||||
def reconfigure(self, config):
|
||||
self.log.debug("Submitting reconfiguration event")
|
||||
event = ReconfigureEvent(config)
|
||||
|
|
Loading…
Reference in New Issue