Add command socket handler for full reconfiguration

In future we want to support different types of reconfigurations so
only relying on signals won't scale. Thus we should make the full
reconfiguration available via the command socket which will be
extensible in the future. A later change will add a reconfiguration
without clearing the cache to be able to quuickly add or remove
projects from the tenant configuration without having too much impact
into the system.

Change-Id: I9748ecbcffa8c9b65f98d8768735bdf00e78cf25
This commit is contained in:
Tobias Henkel 2018-07-11 15:50:19 +02:00
parent 08913074d3
commit e20ebbe5cc
6 changed files with 66 additions and 9 deletions

View File

@ -284,9 +284,10 @@ PID which was saved in the pidfile specified in the configuration.
Most of Zuul's configuration is automatically updated as changes to
the repositories which contain it are merged. However, Zuul must be
explicitly notified of changes to the tenant config file, since it is
not read from a git repository. To do so, send the scheduler PID
(saved in the pidfile specified in the configuration) a `SIGHUP`
signal.
not read from a git repository. To do so, run
``zuul-scheduler full-reconfigure``. The signal based method by sending
a `SIGHUP` signal to the scheduler PID is deprecated.
Merger
------

View File

@ -0,0 +1,10 @@
---
features:
- |
Zuul now supports triggering a full reconfiguration by using the command
``zuul-scheduler full-reconfigure``.
deprecations:
- |
Signal based triggering of a full reconfiguration via sending `SIGHUP` to
the zuul-scheduler PID is deprecated. Use the command
``zuul-scheduler full-reconfigure`` now.

View File

@ -2397,6 +2397,7 @@ class ZuulTestCase(BaseTestCase):
gerritconnection.GerritEventConnector.delay = 0.0
self.sched = zuul.scheduler.Scheduler(self.config)
self.sched.setZuulApp(self)
self.sched._stats_interval = 1
self.event_queues = [
@ -2447,6 +2448,12 @@ class ZuulTestCase(BaseTestCase):
self.sched.reconfigure(self.config)
self.sched.resume()
def fullReconfigure(self):
try:
self.sched.reconfigure(self.config)
except Exception:
self.log.exception("Reconfiguration failed:")
def configure_connections(self, source_only=False):
# Set up gerrit related fakes
# Set a changes database so multiple FakeGerrit's can report back to

View File

@ -18,6 +18,7 @@ import textwrap
import os
import shutil
import socket
import time
from unittest import mock
from unittest import skip
@ -2884,6 +2885,32 @@ class TestScheduler(ZuulTestCase):
self.assertEqual(A.data['status'], 'MERGED')
self.assertEqual(A.reported, 2)
def test_live_reconfiguration_command_socket(self):
"Test that live reconfiguration via command socket works"
# record previous tenant reconfiguration time, which may not be set
old = self.sched.tenant_last_reconfigured.get('tenant-one', 0)
time.sleep(1)
self.waitUntilSettled()
command_socket = self.config.get('scheduler', 'command_socket')
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
s.connect(command_socket)
s.sendall('full-reconfigure\n'.encode('utf8'))
# Wait for full reconfiguration. Note that waitUntilSettled is not
# reliable here because the reconfigure event may arrive in the
# event queue after waitUntilSettled.
start = time.time()
while True:
if time.time() - start > 15:
raise Exception("Timeout waiting for full reconfiguration")
new = self.sched.tenant_last_reconfigured.get('tenant-one', 0)
if old < new:
break
else:
time.sleep(0)
def test_live_reconfiguration_abort(self):
# Raise an exception during reconfiguration and verify we
# still function.

View File

@ -50,8 +50,7 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
if self.args.command:
self.args.nodaemon = True
def reconfigure_handler(self, signum, frame):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
def fullReconfigure(self):
self.log.debug("Reconfiguration triggered")
self.readConfig()
self.setup_logging('scheduler', 'log_config')
@ -59,6 +58,10 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
self.sched.reconfigure(self.config)
except Exception:
self.log.exception("Reconfiguration failed:")
def reconfigure_handler(self, signum, frame):
signal.signal(signal.SIGHUP, signal.SIG_IGN)
self.fullReconfigure()
signal.signal(signal.SIGHUP, self.reconfigure_handler)
def exit_handler(self, signum, frame):
@ -129,6 +132,7 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
self.sched = zuul.scheduler.Scheduler(self.config)
gearman = zuul.executor.client.ExecutorClient(self.config, self.sched)
self.sched.setZuulApp(self)
merger = zuul.merger.client.MergeClient(self.config, self.sched)
nodepool = zuul.nodepool.Nodepool(self.sched)

View File

@ -36,7 +36,7 @@ from zuul.lib.config import get_default
from zuul.lib.statsd import get_statsd
import zuul.lib.queue
COMMANDS = ['stop']
COMMANDS = ['full-reconfigure', 'stop']
class ManagementEvent(object):
@ -257,12 +257,14 @@ class Scheduler(threading.Thread):
self.wake_event = threading.Event()
self.layout_lock = threading.Lock()
self.run_handler_lock = threading.Lock()
self.command_map = dict(
stop=self.stop,
)
self.command_map = {
'stop': self.stop,
'full-reconfigure': self.fullReconfigureCommandHandler,
}
self._pause = False
self._exit = False
self._stopped = False
self._zuul_app = None
self.executor = None
self.merger = None
self.connections = None
@ -346,6 +348,9 @@ class Scheduler(threading.Thread):
def stopConnections(self):
self.connections.stop()
def setZuulApp(self, app):
self._zuul_app = app
def setExecutor(self, executor):
self.executor = executor
@ -478,6 +483,9 @@ class Scheduler(threading.Thread):
self.management_event_queue.put(event)
self.wake_event.set()
def fullReconfigureCommandHandler(self):
self._zuul_app.fullReconfigure()
def reconfigure(self, config):
self.log.debug("Submitting reconfiguration event")
event = ReconfigureEvent(config)