summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2018-12-29 14:35:28 +0000
committerGerrit Code Review <review@openstack.org>2018-12-29 14:35:28 +0000
commit0fc3485454e2a0c2ac57fcb9b7a8b5d9ffbaaab4 (patch)
treeb181bd516022c4b304b2f0d3f0252a8b6a8b33e4
parent46ff3fd009822ac99bb3001773b5678e6be4374a (diff)
parent22efec8423940821a0068c63c471e808d24bd904 (diff)
Merge "web: add /{tenant}/labels route"
-rw-r--r--tests/base.py14
-rwxr-xr-xtests/unit/test_web.py9
-rwxr-xr-xzuul/cmd/web.py3
-rwxr-xr-xzuul/web/__init__.py24
-rw-r--r--zuul/zk.py66
5 files changed, 112 insertions, 4 deletions
diff --git a/tests/base.py b/tests/base.py
index cac2102..97af80d 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -1736,6 +1736,7 @@ class FakeSMTP(object):
1736class FakeNodepool(object): 1736class FakeNodepool(object):
1737 REQUEST_ROOT = '/nodepool/requests' 1737 REQUEST_ROOT = '/nodepool/requests'
1738 NODE_ROOT = '/nodepool/nodes' 1738 NODE_ROOT = '/nodepool/nodes'
1739 LAUNCHER_ROOT = '/nodepool/launchers'
1739 1740
1740 log = logging.getLogger("zuul.test.FakeNodepool") 1741 log = logging.getLogger("zuul.test.FakeNodepool")
1741 1742
@@ -1745,6 +1746,7 @@ class FakeNodepool(object):
1745 self.client = kazoo.client.KazooClient( 1746 self.client = kazoo.client.KazooClient(
1746 hosts='%s:%s%s' % (host, port, chroot)) 1747 hosts='%s:%s%s' % (host, port, chroot))
1747 self.client.start() 1748 self.client.start()
1749 self.registerLauncher()
1748 self._running = True 1750 self._running = True
1749 self.paused = False 1751 self.paused = False
1750 self.thread = threading.Thread(target=self.run) 1752 self.thread = threading.Thread(target=self.run)
@@ -1783,6 +1785,12 @@ class FakeNodepool(object):
1783 for req in self.getNodeRequests(): 1785 for req in self.getNodeRequests():
1784 self.fulfillRequest(req) 1786 self.fulfillRequest(req)
1785 1787
1788 def registerLauncher(self, labels=["label1"], id="FakeLauncher"):
1789 path = os.path.join(self.LAUNCHER_ROOT, id)
1790 data = {'id': id, 'supported_labels': labels}
1791 self.client.create(
1792 path, json.dumps(data).encode('utf8'), makepath=True)
1793
1786 def getNodeRequests(self): 1794 def getNodeRequests(self):
1787 try: 1795 try:
1788 reqids = self.client.get_children(self.REQUEST_ROOT) 1796 reqids = self.client.get_children(self.REQUEST_ROOT)
@@ -2054,7 +2062,7 @@ class WebProxyFixture(fixtures.Fixture):
2054 2062
2055 2063
2056class ZuulWebFixture(fixtures.Fixture): 2064class ZuulWebFixture(fixtures.Fixture):
2057 def __init__(self, gearman_server_port, config, info=None): 2065 def __init__(self, gearman_server_port, config, info=None, zk_hosts=None):
2058 super(ZuulWebFixture, self).__init__() 2066 super(ZuulWebFixture, self).__init__()
2059 self.gearman_server_port = gearman_server_port 2067 self.gearman_server_port = gearman_server_port
2060 self.connections = zuul.lib.connections.ConnectionRegistry() 2068 self.connections = zuul.lib.connections.ConnectionRegistry()
@@ -2066,6 +2074,7 @@ class ZuulWebFixture(fixtures.Fixture):
2066 self.info = zuul.model.WebInfo() 2074 self.info = zuul.model.WebInfo()
2067 else: 2075 else:
2068 self.info = info 2076 self.info = info
2077 self.zk_hosts = zk_hosts
2069 2078
2070 def _setUp(self): 2079 def _setUp(self):
2071 # Start the web server 2080 # Start the web server
@@ -2073,7 +2082,8 @@ class ZuulWebFixture(fixtures.Fixture):
2073 listen_address='::', listen_port=0, 2082 listen_address='::', listen_port=0,
2074 gear_server='127.0.0.1', gear_port=self.gearman_server_port, 2083 gear_server='127.0.0.1', gear_port=self.gearman_server_port,
2075 info=self.info, 2084 info=self.info,
2076 connections=self.connections) 2085 connections=self.connections,
2086 zk_hosts=self.zk_hosts)
2077 self.web.start() 2087 self.web.start()
2078 self.addCleanup(self.stop) 2088 self.addCleanup(self.stop)
2079 2089
diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py
index 31b2d9b..063cf8d 100755
--- a/tests/unit/test_web.py
+++ b/tests/unit/test_web.py
@@ -52,7 +52,8 @@ class BaseTestWeb(ZuulTestCase):
52 ZuulWebFixture( 52 ZuulWebFixture(
53 self.gearman_server.port, 53 self.gearman_server.port,
54 self.config, 54 self.config,
55 info=zuul.model.WebInfo.fromConfig(self.zuul_ini_config))) 55 info=zuul.model.WebInfo.fromConfig(self.zuul_ini_config),
56 zk_hosts=self.zk_config))
56 57
57 self.executor_server.hold_jobs_in_build = True 58 self.executor_server.hold_jobs_in_build = True
58 59
@@ -372,6 +373,12 @@ class TestWeb(BaseTestWeb):
372 'voting': True 373 'voting': True
373 }], data) 374 }], data)
374 375
376 def test_web_labels_list(self):
377 # can we fetch the labels list
378 data = self.get_url('api/tenant/tenant-one/labels').json()
379 expected_list = [{'name': 'label1'}]
380 self.assertEqual(expected_list, data)
381
375 def test_web_pipeline_list(self): 382 def test_web_pipeline_list(self):
376 # can we fetch the list of pipelines 383 # can we fetch the list of pipelines
377 data = self.get_url('api/tenant/tenant-one/pipelines').json() 384 data = self.get_url('api/tenant/tenant-one/pipelines').json()
diff --git a/zuul/cmd/web.py b/zuul/cmd/web.py
index 89c910f..c77a5db 100755
--- a/zuul/cmd/web.py
+++ b/zuul/cmd/web.py
@@ -64,6 +64,9 @@ class WebServer(zuul.cmd.ZuulDaemonApp):
64 self.log.exception("Error validating config") 64 self.log.exception("Error validating config")
65 sys.exit(1) 65 sys.exit(1)
66 66
67 params["zk_hosts"] = get_default(
68 self.config, 'zookeeper', 'hosts', '127.0.0.1:2181')
69
67 try: 70 try:
68 self.web = zuul.web.ZuulWeb(**params) 71 self.web = zuul.web.ZuulWeb(**params)
69 except Exception: 72 except Exception:
diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py
index 1b49e8f..ff31543 100755
--- a/zuul/web/__init__.py
+++ b/zuul/web/__init__.py
@@ -31,6 +31,7 @@ import threading
31 31
32import zuul.model 32import zuul.model
33import zuul.rpcclient 33import zuul.rpcclient
34import zuul.zk
34 35
35STATIC_DIR = os.path.join(os.path.dirname(__file__), 'static') 36STATIC_DIR = os.path.join(os.path.dirname(__file__), 'static')
36cherrypy.tools.websocket = WebSocketTool() 37cherrypy.tools.websocket = WebSocketTool()
@@ -195,6 +196,7 @@ class ZuulWebAPI(object):
195 196
196 def __init__(self, zuulweb): 197 def __init__(self, zuulweb):
197 self.rpc = zuulweb.rpc 198 self.rpc = zuulweb.rpc
199 self.zk = zuulweb.zk
198 self.zuulweb = zuulweb 200 self.zuulweb = zuulweb
199 self.cache = {} 201 self.cache = {}
200 self.cache_time = {} 202 self.cache_time = {}
@@ -348,6 +350,19 @@ class ZuulWebAPI(object):
348 350
349 @cherrypy.expose 351 @cherrypy.expose
350 @cherrypy.tools.save_params() 352 @cherrypy.tools.save_params()
353 @cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
354 def labels(self, tenant):
355 labels = set()
356 for launcher in self.zk.getRegisteredLaunchers():
357 for label in launcher.supported_labels:
358 labels.add(label)
359 ret = [{'name': label} for label in sorted(labels)]
360 resp = cherrypy.response
361 resp.headers['Access-Control-Allow-Origin'] = '*'
362 return ret
363
364 @cherrypy.expose
365 @cherrypy.tools.save_params()
351 def key(self, tenant, project): 366 def key(self, tenant, project):
352 job = self.rpc.submitJob('zuul:key_get', {'tenant': tenant, 367 job = self.rpc.submitJob('zuul:key_get', {'tenant': tenant,
353 'project': project, 368 'project': project,
@@ -560,7 +575,8 @@ class ZuulWeb(object):
560 static_cache_expiry=3600, 575 static_cache_expiry=3600,
561 connections=None, 576 connections=None,
562 info=None, 577 info=None,
563 static_path=None): 578 static_path=None,
579 zk_hosts=None):
564 self.start_time = time.time() 580 self.start_time = time.time()
565 self.listen_address = listen_address 581 self.listen_address = listen_address
566 self.listen_port = listen_port 582 self.listen_port = listen_port
@@ -573,6 +589,9 @@ class ZuulWeb(object):
573 # instanciate handlers 589 # instanciate handlers
574 self.rpc = zuul.rpcclient.RPCClient(gear_server, gear_port, 590 self.rpc = zuul.rpcclient.RPCClient(gear_server, gear_port,
575 ssl_key, ssl_cert, ssl_ca) 591 ssl_key, ssl_cert, ssl_ca)
592 self.zk = zuul.zk.ZooKeeper()
593 if zk_hosts:
594 self.zk.connect(hosts=zk_hosts, read_only=True)
576 self.connections = connections 595 self.connections = connections
577 self.stream_manager = StreamManager() 596 self.stream_manager = StreamManager()
578 597
@@ -598,6 +617,8 @@ class ZuulWeb(object):
598 controller=api, action='project') 617 controller=api, action='project')
599 route_map.connect('api', '/api/tenant/{tenant}/pipelines', 618 route_map.connect('api', '/api/tenant/{tenant}/pipelines',
600 controller=api, action='pipelines') 619 controller=api, action='pipelines')
620 route_map.connect('api', '/api/tenant/{tenant}/labels',
621 controller=api, action='labels')
601 route_map.connect('api', '/api/tenant/{tenant}/key/{project:.*}.pub', 622 route_map.connect('api', '/api/tenant/{tenant}/key/{project:.*}.pub',
602 controller=api, action='key') 623 controller=api, action='key')
603 route_map.connect('api', '/api/tenant/{tenant}/' 624 route_map.connect('api', '/api/tenant/{tenant}/'
@@ -661,6 +682,7 @@ class ZuulWeb(object):
661 cherrypy.server.httpserver = None 682 cherrypy.server.httpserver = None
662 self.wsplugin.unsubscribe() 683 self.wsplugin.unsubscribe()
663 self.stream_manager.stop() 684 self.stream_manager.stop()
685 self.zk.disconnect()
664 686
665 687
666if __name__ == "__main__": 688if __name__ == "__main__":
diff --git a/zuul/zk.py b/zuul/zk.py
index bd476c0..0ccc1f6 100644
--- a/zuul/zk.py
+++ b/zuul/zk.py
@@ -382,3 +382,69 @@ class ZooKeeper(object):
382 node_data.get('hold_job') == identifier): 382 node_data.get('hold_job') == identifier):
383 count += 1 383 count += 1
384 return count 384 return count
385
386 # Copy of nodepool/zk.py begins here
387 LAUNCHER_ROOT = "/nodepool/launchers"
388
389 def _bytesToDict(self, data):
390 return json.loads(data.decode('utf8'))
391
392 def _launcherPath(self, launcher):
393 return "%s/%s" % (self.LAUNCHER_ROOT, launcher)
394
395 def getRegisteredLaunchers(self):
396 '''
397 Get a list of all launchers that have registered with ZooKeeper.
398
399 :returns: A list of Launcher objects, or empty list if none are found.
400 '''
401 try:
402 launcher_ids = self.client.get_children(self.LAUNCHER_ROOT)
403 except kze.NoNodeError:
404 return []
405
406 objs = []
407 for launcher in launcher_ids:
408 path = self._launcherPath(launcher)
409 try:
410 data, _ = self.client.get(path)
411 except kze.NoNodeError:
412 # launcher disappeared
413 continue
414
415 objs.append(Launcher.fromDict(self._bytesToDict(data)))
416 return objs
417
418
419class Launcher():
420 '''
421 Class to describe a nodepool launcher.
422 '''
423
424 def __init__(self):
425 self.id = None
426 self._supported_labels = set()
427
428 def __eq__(self, other):
429 if isinstance(other, Launcher):
430 return (self.id == other.id and
431 self.supported_labels == other.supported_labels)
432 else:
433 return False
434
435 @property
436 def supported_labels(self):
437 return self._supported_labels
438
439 @supported_labels.setter
440 def supported_labels(self, value):
441 if not isinstance(value, set):
442 raise TypeError("'supported_labels' attribute must be a set")
443 self._supported_labels = value
444
445 @staticmethod
446 def fromDict(d):
447 obj = Launcher()
448 obj.id = d.get('id')
449 obj.supported_labels = set(d.get('supported_labels', []))
450 return obj