Use shade to get the client objects
As an initial step in the transition to shade, just replace the getting of the client objects with shade. This adds support for but does not require using a clouds.yaml file and named clouds. This is an opt-in feature and is not required by people using nodepool. But, having a clouds.yaml on a server allows the same cloud configuration to also be used by other tools, such as ansible and python-openstackclient, so while not required, it's a nice to have. Using shade to get the client connection handles also gives us access to other client libraries, such as neutron, without structural changes, although the attempt here is to not change anything logically or move the ownership of any code tasks. The one exception is the removal of using the novaclient extension for finding the neutron networks, since shade neither has a facility to passing novaclient extensions in as parameters, nor does it want to grow one. Co-Authored By: Gregory Haynes <greg@greghaynes.net> Depends-On: I22c33648e32cc7ce8fc163433b7c72912c28beb9 Change-Id: Iba84b8d578efa8cb7f5af85a5ffbc97f945a47c9
This commit is contained in:
parent
3c635ec9c2
commit
9e2937cedc
|
@ -50,6 +50,7 @@ class ConfigValidator:
|
|||
'service-name': str,
|
||||
'availability-zones': [str],
|
||||
'keypair': str,
|
||||
'cloud': str,
|
||||
'username': str,
|
||||
'password': str,
|
||||
'auth-url': str,
|
||||
|
|
|
@ -53,6 +53,26 @@ def get_fake_images_list():
|
|||
return fake_images_list
|
||||
|
||||
|
||||
BAD_CLIENT = None
|
||||
|
||||
|
||||
def get_bad_client():
|
||||
global BAD_CLIENT
|
||||
if BAD_CLIENT is None:
|
||||
BAD_CLIENT = BadOpenstackCloud()
|
||||
return BAD_CLIENT
|
||||
|
||||
|
||||
FAKE_CLIENT = None
|
||||
|
||||
|
||||
def get_fake_client(**kwargs):
|
||||
global FAKE_CLIENT
|
||||
if FAKE_CLIENT is None:
|
||||
FAKE_CLIENT = FakeOpenStackCloud()
|
||||
return FAKE_CLIENT
|
||||
|
||||
|
||||
class FakeList(object):
|
||||
def __init__(self, l):
|
||||
self._list = l
|
||||
|
@ -143,9 +163,13 @@ class BadClient(FakeClient):
|
|||
self.client = BadHTTPClient()
|
||||
|
||||
|
||||
class BadOpenstackCloud(object):
|
||||
nova_client = BadClient()
|
||||
|
||||
|
||||
class FakeGlanceClient(object):
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.id = 'fake-glance-id'
|
||||
def __init__(self, **kwargs):
|
||||
self.kwargs = kwargs
|
||||
self.images = get_fake_images_list()
|
||||
|
||||
|
||||
|
@ -160,6 +184,11 @@ class FakeKeystoneClient(object):
|
|||
self.auth_token = 'fake-auth-token'
|
||||
|
||||
|
||||
class FakeOpenStackCloud(object):
|
||||
nova_client = FakeClient()
|
||||
glance_client = FakeGlanceClient()
|
||||
|
||||
|
||||
class FakeFile(StringIO.StringIO):
|
||||
def __init__(self, path):
|
||||
StringIO.StringIO.__init__(self)
|
||||
|
@ -236,7 +265,3 @@ class FakeJenkins(object):
|
|||
{u'name': u'test-view',
|
||||
u'url': u'https://jenkins.example.com/view/test-view/'}]}
|
||||
return d
|
||||
|
||||
|
||||
FAKE_CLIENT = FakeClient()
|
||||
BAD_CLIENT = BadClient()
|
||||
|
|
|
@ -1249,10 +1249,11 @@ class NodePool(threading.Thread):
|
|||
p = Provider()
|
||||
p.name = provider['name']
|
||||
newconfig.providers[p.name] = p
|
||||
p.username = provider['username']
|
||||
p.password = provider['password']
|
||||
p.project_id = provider['project-id']
|
||||
p.auth_url = provider['auth-url']
|
||||
p.username = provider.get('username')
|
||||
p.password = provider.get('password')
|
||||
p.project_id = provider.get('project-id')
|
||||
p.auth_url = provider.get('auth-url')
|
||||
p.cloud = provider.get('cloud')
|
||||
p.service_type = provider.get('service-type')
|
||||
p.service_name = provider.get('service-name')
|
||||
p.region_name = provider.get('region-name')
|
||||
|
@ -1397,6 +1398,7 @@ class NodePool(threading.Thread):
|
|||
new_pm.password != old_pm.provider.password or
|
||||
new_pm.project_id != old_pm.provider.project_id or
|
||||
new_pm.auth_url != old_pm.provider.auth_url or
|
||||
new_pm.cloud != old_pm.provider.cloud or
|
||||
new_pm.service_type != old_pm.provider.service_type or
|
||||
new_pm.service_name != old_pm.provider.service_name or
|
||||
new_pm.max_servers != old_pm.provider.max_servers or
|
||||
|
|
|
@ -18,18 +18,15 @@
|
|||
|
||||
import logging
|
||||
import paramiko
|
||||
import novaclient
|
||||
import novaclient.client
|
||||
import novaclient.extension
|
||||
import novaclient.v2.contrib.tenant_networks
|
||||
|
||||
import threading
|
||||
import glanceclient
|
||||
import glanceclient.client
|
||||
import keystoneclient.v2_0.client as ksclient
|
||||
import time
|
||||
import requests.exceptions
|
||||
import sys
|
||||
|
||||
import shade
|
||||
import novaclient
|
||||
|
||||
from nodeutils import iterate_timeout
|
||||
from task_manager import Task, TaskManager, ManagerStoppedException
|
||||
|
||||
|
@ -113,14 +110,14 @@ class NotFound(Exception):
|
|||
|
||||
class CreateServerTask(Task):
|
||||
def main(self, client):
|
||||
server = client.servers.create(**self.args)
|
||||
server = client.nova_client.servers.create(**self.args)
|
||||
return str(server.id)
|
||||
|
||||
|
||||
class GetServerTask(Task):
|
||||
def main(self, client):
|
||||
try:
|
||||
server = client.servers.get(self.args['server_id'])
|
||||
server = client.nova_client.servers.get(self.args['server_id'])
|
||||
except novaclient.exceptions.NotFound:
|
||||
raise NotFound()
|
||||
return make_server_dict(server)
|
||||
|
@ -128,52 +125,52 @@ class GetServerTask(Task):
|
|||
|
||||
class DeleteServerTask(Task):
|
||||
def main(self, client):
|
||||
client.servers.delete(self.args['server_id'])
|
||||
client.nova_client.servers.delete(self.args['server_id'])
|
||||
|
||||
|
||||
class ListServersTask(Task):
|
||||
def main(self, client):
|
||||
servers = client.servers.list()
|
||||
servers = client.nova_client.servers.list()
|
||||
return [make_server_dict(server) for server in servers]
|
||||
|
||||
|
||||
class AddKeypairTask(Task):
|
||||
def main(self, client):
|
||||
client.keypairs.create(**self.args)
|
||||
client.nova_client.keypairs.create(**self.args)
|
||||
|
||||
|
||||
class ListKeypairsTask(Task):
|
||||
def main(self, client):
|
||||
keys = client.keypairs.list()
|
||||
keys = client.nova_client.keypairs.list()
|
||||
return [dict(id=str(key.id), name=key.name) for
|
||||
key in keys]
|
||||
|
||||
|
||||
class DeleteKeypairTask(Task):
|
||||
def main(self, client):
|
||||
client.keypairs.delete(self.args['name'])
|
||||
client.nova_client.keypairs.delete(self.args['name'])
|
||||
|
||||
|
||||
class CreateFloatingIPTask(Task):
|
||||
def main(self, client):
|
||||
ip = client.floating_ips.create(**self.args)
|
||||
ip = client.nova_client.floating_ips.create(**self.args)
|
||||
return dict(id=str(ip.id), ip=ip.ip)
|
||||
|
||||
|
||||
class AddFloatingIPTask(Task):
|
||||
def main(self, client):
|
||||
client.servers.add_floating_ip(**self.args)
|
||||
client.nova_client.servers.add_floating_ip(**self.args)
|
||||
|
||||
|
||||
class GetFloatingIPTask(Task):
|
||||
def main(self, client):
|
||||
ip = client.floating_ips.get(self.args['ip_id'])
|
||||
ip = client.nova_client.floating_ips.get(self.args['ip_id'])
|
||||
return dict(id=str(ip.id), ip=ip.ip, instance_id=str(ip.instance_id))
|
||||
|
||||
|
||||
class ListFloatingIPsTask(Task):
|
||||
def main(self, client):
|
||||
ips = client.floating_ips.list()
|
||||
ips = client.nova_client.floating_ips.list()
|
||||
return [dict(id=str(ip.id), ip=ip.ip,
|
||||
instance_id=str(ip.instance_id)) for
|
||||
ip in ips]
|
||||
|
@ -181,24 +178,24 @@ class ListFloatingIPsTask(Task):
|
|||
|
||||
class RemoveFloatingIPTask(Task):
|
||||
def main(self, client):
|
||||
client.servers.remove_floating_ip(**self.args)
|
||||
client.nova_client.servers.remove_floating_ip(**self.args)
|
||||
|
||||
|
||||
class DeleteFloatingIPTask(Task):
|
||||
def main(self, client):
|
||||
client.floating_ips.delete(self.args['ip_id'])
|
||||
client.nova_client.floating_ips.delete(self.args['ip_id'])
|
||||
|
||||
|
||||
class CreateImageTask(Task):
|
||||
def main(self, client):
|
||||
# This returns an id
|
||||
return str(client.servers.create_image(**self.args))
|
||||
return str(client.nova_client.servers.create_image(**self.args))
|
||||
|
||||
|
||||
class GetImageTask(Task):
|
||||
def main(self, client):
|
||||
try:
|
||||
image = client.images.get(**self.args)
|
||||
image = client.nova_client.images.get(**self.args)
|
||||
except novaclient.exceptions.NotFound:
|
||||
raise NotFound()
|
||||
# HP returns 404, rackspace can return a 'DELETED' image.
|
||||
|
@ -210,7 +207,7 @@ class GetImageTask(Task):
|
|||
class ListExtensionsTask(Task):
|
||||
def main(self, client):
|
||||
try:
|
||||
resp, body = client.client.get('/extensions')
|
||||
resp, body = client.nova_client.client.get('/extensions')
|
||||
return [x['alias'] for x in body['extensions']]
|
||||
except novaclient.exceptions.NotFound:
|
||||
# No extensions present.
|
||||
|
@ -219,32 +216,33 @@ class ListExtensionsTask(Task):
|
|||
|
||||
class ListFlavorsTask(Task):
|
||||
def main(self, client):
|
||||
flavors = client.flavors.list()
|
||||
flavors = client.nova_client.flavors.list()
|
||||
return [dict(id=str(flavor.id), ram=flavor.ram, name=flavor.name)
|
||||
for flavor in flavors]
|
||||
|
||||
|
||||
class ListImagesTask(Task):
|
||||
def main(self, client):
|
||||
images = client.images.list()
|
||||
images = client.nova_client.images.list()
|
||||
return [make_image_dict(image) for image in images]
|
||||
|
||||
|
||||
class FindImageTask(Task):
|
||||
def main(self, client):
|
||||
image = client.images.find(**self.args)
|
||||
image = client.nova_client.images.find(**self.args)
|
||||
return dict(id=str(image.id))
|
||||
|
||||
|
||||
class DeleteImageTask(Task):
|
||||
def main(self, client):
|
||||
client.images.delete(**self.args)
|
||||
client.nova_client.images.delete(**self.args)
|
||||
|
||||
|
||||
class FindNetworkTask(Task):
|
||||
def main(self, client):
|
||||
network = client.tenant_networks.find(**self.args)
|
||||
return dict(id=str(network.id))
|
||||
for network in client.neutron_client.list_networks()['networks']:
|
||||
if self.args['label'] == network['name']:
|
||||
return dict(id=str(network['id']))
|
||||
|
||||
|
||||
class ProviderManager(TaskManager):
|
||||
|
@ -285,20 +283,31 @@ class ProviderManager(TaskManager):
|
|||
self._cloud_metadata_read = True
|
||||
|
||||
def _getClient(self):
|
||||
tenant_networks = novaclient.extension.Extension(
|
||||
'tenant_networks', novaclient.v2.contrib.tenant_networks)
|
||||
args = ['1.1', self.provider.username, self.provider.password,
|
||||
self.provider.project_id, self.provider.auth_url]
|
||||
kwargs = {'extensions': [tenant_networks]}
|
||||
if self.provider.service_type:
|
||||
kwargs['service_type'] = self.provider.service_type
|
||||
if self.provider.service_name:
|
||||
kwargs['service_name'] = self.provider.service_name
|
||||
kwargs = {}
|
||||
if self.provider.region_name:
|
||||
kwargs['region_name'] = self.provider.region_name
|
||||
if self.provider.api_timeout:
|
||||
kwargs['timeout'] = self.provider.api_timeout
|
||||
return novaclient.client.Client(*args, **kwargs)
|
||||
kwargs['api_timeout'] = self.provider.api_timeout
|
||||
# These are named from back when we only talked to Nova. They're
|
||||
# actually compute service related
|
||||
if self.provider.service_type:
|
||||
kwargs['compute_service_type'] = self.provider.service_type
|
||||
if self.provider.service_name:
|
||||
kwargs['compute_service_name'] = self.provider.service_name
|
||||
if self.provider.cloud is not None:
|
||||
kwargs['cloud'] = self.provider.cloud
|
||||
|
||||
auth_kwargs = {}
|
||||
for auth_attr in ('username', 'password', 'auth_url'):
|
||||
auth_val = getattr(self.provider, auth_attr)
|
||||
if auth_val is not None:
|
||||
auth_kwargs[auth_attr] = auth_val
|
||||
|
||||
if self.provider.project_id is not None:
|
||||
auth_kwargs['project_name'] = self.provider.project_id
|
||||
|
||||
kwargs['auth'] = auth_kwargs
|
||||
return shade.openstack_cloud(**kwargs)
|
||||
|
||||
def runTask(self, task):
|
||||
try:
|
||||
|
@ -454,6 +463,8 @@ class ProviderManager(TaskManager):
|
|||
return
|
||||
|
||||
def waitForImage(self, image_id, timeout=3600):
|
||||
# TODO(mordred): This should just be handled by the Fake, but we're
|
||||
# not quite plumbed through for that yet
|
||||
if image_id == 'fake-glance-id':
|
||||
return True
|
||||
return self._waitForResource('image', image_id, timeout)
|
||||
|
@ -496,37 +507,11 @@ class ProviderManager(TaskManager):
|
|||
def getImage(self, image_id):
|
||||
return self.submitTask(GetImageTask(image=image_id))
|
||||
|
||||
def get_glance_client(self, provider):
|
||||
keystone_kwargs = {'auth_url': provider.auth_url,
|
||||
'username': provider.username,
|
||||
'password': provider.password,
|
||||
'tenant_name': provider.project_id}
|
||||
glance_kwargs = {'service_type': 'image'}
|
||||
glance_endpoint_kwargs = {'service_type': 'image'}
|
||||
|
||||
if provider.region_name:
|
||||
keystone_kwargs['region_name'] = provider.region_name
|
||||
glance_endpoint_kwargs['attr'] = 'region'
|
||||
glance_endpoint_kwargs['filter_value'] = provider.region_name
|
||||
|
||||
# get endpoint and authtoken
|
||||
keystone = ksclient.Client(**keystone_kwargs)
|
||||
glance_endpoint = keystone.service_catalog.url_for(
|
||||
**glance_endpoint_kwargs)
|
||||
glance_endpoint = glance_endpoint.replace('/v1.0', '')
|
||||
|
||||
# configure glance client
|
||||
glance = glanceclient.client.Client('1', glance_endpoint,
|
||||
token=keystone.auth_token,
|
||||
**glance_kwargs)
|
||||
return glance
|
||||
|
||||
def uploadImage(self, image_name, filename, disk_format, container_format,
|
||||
meta):
|
||||
# configure glance and upload image. Note the meta flags
|
||||
# are provided as custom glance properties
|
||||
glanceclient = self.get_glance_client(self.provider)
|
||||
image = glanceclient.images.create(
|
||||
image = self._client.glance_client.images.create(
|
||||
name=image_name,
|
||||
is_public=False,
|
||||
disk_format=disk_format,
|
||||
|
@ -534,7 +519,6 @@ class ProviderManager(TaskManager):
|
|||
**meta)
|
||||
filename = '%s.%s' % (filename, disk_format)
|
||||
image.update(data=open(filename, 'rb'))
|
||||
glanceclient = None
|
||||
return image.id
|
||||
|
||||
def listExtensions(self):
|
||||
|
|
|
@ -39,7 +39,6 @@ class LoggingPopen(subprocess.Popen):
|
|||
|
||||
|
||||
class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(BaseTestCase, self).setUp()
|
||||
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 60)
|
||||
|
@ -79,6 +78,8 @@ class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase):
|
|||
self.setUpFakes()
|
||||
|
||||
def setUpFakes(self):
|
||||
self.useFixture(fixtures.MonkeyPatch('shade.openstack_cloud',
|
||||
fakeprovider.get_fake_client))
|
||||
self.useFixture(fixtures.MonkeyPatch('keystoneclient.v2_0.client.'
|
||||
'Client',
|
||||
fakeprovider.FakeKeystoneClient))
|
||||
|
@ -226,3 +227,8 @@ class DBTestCase(BaseTestCase):
|
|||
break
|
||||
time.sleep(1)
|
||||
self.wait_for_threads()
|
||||
|
||||
|
||||
class IntegrationTestCase(DBTestCase):
|
||||
def setUpFakes(self):
|
||||
pass
|
||||
|
|
|
@ -441,13 +441,8 @@ providers:
|
|||
username: jenkins
|
||||
private-key: /home/nodepool/.ssh/id_rsa
|
||||
- name: hpcloud-b1
|
||||
cloud: hpcloud
|
||||
region-name: 'region-b.geo-1'
|
||||
service-type: 'compute'
|
||||
service-name: 'Compute'
|
||||
username: '<%= hpcloud_username %>'
|
||||
password: '<%= hpcloud_password %>'
|
||||
project-id: '<%= hpcloud_project %>'
|
||||
auth-url: 'https://region-b.geo-1.identity.hpcloudsvc.com:35357/v2.0'
|
||||
api-timeout: 60
|
||||
boot-timeout: 120
|
||||
max-servers: 100
|
||||
|
|
|
@ -0,0 +1,47 @@
|
|||
script-dir: .
|
||||
dburi: '{dburi}'
|
||||
images-dir: '{images_dir}'
|
||||
|
||||
cron:
|
||||
check: '*/15 * * * *'
|
||||
cleanup: '*/1 * * * *'
|
||||
image-update: '14 2 * * *'
|
||||
|
||||
zmq-publishers:
|
||||
- tcp://localhost:8881
|
||||
|
||||
#gearman-servers:
|
||||
# - host: localhost
|
||||
|
||||
labels:
|
||||
- name: fake-label
|
||||
image: fake-image
|
||||
min-ready: 1
|
||||
providers:
|
||||
- name: fake-provider
|
||||
|
||||
providers:
|
||||
- name: fake-provider
|
||||
cloud: fake-cloud
|
||||
keypair: 'if-present-use-this-keypair'
|
||||
max-servers: 96
|
||||
pool: 'fake'
|
||||
networks:
|
||||
- net-id: 'some-uuid'
|
||||
rate: 0.0001
|
||||
images:
|
||||
- name: fake-image
|
||||
base-image: 'Fake Precise'
|
||||
min-ram: 8192
|
||||
name-filter: 'Fake'
|
||||
meta:
|
||||
key: value
|
||||
key2: value
|
||||
setup: prepare_node_devstack.sh
|
||||
|
||||
targets:
|
||||
- name: fake-target
|
||||
jenkins:
|
||||
url: https://jenkins.example.org/
|
||||
user: fake
|
||||
apikey: fake
|
|
@ -299,7 +299,7 @@ class TestNodepool(tests.DBTestCase):
|
|||
# always raises a ProxyError. If our client reset works correctly
|
||||
# then we will create a new client object, which in this case would
|
||||
# be a new fake client in place of the bad client.
|
||||
manager._client = nodepool.fakeprovider.BAD_CLIENT
|
||||
manager._client = nodepool.fakeprovider.get_bad_client()
|
||||
|
||||
# The only implemented function for the fake and bad clients
|
||||
# If we don't raise an uncaught exception, we pass
|
||||
|
@ -308,7 +308,7 @@ class TestNodepool(tests.DBTestCase):
|
|||
# Now let's do it again, but let's prevent the client object from being
|
||||
# replaced and then assert that we raised the exception that we expect.
|
||||
manager._client = nodepool.fakeprovider.BAD_CLIENT
|
||||
manager._getClient = lambda: nodepool.fakeprovider.BAD_CLIENT
|
||||
manager._getClient = lambda: nodepool.fakeprovider.get_bad_client()
|
||||
|
||||
with ExpectedException(requests.exceptions.ProxyError):
|
||||
manager.listExtensions()
|
||||
|
|
|
@ -0,0 +1,54 @@
|
|||
# Copyright (C) 2015 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
|
||||
import yaml
|
||||
|
||||
from nodepool import tests
|
||||
|
||||
|
||||
class TestShadeIntegration(tests.IntegrationTestCase):
|
||||
def _cleanup_cloud_config(self):
|
||||
os.remove('clouds.yaml')
|
||||
|
||||
def _use_cloud_config(self, config):
|
||||
with open('clouds.yaml', 'w') as h:
|
||||
yaml.safe_dump(config, h)
|
||||
self.addCleanup(self._cleanup_cloud_config)
|
||||
|
||||
def test_nodepool_provider_config(self):
|
||||
configfile = self.setup_config('node.yaml')
|
||||
pool = self.useNodepool(configfile, watermark_sleep=1)
|
||||
pool.updateConfig()
|
||||
provider_manager = pool.config.provider_managers['fake-provider']
|
||||
auth_data = {'username': 'fake',
|
||||
'project_name': 'fake',
|
||||
'password': 'fake',
|
||||
'auth_url': 'fake'}
|
||||
self.assertEqual(provider_manager._client.auth, auth_data)
|
||||
|
||||
def test_nodepool_osc_config(self):
|
||||
configfile = self.setup_config('node_osc.yaml')
|
||||
auth_data = {'username': 'os_fake',
|
||||
'project_name': 'os_fake',
|
||||
'password': 'os_fake',
|
||||
'auth_url': 'os_fake'}
|
||||
osc_config = {'clouds': {'fake-cloud': {'auth': auth_data}}}
|
||||
self._use_cloud_config(osc_config)
|
||||
|
||||
pool = self.useNodepool(configfile, watermark_sleep=1)
|
||||
pool.updateConfig()
|
||||
provider_manager = pool.config.provider_managers['fake-provider']
|
||||
self.assertEqual(provider_manager._client.auth, auth_data)
|
|
@ -16,5 +16,7 @@ python-novaclient>=2.21.0
|
|||
PyMySQL
|
||||
PrettyTable>=0.6,<0.8
|
||||
six>=1.7.0
|
||||
# shade has a looser requirement on six than nodepool, so install six first
|
||||
shade
|
||||
diskimage-builder
|
||||
voluptuous
|
||||
|
|
Loading…
Reference in New Issue