summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2019-02-04 16:13:05 +0000
committerGerrit Code Review <review@openstack.org>2019-02-04 16:13:05 +0000
commit7b640f7f48e46c2ba6b4603d38e077716a13c57e (patch)
tree7b332ce8862b75cb1cb0becf713d2847b38abe49
parentacf038aef4c4ebb3fd410a333e6c35b002e76e8a (diff)
parentaa16b8b8911fdaa07a68ad933de9fb4d07461134 (diff)
Merge "Amazon EC2 driver"
-rw-r--r--nodepool/driver/aws/__init__.py27
-rw-r--r--nodepool/driver/aws/config.py285
-rw-r--r--nodepool/driver/aws/handler.py157
-rw-r--r--nodepool/driver/aws/provider.py151
-rw-r--r--nodepool/tests/__init__.py29
-rw-r--r--nodepool/tests/fixtures/aws.yaml26
-rw-r--r--nodepool/tests/fixtures/config_validate/good.yaml22
-rw-r--r--nodepool/tests/unit/test_driver_aws.py95
-rw-r--r--releasenotes/notes/aws-driver-6d6c25381066b9ca.yaml7
-rw-r--r--requirements.txt1
-rw-r--r--test-requirements.txt1
11 files changed, 788 insertions, 13 deletions
diff --git a/nodepool/driver/aws/__init__.py b/nodepool/driver/aws/__init__.py
new file mode 100644
index 0000000..d3120c9
--- /dev/null
+++ b/nodepool/driver/aws/__init__.py
@@ -0,0 +1,27 @@
1# Copyright 2018 Red Hat
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13#
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17from nodepool.driver import Driver
18from nodepool.driver.aws.config import AwsProviderConfig
19from nodepool.driver.aws.provider import AwsProvider
20
21
22class AwsDriver(Driver):
23 def getProviderConfig(self, provider):
24 return AwsProviderConfig(self, provider)
25
26 def getProvider(self, provider_config, use_taskmanager):
27 return AwsProvider(provider_config, use_taskmanager)
diff --git a/nodepool/driver/aws/config.py b/nodepool/driver/aws/config.py
new file mode 100644
index 0000000..5742811
--- /dev/null
+++ b/nodepool/driver/aws/config.py
@@ -0,0 +1,285 @@
1# Copyright 2018 Red Hat
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13#
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import math
18import voluptuous as v
19
20from nodepool.driver import ConfigPool
21from nodepool.driver import ConfigValue
22from nodepool.driver import ProviderConfig
23
24
25class ProviderCloudImage(ConfigValue):
26 def __init__(self):
27 self.name = None
28 self.image_id = None
29 self.image_name = None
30 self.username = None
31 self.connection_type = None
32 self.connection_port = None
33
34 def __eq__(self, other):
35 if isinstance(other, ProviderCloudImage):
36 return (self.name == other.name
37 and self.image_id == other.image_id
38 and self.image_name == other.image_name
39 and self.username == other.username
40 and self.connection_type == other.connection_type
41 and self.connection_port == other.connection_port)
42 return False
43
44 def __repr__(self):
45 return "<ProviderCloudImage %s>" % self.name
46
47 @property
48 def external_name(self):
49 '''Human readable version of external.'''
50 return self.image_id or self.image_name or self.name
51
52
53class ProviderLabel(ConfigValue):
54 def __init__(self):
55 self.name = None
56 self.cloud_image = None
57 self.flavor_name = None
58 self.key_name = None
59 self.volume_size = None
60 self.volume_type = None
61 # The ProviderPool object that owns this label.
62 self.pool = None
63
64 def __eq__(self, other):
65 if isinstance(other, ProviderLabel):
66 # NOTE(Shrews): We intentionally do not compare 'pool' here
67 # since this causes recursive checks with ProviderPool.
68 return (other.name == self.name
69 and other.cloud_image == self.cloud_image
70 and other.flavor_name == self.flavor_name
71 and other.key_name == self.key_name
72 and other.volume_size == self.volume_size
73 and other.volume_type == self.volume_type)
74 return False
75
76 def __repr__(self):
77 return "<ProviderLabel %s>" % self.name
78
79
80class ProviderPool(ConfigPool):
81 def __init__(self):
82 self.name = None
83 self.max_cores = None
84 self.max_ram = None
85 self.ignore_provider_quota = False
86 self.availability_zone = None
87 self.subnet_id = None
88 self.security_group_id = None
89 self.host_key_checking = True
90 self.labels = None
91 # The ProviderConfig object that owns this pool.
92 self.provider = None
93
94 # Initialize base class attributes
95 super().__init__()
96
97 def load(self, pool_config, full_config, provider):
98 super().load(pool_config)
99 self.name = pool_config['name']
100 self.provider = provider
101
102 self.max_cores = pool_config.get('max-cores', math.inf)
103 self.max_ram = pool_config.get('max-ram', math.inf)
104 self.ignore_provider_quota = pool_config.get(
105 'ignore-provider-quota', False)
106 self.availability_zone = pool_config.get('availability-zone')
107 self.security_group_id = pool_config.get('security-group-id')
108 self.subnet_id = pool_config.get('subnet-id')
109 self.host_key_checking = bool(
110 pool_config.get('host-key-checking', True))
111
112 for label in pool_config.get('labels', []):
113 pl = ProviderLabel()
114 pl.name = label['name']
115 pl.pool = self
116 self.labels[pl.name] = pl
117 cloud_image_name = label.get('cloud-image', None)
118 if cloud_image_name:
119 cloud_image = self.provider.cloud_images.get(
120 cloud_image_name, None)
121 if not cloud_image:
122 raise ValueError(
123 "cloud-image %s does not exist in provider %s"
124 " but is referenced in label %s" %
125 (cloud_image_name, self.name, pl.name))
126 else:
127 cloud_image = None
128 pl.cloud_image = cloud_image
129 pl.flavor_name = label['flavor-name']
130 pl.key_name = label['key-name']
131 pl.volume_type = label.get('volume-type')
132 pl.volume_size = label.get('volume-size')
133 full_config.labels[label['name']].pools.append(self)
134
135 def __eq__(self, other):
136 if isinstance(other, ProviderPool):
137 # NOTE(Shrews): We intentionally do not compare 'provider' here
138 # since this causes recursive checks with OpenStackProviderConfig.
139 return (super().__eq__(other)
140 and other.name == self.name
141 and other.max_cores == self.max_cores
142 and other.max_ram == self.max_ram
143 and other.ignore_provider_quota == (
144 self.ignore_provider_quota)
145 and other.availability_zone == self.availability_zone
146 and other.subnet_id == self.subnet_id
147 and other.security_group_id == self.security_group_id
148 and other.host_key_checking == self.host_key_checking
149 and other.labels == self.labels)
150 return False
151
152 def __repr__(self):
153 return "<ProviderPool %s>" % self.name
154
155
156class AwsProviderConfig(ProviderConfig):
157 def __init__(self, driver, provider):
158 self.driver_object = driver
159 self.__pools = {}
160 self.profile_name = None
161 self.region_name = None
162 self.rate = None
163 self.boot_timeout = None
164 self.launch_retries = None
165 self.launch_timeout = None
166 self.cloud_images = {}
167 self.hostname_format = None
168 self.image_name_format = None
169 super().__init__(provider)
170
171 def __eq__(self, other):
172 if isinstance(other, AwsProviderConfig):
173 return (super().__eq__(other)
174 and other.profile_name == self.profile_name
175 and other.region_name == self.region_name
176 and other.pools == self.pools
177 and other.rate == self.rate
178 and other.boot_timeout == self.boot_timeout
179 and other.launch_retries == self.launch_retries
180 and other.launch_timeout == self.launch_timeout
181 and other.cloud_images == self.cloud_images)
182 return False
183
184 @property
185 def pools(self):
186 return self.__pools
187
188 @property
189 def manage_images(self):
190 return True
191
192 @staticmethod
193 def reset():
194 pass
195
196 def load(self, config):
197 self.profile_name = self.provider.get('profile-name')
198 self.region_name = self.provider.get('region-name')
199 self.rate = float(self.provider.get('rate', 1.0))
200 self.boot_timeout = self.provider.get('boot-timeout', 60)
201 self.launch_retries = self.provider.get('launch-retries', 3)
202 self.launch_timeout = self.provider.get('launch-timeout', 3600)
203 self.hostname_format = self.provider.get(
204 'hostname-format',
205 '{label.name}-{provider.name}-{node.id}'
206 )
207 self.image_name_format = self.provider.get(
208 'image-name-format',
209 '{image_name}-{timestamp}'
210 )
211
212 default_port_mapping = {
213 'ssh': 22,
214 'winrm': 5986,
215 }
216 # TODO: diskimages
217
218 for image in self.provider.get('cloud-images', []):
219 i = ProviderCloudImage()
220 i.name = image['name']
221 i.image_id = image.get('image-id', None)
222 i.image_name = image.get('image-name', None)
223 i.username = image.get('username', None)
224 i.connection_type = image.get('connection-type', 'ssh')
225 i.connection_port = image.get(
226 'connection-port',
227 default_port_mapping.get(i.connection_type, 22))
228 self.cloud_images[i.name] = i
229
230 for pool in self.provider.get('pools', []):
231 pp = ProviderPool()
232 pp.load(pool, config, self)
233 self.pools[pp.name] = pp
234
235 def getSchema(self):
236 pool_label = {
237 v.Required('name'): str,
238 v.Exclusive('cloud-image', 'label-image'): str,
239 v.Required('flavor-name'): str,
240 v.Required('key-name'): str,
241 'volume-type': str,
242 'volume-size': int
243 }
244
245 pool = ConfigPool.getCommonSchemaDict()
246 pool.update({
247 v.Required('name'): str,
248 v.Required('labels'): [pool_label],
249 'max-cores': int,
250 'max-ram': int,
251 'availability-zone': str,
252 'security-group-id': str,
253 'subnet-id': str,
254 })
255
256 provider_cloud_images = {
257 'name': str,
258 'connection-type': str,
259 'connection-port': int,
260 v.Exclusive('image-id', 'cloud-image-name-or-id'): str,
261 v.Exclusive('image-name', 'cloud-image-name-or-id'): str,
262 'username': str,
263 }
264
265 provider = ProviderConfig.getCommonSchemaDict()
266 provider.update({
267 v.Required('pools'): [pool],
268 v.Required('region-name'): str,
269 'profile-name': str,
270 'cloud-images': [provider_cloud_images],
271 'rate': v.Coerce(float),
272 'hostname-format': str,
273 'image-name-format': str,
274 'boot-timeout': int,
275 'launch-timeout': int,
276 'launch-retries': int,
277 })
278 return v.Schema(provider)
279
280 def getSupportedLabels(self, pool_name=None):
281 labels = set()
282 for pool in self.pools.values():
283 if not pool_name or (pool.name == pool_name):
284 labels.update(pool.labels.keys())
285 return labels
diff --git a/nodepool/driver/aws/handler.py b/nodepool/driver/aws/handler.py
new file mode 100644
index 0000000..c18f2f6
--- /dev/null
+++ b/nodepool/driver/aws/handler.py
@@ -0,0 +1,157 @@
1# Copyright 2018 Red Hat
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import logging
16import time
17
18from nodepool import exceptions
19from nodepool import zk
20from nodepool.driver.utils import NodeLauncher
21from nodepool.driver import NodeRequestHandler
22from nodepool.nodeutils import nodescan
23
24
25class AwsInstanceLauncher(NodeLauncher):
26 def __init__(self, handler, node, provider_config, provider_label):
27 super().__init__(handler.zk, node, provider_config)
28 self.retries = provider_config.launch_retries
29 self.pool = provider_config.pools[provider_label.pool.name]
30 self.handler = handler
31 self.zk = handler.zk
32 self.boot_timeout = provider_config.boot_timeout
33 self.label = provider_label
34
35 def launch(self):
36 self.log.debug("Starting %s instance" % self.node.type)
37 attempts = 1
38 while attempts <= self.retries:
39 try:
40 instance = self.handler.manager.createInstance(self.label)
41 break
42 except Exception:
43 if attempts <= self.retries:
44 self.log.exception(
45 "Launch attempt %d/%d failed for node %s:",
46 attempts, self.retries, self.node.id)
47 if attempts == self.retries:
48 raise
49 attempts += 1
50 time.sleep(1)
51
52 instance.create_tags(Tags=[{'Key': 'nodepool_id',
53 'Value': str(self.node.id)}])
54 instance_id = instance.id
55 self.node.external_id = instance_id
56 self.zk.storeNode(self.node)
57
58 boot_start = time.monotonic()
59 while time.monotonic() - boot_start < self.boot_timeout:
60 state = instance.state.get('Name')
61 self.log.debug("Instance %s is %s" % (instance_id, state))
62 if state == 'running':
63 break
64 time.sleep(0.5)
65 instance.reload()
66 if state != 'running':
67 raise exceptions.LaunchStatusException(
68 "Instance %s failed to start: %s" % (instance_id, state))
69
70 server_ip = instance.public_ip_address
71 if not server_ip:
72 raise exceptions.LaunchStatusException(
73 "Instance %s doesn't have a public ip" % instance_id)
74
75 self.node.connection_port = self.label.cloud_image.connection_port
76 self.node.connection_type = self.label.cloud_image.connection_type
77 if self.pool.host_key_checking:
78 try:
79 if self.node.connection_type == 'ssh':
80 gather_hostkeys = True
81 else:
82 gather_hostkeys = False
83 keys = nodescan(server_ip, port=self.node.connection_port,
84 timeout=180, gather_hostkeys=gather_hostkeys)
85 except Exception:
86 raise exceptions.LaunchKeyscanException(
87 "Can't scan instance %s key" % instance_id)
88
89 self.log.info("Instance %s ready" % instance_id)
90 self.node.state = zk.READY
91 self.node.external_id = instance_id
92 self.node.hostname = server_ip
93 self.node.interface_ip = server_ip
94 self.node.public_ipv4 = server_ip
95 self.node.host_keys = keys
96 self.node.username = self.label.cloud_image.username
97 self.zk.storeNode(self.node)
98 self.log.info("Instance %s is ready", instance_id)
99
100
101class AwsNodeRequestHandler(NodeRequestHandler):
102 log = logging.getLogger("nodepool.driver.aws."
103 "AwsNodeRequestHandler")
104
105 def __init__(self, pw, request):
106 super().__init__(pw, request)
107 self._threads = []
108
109 @property
110 def alive_thread_count(self):
111 count = 0
112 for t in self._threads:
113 if t.isAlive():
114 count += 1
115 return count
116
117 def imagesAvailable(self):
118 '''
119 Determines if the requested images are available for this provider.
120
121 :returns: True if it is available, False otherwise.
122 '''
123 if self.provider.manage_images:
124 for label in self.request.node_types:
125 if self.pool.labels[label].cloud_image:
126 if not self.manager.labelReady(self.pool.labels[label]):
127 return False
128 return True
129
130 def launchesComplete(self):
131 '''
132 Check if all launch requests have completed.
133
134 When all of the Node objects have reached a final state (READY or
135 FAILED), we'll know all threads have finished the launch process.
136 '''
137 if not self._threads:
138 return True
139
140 # Give the NodeLaunch threads time to finish.
141 if self.alive_thread_count:
142 return False
143
144 node_states = [node.state for node in self.nodeset]
145
146 # NOTE: It very important that NodeLauncher always sets one of
147 # these states, no matter what.
148 if not all(s in (zk.READY, zk.FAILED) for s in node_states):
149 return False
150
151 return True
152
153 def launch(self, node):
154 label = self.pool.labels[node.type[0]]
155 thd = AwsInstanceLauncher(self, node, self.provider, label)
156 thd.start()
157 self._threads.append(thd)
diff --git a/nodepool/driver/aws/provider.py b/nodepool/driver/aws/provider.py
new file mode 100644
index 0000000..941508c
--- /dev/null
+++ b/nodepool/driver/aws/provider.py
@@ -0,0 +1,151 @@
1# Copyright 2018 Red Hat
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import logging
16import boto3
17
18from nodepool.driver import Provider
19from nodepool.driver.aws.handler import AwsNodeRequestHandler
20
21
22class AwsInstance:
23 def __init__(self, name, metadatas, provider):
24 self.id = name
25 self.name = name
26 self.metadata = {}
27 if metadatas:
28 for metadata in metadatas:
29 if metadata["Key"] == "nodepool_id":
30 self.metadata = {
31 'nodepool_provider_name': provider.name,
32 'nodepool_node_id': metadata["Value"],
33 }
34 break
35
36 def get(self, name, default=None):
37 return getattr(self, name, default)
38
39
40class AwsProvider(Provider):
41 log = logging.getLogger("nodepool.driver.aws.AwsProvider")
42
43 def __init__(self, provider, *args):
44 self.provider = provider
45 self.ec2 = None
46
47 def getRequestHandler(self, poolworker, request):
48 return AwsNodeRequestHandler(poolworker, request)
49
50 def start(self, zk_conn):
51 if self.ec2 is not None:
52 return True
53 self.log.debug("Starting")
54 self.aws = boto3.Session(
55 region_name=self.provider.region_name,
56 profile_name=self.provider.profile_name)
57 self.ec2 = self.aws.resource('ec2')
58
59 def stop(self):
60 self.log.debug("Stopping")
61
62 def listNodes(self):
63 servers = []
64
65 for instance in self.ec2.instances.all():
66 if instance.state["Name"].lower() == "terminated":
67 continue
68 servers.append(AwsInstance(
69 instance.id, instance.tags, self.provider))
70 return servers
71
72 def getImage(self, image_id):
73 return self.ec2.Image(image_id)
74
75 def labelReady(self, label):
76 if not label.cloud_image:
77 msg = "A cloud-image (AMI) must be supplied with the AWS driver."
78 raise Exception(msg)
79
80 image = self.getImage(label.cloud_image.external_name)
81 # Image loading is deferred, check if it's really there
82 if image.state != 'available':
83 self.log.warning(
84 "Provider %s is configured to use %s as the AMI for"
85 " label %s and that AMI is there but unavailable in the"
86 " cloud." % (self.provider.name,
87 label.cloud_image.external_name,
88 label.name))
89 return False
90 return True
91
92 def join(self):
93 return True
94
95 def cleanupLeakedResources(self):
96 # TODO: remove leaked resources if any
97 pass
98
99 def cleanupNode(self, server_id):
100 if self.ec2 is None:
101 return False
102 instance = self.ec2.Instance(server_id)
103 instance.terminate()
104
105 def waitForNodeCleanup(self, server_id):
106 # TODO: track instance deletion
107 return True
108
109 def createInstance(self, label):
110 image_name = label.cloud_image.external_name
111 args = dict(
112 ImageId=image_name,
113 MinCount=1,
114 MaxCount=1,
115 KeyName=label.key_name,
116 InstanceType=label.flavor_name,
117 NetworkInterfaces=[{
118 'AssociatePublicIpAddress': True,
119 'DeviceIndex': 0}])
120
121 if label.pool.security_group_id:
122 args['NetworkInterfaces'][0]['Groups'] = [
123 label.pool.security_group_id
124 ]
125 if label.pool.subnet_id:
126 args['NetworkInterfaces'][0]['SubnetId'] = label.pool.subnet_id
127
128 # Default block device mapping parameters are embedded in AMIs.
129 # We might need to supply our own mapping before lauching the instance.
130 # We basically want to make sure DeleteOnTermination is true and be
131 # able to set the volume type and size.
132 image = self.getImage(image_name)
133 # TODO: Flavors can also influence whether or not the VM spawns with a
134 # volume -- we basically need to ensure DeleteOnTermination is true
135 if hasattr(image, 'block_device_mappings'):
136 bdm = image.block_device_mappings
137 mapping = bdm[0]
138 if 'Ebs' in mapping:
139 mapping['Ebs']['DeleteOnTermination'] = True
140 if label.volume_size:
141 mapping['Ebs']['VolumeSize'] = label.volume_size
142 if label.volume_type:
143 mapping['Ebs']['VolumeType'] = label.volume_type
144 # If the AMI is a snapshot, we cannot supply an "encrypted"
145 # parameter
146 if 'Encrypted' in mapping['Ebs']:
147 del mapping['Ebs']['Encrypted']
148 args['BlockDeviceMappings'] = [mapping]
149
150 instances = self.ec2.create_instances(**args)
151 return self.ec2.Instance(instances[0].id)
diff --git a/nodepool/tests/__init__.py b/nodepool/tests/__init__.py
index 6cf67ff..bce28cd 100644
--- a/nodepool/tests/__init__.py
+++ b/nodepool/tests/__init__.py
@@ -334,19 +334,22 @@ class DBTestCase(BaseTestCase):
334 self.useFixture(images_dir) 334 self.useFixture(images_dir)
335 build_log_dir = fixtures.TempDir() 335 build_log_dir = fixtures.TempDir()
336 self.useFixture(build_log_dir) 336 self.useFixture(build_log_dir)
337 configfile = os.path.join(os.path.dirname(__file__), 337 if filename.startswith('/'):
338 'fixtures', filename) 338 path = filename
339 (fd, path) = tempfile.mkstemp() 339 else:
340 with open(configfile, 'rb') as conf_fd: 340 configfile = os.path.join(os.path.dirname(__file__),
341 config = conf_fd.read().decode('utf8') 341 'fixtures', filename)
342 data = config.format(images_dir=images_dir.path, 342 (fd, path) = tempfile.mkstemp()
343 build_log_dir=build_log_dir.path, 343 with open(configfile, 'rb') as conf_fd:
344 context_name=context_name, 344 config = conf_fd.read().decode('utf8')
345 zookeeper_host=self.zookeeper_host, 345 data = config.format(images_dir=images_dir.path,
346 zookeeper_port=self.zookeeper_port, 346 build_log_dir=build_log_dir.path,
347 zookeeper_chroot=self.zookeeper_chroot) 347 context_name=context_name,
348 os.write(fd, data.encode('utf8')) 348 zookeeper_host=self.zookeeper_host,
349 os.close(fd) 349 zookeeper_port=self.zookeeper_port,
350 zookeeper_chroot=self.zookeeper_chroot)
351 os.write(fd, data.encode('utf8'))
352 os.close(fd)
350 self._config_images_dir = images_dir 353 self._config_images_dir = images_dir
351 self._config_build_log_dir = build_log_dir 354 self._config_build_log_dir = build_log_dir
352 validator = ConfigValidator(path) 355 validator = ConfigValidator(path)
diff --git a/nodepool/tests/fixtures/aws.yaml b/nodepool/tests/fixtures/aws.yaml
new file mode 100644
index 0000000..7a5fc7a
--- /dev/null
+++ b/nodepool/tests/fixtures/aws.yaml
@@ -0,0 +1,26 @@
1zookeeper-servers:
2 - host: null
3 port: null
4 chroot: null
5
6labels:
7 - name: ubuntu1404
8
9providers:
10 - name: ec2-us-west-2
11 driver: aws
12 region-name: us-west-2
13 cloud-images:
14 - name: ubuntu1404
15 image-id: ami-1e749f67
16 username: ubuntu
17 pools:
18 - name: main
19 max-servers: 5
20 subnet-id: null
21 security-group-id: null
22 labels:
23 - name: ubuntu1404
24 cloud-image: ubuntu1404
25 flavor-name: t3.medium
26 key-name: zuul
diff --git a/nodepool/tests/fixtures/config_validate/good.yaml b/nodepool/tests/fixtures/config_validate/good.yaml
index 9aba134..3bb327b 100644
--- a/nodepool/tests/fixtures/config_validate/good.yaml
+++ b/nodepool/tests/fixtures/config_validate/good.yaml
@@ -23,6 +23,7 @@ labels:
23 - name: pod-fedora 23 - name: pod-fedora
24 - name: openshift-project 24 - name: openshift-project
25 - name: openshift-pod 25 - name: openshift-pod
26 - name: centos-ami
26 27
27providers: 28providers:
28 - name: cloud1 29 - name: cloud1
@@ -138,6 +139,27 @@ providers:
138 memory: 512 139 memory: 512
139 cpu: 2 140 cpu: 2
140 141
142 - name: ec2-us-east-2
143 driver: aws
144 region-name: us-east-2
145 profile-name: default
146 cloud-images:
147 - name: centos-ami
148 image-id: ami-cfdafaaa
149 username: centos
150 pools:
151 - name: main
152 max-servers: 42
153 security-group-id: sg-8bfe86352e334a80a
154 subnet-id: subnet-bb3605b5f0fa40e1b
155 labels:
156 - name: centos-ami
157 cloud-image: centos-ami
158 flavor-name: t2.micro
159 key-name: zuul
160 volume-type: gp2
161 volume-size: 80
162
141diskimages: 163diskimages:
142 - name: trusty 164 - name: trusty
143 formats: 165 formats:
diff --git a/nodepool/tests/unit/test_driver_aws.py b/nodepool/tests/unit/test_driver_aws.py
new file mode 100644
index 0000000..835fb0a
--- /dev/null
+++ b/nodepool/tests/unit/test_driver_aws.py
@@ -0,0 +1,95 @@
1# Copyright (C) 2018 Red Hat
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import fixtures
17import logging
18import os
19import tempfile
20from unittest.mock import patch
21
22import boto3
23from moto import mock_ec2
24import yaml
25
26from nodepool import tests
27from nodepool import zk
28
29
30class TestDriverAws(tests.DBTestCase):
31 log = logging.getLogger("nodepool.TestDriverAws")
32
33 @mock_ec2
34 def test_ec2_machine(self):
35 aws_id = 'AK000000000000000000'
36 aws_key = '0123456789abcdef0123456789abcdef0123456789abcdef'
37 self.useFixture(
38 fixtures.EnvironmentVariable('AWS_ACCESS_KEY_ID', aws_id))
39 self.useFixture(
40 fixtures.EnvironmentVariable('AWS_SECRET_ACCESS_KEY', aws_key))
41
42 ec2 = boto3.client('ec2', region_name='us-west-2')
43
44 # TEST-NET-3
45 vpc = ec2.create_vpc(CidrBlock='203.0.113.0/24')
46
47 subnet = ec2.create_subnet(
48 CidrBlock='203.0.113.128/25', VpcId=vpc['Vpc']['VpcId'])
49 subnet_id = subnet['Subnet']['SubnetId']
50 sg = ec2.create_security_group(
51 GroupName='zuul-nodes', VpcId=vpc['Vpc']['VpcId'],
52 Description='Zuul Nodes')
53 sg_id = sg['GroupId']
54
55 ec2_template = os.path.join(
56 os.path.dirname(__file__), '..', 'fixtures', 'aws.yaml')
57 raw_config = yaml.safe_load(open(ec2_template))
58 raw_config['zookeeper-servers'][0] = {
59 'host': self.zookeeper_host,
60 'port': self.zookeeper_port,
61 'chroot': self.zookeeper_chroot,
62 }
63 raw_config['providers'][0]['pools'][0]['subnet-id'] = subnet_id
64 raw_config['providers'][0]['pools'][0]['security-group-id'] = sg_id
65 with tempfile.NamedTemporaryFile() as tf:
66 tf.write(yaml.safe_dump(
67 raw_config, default_flow_style=False).encode('utf-8'))
68 tf.flush()
69 configfile = self.setup_config(tf.name)
70 pool = self.useNodepool(configfile, watermark_sleep=1)
71 pool.start()
72 req = zk.NodeRequest()
73 req.state = zk.REQUESTED
74 req.node_types.append('ubuntu1404')
75 with patch('nodepool.driver.aws.handler.nodescan') as nodescan:
76 nodescan.return_value = 'MOCK KEY'
77 self.zk.storeNodeRequest(req)
78
79 self.log.debug("Waiting for request %s", req.id)
80 req = self.waitForNodeRequest(req)
81 self.assertEqual(req.state, zk.FULFILLED)
82
83 self.assertNotEqual(req.nodes, [])
84 node = self.zk.getNode(req.nodes[0])
85 self.assertEqual(node.allocated_to, req.id)
86 self.assertEqual(node.state, zk.READY)
87 self.assertIsNotNone(node.launcher)
88 self.assertEqual(node.connection_type, 'ssh')
89 nodescan.assert_called_with(
90 node.interface_ip, port=22, timeout=180, gather_hostkeys=True)
91
92 node.state = zk.DELETING
93 self.zk.storeNode(node)
94
95 self.waitForNodeDeletion(node)
diff --git a/releasenotes/notes/aws-driver-6d6c25381066b9ca.yaml b/releasenotes/notes/aws-driver-6d6c25381066b9ca.yaml
new file mode 100644
index 0000000..556daf4
--- /dev/null
+++ b/releasenotes/notes/aws-driver-6d6c25381066b9ca.yaml
@@ -0,0 +1,7 @@
1---
2prelude: Amazon Web Services (AWS) EC2 Driver
3features:
4 - The new Amazon Web Services (AWS) EC2 Driver allows launching EC2 instances as nodes.
5issues:
6 - The AWS driver does not support quota management at this time.
7 - The AWS driver does not support custom image building.
diff --git a/requirements.txt b/requirements.txt
index f310349..a9d6678 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,3 +14,4 @@ kazoo
14Paste 14Paste
15WebOb>=1.8.1 15WebOb>=1.8.1
16openshift 16openshift
17boto3
diff --git a/test-requirements.txt b/test-requirements.txt
index 18e1097..0b7c380 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -7,3 +7,4 @@ python-subunit
7stestr>=1.0.0 # Apache-2.0 7stestr>=1.0.0 # Apache-2.0
8testscenarios 8testscenarios
9testtools>=0.9.27 9testtools>=0.9.27
10moto