Add config validation option

Add a configuration validation command-line option.  This will parse
the yaml file -- sufficient to find many indenting errors -- and then
run the file through a schema check.  There is one example of checking
image/provider mappings; more such validation can be added.

Test-cases are updated

The intent is to use this within a gate-job for system-config to avoid
broken configs being committed.

Change-Id: I250ea4959740cfc4650e9057caba09ae7bc25768

nodepool/cmd/config_validator.py

@@ -0,0 +1,126 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import voluptuous as v
+import yaml
+
+log = logging.getLogger(__name__)
+
+
+class ConfigValidator:
+    """Check the layout and certain configuration options"""
+
+    def __init__(self, config_file):
+        self.config_file = config_file
+
+    def validate(self):
+        cron = {
+            'check': str,
+            'cleanup': str,
+            'image-update': str,
+        }
+
+        images = {
+            'name': str,
+            'base-image': str,
+            'min-ram': int,
+            'name-filter': str,
+            'diskimage': str,
+            'meta': dict,
+            'setup': str,
+            'username': str,
+            'private-key': str,
+        }
+
+        providers = {
+            'name': str,
+            'region-name': str,
+            'service-type': str,
+            'service-name': str,
+            'availability-zones': [str],
+            'keypair': str,
+            'username': str,
+            'password': str,
+            'auth-url': str,
+            'project-id': str,
+            'max-servers': int,
+            'pool': str,
+            'networks': [{
+                'net-id': str,
+                'net-label': str,
+            }],
+            'boot-timeout': int,
+            'api-timeout': int,
+            'rate': float,
+            'images': [images]
+        }
+
+        labels = {
+            'name': str,
+            'image': str,
+            'min-ready': int,
+            'ready-script': str,
+            'subnodes': int,
+            'providers': [{
+                'name': str,
+            }],
+        }
+
+        targets = {
+            'name': str,
+            'jenkins': {
+                'url': str,
+                'user': str,
+                'apikey': str,
+                'credentials-id': str,
+            }
+        }
+
+        diskimages = {
+            'name': str,
+            'elements': [str],
+            'release': v.Any(str, int),
+            'env-vars': dict,
+        }
+
+        top_level = {
+            'script-dir': str,
+            'elements-dir': str,
+            'images-dir': str,
+            'dburi': str,
+            'zmq-publishers': [str],
+            'gearman-servers': [{
+                'host': str
+            }],
+            'cron': cron,
+            'providers': [providers],
+            'labels': [labels],
+            'targets': [targets],
+            'diskimages': [diskimages],
+        }
+
+        log.info("validating %s" % self.config_file)
+        config = yaml.load(open(self.config_file))
+
+        # validate the overall schema
+        schema = v.Schema(top_level)
+        schema(config)
+
+        # labels must list valid providers
+        all_providers = [p['name'] for p in config['providers']]
+        for label in config['labels']:
+            for provider in label['providers']:
+                if not provider['name'] in all_providers:
+                    raise AssertionError('label %s requests '
+                                         'non-existent provider %s'
+                                         % (label['name'], provider['name']))

nodepool/cmd/nodepoolcmd.py

@@ -22,8 +22,11 @@ import time
 from nodepool import nodedb
 from nodepool import nodepool
 from nodepool.version import version_info as npc_version_info
+from config_validator import ConfigValidator
 from prettytable import PrettyTable
 
+log = logging.getLogger(__name__)
+
 
 class NodePoolCmd(object):
     def __init__(self):
@@ -42,6 +45,7 @@ class NodePoolCmd(object):
 
         subparsers = parser.add_subparsers(title='commands',
                                            description='valid commands',
+                                           dest='command',
                                            help='additional help')
 
         cmd_list = subparsers.add_parser('list', help='list nodes')
@@ -121,6 +125,11 @@ class NodePoolCmd(object):
             nargs='?', default='all')
         cmd_image_upload.add_argument('image', help='image name')
 
+        cmd_config_validate = subparsers.add_parser(
+            'config-validate',
+            help='Validate configuration file')
+        cmd_config_validate.set_defaults(func=self.config_validate)
+
         self.args = parser.parse_args()
 
     def setup_logging(self):
@@ -311,7 +320,16 @@ class NodePoolCmd(object):
         self.pool.reconfigureManagers(self.pool.config, False)
         self.pool.deleteImage(self.args.id)
 
+    def config_validate(self):
+        validator = ConfigValidator(self.args.config)
+        validator.validate()
+        log.info("Configuation validation complete")
+
     def main(self):
+        # commands which do not need to start-up or parse config
+        if self.args.command in ('config-validate'):
+            return self.args.func()
+
         self.pool = nodepool.NodePool(self.args.config)
         config = self.pool.loadConfig()
         self.pool.reconfigureDatabase(config)

nodepool/tests/fixtures/config_validate/schema_error.yaml

@@ -0,0 +1,6 @@
+script-dir: /etc/nodepool/scripts
+elements-dir: /etc/nodepool/elements
+images-dir: /opt/nodepool_dib
+dburi: 'mysql://nodepool:<%= mysql_password %>@localhost/nodepool'
+
+nothandled: element

nodepool/tests/test_config_validator.py

@@ -0,0 +1,48 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from nodepool.cmd.config_validator import ConfigValidator
+
+from nodepool import tests
+from yaml.parser import ParserError
+from voluptuous import MultipleInvalid
+
+
+class TestConfigValidation(tests.BaseTestCase):
+
+    def setUp(self):
+        super(TestConfigValidation, self).setUp()
+
+    def test_good_config(self):
+        config = os.path.join(os.path.dirname(tests.__file__),
+                              'fixtures', 'config_validate', 'good.yaml')
+
+        validator = ConfigValidator(config)
+        validator.validate()
+
+    def test_yaml_error(self):
+        config = os.path.join(os.path.dirname(tests.__file__),
+                              'fixtures', 'config_validate', 'yaml_error.yaml')
+
+        validator = ConfigValidator(config)
+        self.assertRaises(ParserError, validator.validate)
+
+    def test_schema(self):
+        config = os.path.join(os.path.dirname(tests.__file__),
+                              'fixtures', 'config_validate',
+                              'schema_error.yaml')
+
+        validator = ConfigValidator(config)
+        self.assertRaises(MultipleInvalid, validator.validate)

requirements.txt

@@ -17,3 +17,4 @@ MySQL-python
 PrettyTable>=0.6,<0.8
 six>=1.7.0
 diskimage-builder
+voluptuous