summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2018-11-19 19:34:53 +0000
committerGerrit Code Review <review@openstack.org>2018-11-19 19:34:53 +0000
commit54f1a9f7ebcc097e8a095fb62d46e22f335e71af (patch)
tree53351a1c8be9f5d3527b8c4d2376707ef7bce925
parent125eeed6fe76b79d9ca63e720941137f539659fa (diff)
parent7e00ba32dab04ac6a3f952d84dfd9eef1e021e47 (diff)
Merge "validate-host: retry network tests and include unbound logs"
-rw-r--r--roles/validate-host/library/zuul_debug_info.py34
-rw-r--r--roles/validate-host/library/zuul_network_validate.py118
-rw-r--r--roles/validate-host/tasks/main.yaml9
-rw-r--r--roles/validate-host/templates/zuul-info.j224
4 files changed, 139 insertions, 46 deletions
diff --git a/roles/validate-host/library/zuul_debug_info.py b/roles/validate-host/library/zuul_debug_info.py
index 8fc85d3..e3db74b 100644
--- a/roles/validate-host/library/zuul_debug_info.py
+++ b/roles/validate-host/library/zuul_debug_info.py
@@ -18,15 +18,10 @@
18import os 18import os
19import shlex 19import shlex
20import subprocess 20import subprocess
21import traceback
22 21
23 22
24command_map = { 23command_map = {
25 'uname': 'uname -a', 24 'uname': 'uname -a',
26 'network_interfaces': 'ip address show',
27 'network_routing_v4': 'ip route show',
28 'network_routing_v6': 'ip -6 route show',
29 'network_neighbors': 'ip neighbor show',
30} 25}
31 26
32 27
@@ -44,16 +39,14 @@ def main():
44 argument_spec=dict( 39 argument_spec=dict(
45 image_manifest=dict(required=False, type='str'), 40 image_manifest=dict(required=False, type='str'),
46 image_manifest_files=dict(required=False, type='list'), 41 image_manifest_files=dict(required=False, type='list'),
47 traceroute_host=dict(required=False, type='str'),
48 ) 42 )
49 ) 43 )
50 44
51 image_manifest = module.params['image_manifest'] 45 image_manifest = module.params['image_manifest']
52 traceroute_host = module.params['traceroute_host']
53 image_manifest_files = module.params['image_manifest_files'] 46 image_manifest_files = module.params['image_manifest_files']
54 if not image_manifest_files and image_manifest: 47 if not image_manifest_files and image_manifest:
55 image_manifest_files = [image_manifest] 48 image_manifest_files = [image_manifest]
56 ret = {'image_manifest_files': [], 'traceroute': None} 49 ret = {'image_manifest_files': []}
57 50
58 for image_manifest in image_manifest_files: 51 for image_manifest in image_manifest_files:
59 if image_manifest and os.path.exists(image_manifest): 52 if image_manifest and os.path.exists(image_manifest):
@@ -63,31 +56,6 @@ def main():
63 'underline': len(image_manifest) * '-', 56 'underline': len(image_manifest) * '-',
64 'content': open(image_manifest, 'r').read(), 57 'content': open(image_manifest, 'r').read(),
65 }) 58 })
66 if traceroute_host:
67 passed = False
68 try:
69 ret['traceroute_v6'] = run_command(
70 'traceroute6 -n {host}'.format(host=traceroute_host))
71 passed = True
72 except (subprocess.CalledProcessError, OSError) as e:
73 ret['traceroute_v6_exception'] = traceback.format_exc(e)
74 ret['traceroute_v6_output'] = e.output
75 ret['traceroute_v6_return'] = e.returncode
76 pass
77 try:
78 ret['traceroute_v4'] = run_command(
79 'traceroute -n {host}'.format(host=traceroute_host))
80 passed = True
81 except (subprocess.CalledProcessError, OSError) as e:
82 ret['traceroute_v4_exception'] = traceback.format_exc(e)
83 ret['traceroute_v4_output'] = e.output
84 ret['traceroute_v4_return'] = e.returncode
85 pass
86 if not passed:
87 module.fail_json(
88 msg="No viable v4 or v6 route found to {traceroute_host}."
89 " The build node is assumed to be invalid.".format(
90 traceroute_host=traceroute_host), **ret)
91 59
92 for key, command in command_map.items(): 60 for key, command in command_map.items():
93 try: 61 try:
diff --git a/roles/validate-host/library/zuul_network_validate.py b/roles/validate-host/library/zuul_network_validate.py
new file mode 100644
index 0000000..5038eb7
--- /dev/null
+++ b/roles/validate-host/library/zuul_network_validate.py
@@ -0,0 +1,118 @@
1#!/usr/bin/python
2
3# Copyright (c) 2018 Red Hat
4#
5# This module is free software: you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation, either version 3 of the License, or
8# (at your option) any later version.
9#
10# This software is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with this software. If not, see <http://www.gnu.org/licenses/>.
17
18import os
19import shlex
20import subprocess
21import traceback
22
23
24command_map = {
25 'network_interfaces': 'ip address show',
26 'network_routing_v4': 'ip route show',
27 'network_routing_v6': 'ip -6 route show',
28 'network_neighbors': 'ip neighbor show',
29}
30
31
32def run_command(command):
33 env = os.environ.copy()
34 env['PATH'] = '{path}:/sbin:/usr/sbin'.format(path=env['PATH'])
35 return subprocess.check_output(
36 shlex.split(command),
37 stderr=subprocess.STDOUT,
38 env=env)
39
40
41def collect_unbound_logs():
42 '''Look for unbound logs
43
44 This looks for unbound logs in common places and returns the
45 contents. Intended for the failure path to add more info if the
46 traceroutes have failed.
47 '''
48 ret = {}
49
50 # NOTE(ianw): keep this one first, the other exists but isn't
51 # populated on infra rpm images for ... reasons
52 if os.path.exists('/var/lib/unbound/unbound.log'):
53 unbound_log_file = '/var/lib/unbound/unbound.log'
54 elif os.path.exists('/var/log/unbound.log'):
55 unbound_log_file = '/var/log/unbound.log'
56 else:
57 return ret
58
59 with open(unbound_log_file) as f:
60 ret['unbound_log_file'] = unbound_log_file
61 # NOTE(ianw): At high verbosity this can be big ... but this
62 # is also intended to be used early which should limit it's
63 # size. We could tail it ...
64 ret['unbound_log_file_content'] = f.read()
65
66 return ret
67
68
69def main():
70 module = AnsibleModule(
71 argument_spec=dict(
72 traceroute_host=dict(required=True, type='str'),
73 )
74 )
75
76 traceroute_host = module.params['traceroute_host']
77
78 ret = {}
79
80 for key, command in command_map.items():
81 try:
82 ret[key] = run_command(command)
83 except subprocess.CalledProcessError:
84 pass
85
86 passed = False
87 try:
88 ret['traceroute_v6'] = run_command(
89 'traceroute6 -n {host}'.format(host=traceroute_host))
90 passed = True
91 except (subprocess.CalledProcessError, OSError) as e:
92 ret['traceroute_v6_exception'] = traceback.format_exc(e)
93 ret['traceroute_v6_output'] = e.output
94 ret['traceroute_v6_return'] = e.returncode
95 pass
96 try:
97 ret['traceroute_v4'] = run_command(
98 'traceroute -n {host}'.format(host=traceroute_host))
99 passed = True
100 except (subprocess.CalledProcessError, OSError) as e:
101 ret['traceroute_v4_exception'] = traceback.format_exc(e)
102 ret['traceroute_v4_output'] = e.output
103 ret['traceroute_v4_return'] = e.returncode
104 pass
105 if not passed:
106 ret.update(collect_unbound_logs())
107 module.fail_json(
108 msg="No viable v4 or v6 route found to {traceroute_host}."
109 " The build node is assumed to be invalid.".format(
110 traceroute_host=traceroute_host), **ret)
111
112 module.exit_json(changed=False, _zuul_nolog_return=True, **ret)
113
114from ansible.module_utils.basic import * # noqa
115from ansible.module_utils.basic import AnsibleModule
116
117if __name__ == '__main__':
118 main()
diff --git a/roles/validate-host/tasks/main.yaml b/roles/validate-host/tasks/main.yaml
index 08cadfc..7f52a1b 100644
--- a/roles/validate-host/tasks/main.yaml
+++ b/roles/validate-host/tasks/main.yaml
@@ -23,9 +23,16 @@
23 zuul_debug_info: 23 zuul_debug_info:
24 image_manifest: "{{ zuul_site_image_manifest|default(omit) }}" 24 image_manifest: "{{ zuul_site_image_manifest|default(omit) }}"
25 image_manifest_files: "{{ zuul_site_image_manifest_files|default(omit) }}" 25 image_manifest_files: "{{ zuul_site_image_manifest_files|default(omit) }}"
26 traceroute_host: "{{ zuul_site_traceroute_host|default(omit) }}"
27 register: zdi 26 register: zdi
28 27
28- name: Collect network information from zuul worker
29 zuul_network_validate:
30 traceroute_host: "{{ zuul_site_traceroute_host|default(omit) }}"
31 register: znetinfo
32 retries: 3
33 delay: 5
34 until: znetinfo.failed == False
35
29- name: Write out all zuul information for each host 36- name: Write out all zuul information for each host
30 delegate_to: localhost 37 delegate_to: localhost
31 template: 38 template:
diff --git a/roles/validate-host/templates/zuul-info.j2 b/roles/validate-host/templates/zuul-info.j2
index eb3dce0..e8d611b 100644
--- a/roles/validate-host/templates/zuul-info.j2
+++ b/roles/validate-host/templates/zuul-info.j2
@@ -15,40 +15,40 @@ Host & kernel
15{{ zdi.uname }} 15{{ zdi.uname }}
16 16
17{% endif %} 17{% endif %}
18{% if 'network_interfaces' in zdi %} 18{% if 'network_interfaces' in znetinfo %}
19Network interface addresses 19Network interface addresses
20=========================== 20===========================
21{{ zdi.network_interfaces }} 21{{ znetinfo.network_interfaces }}
22 22
23{% endif %} 23{% endif %}
24{% if 'network_routing_v4' in zdi %} 24{% if 'network_routing_v4' in znetinfo %}
25Network routing tables v4 25Network routing tables v4
26========================= 26=========================
27{{ zdi.network_routing_v4 }} 27{{ znetinfo.network_routing_v4 }}
28 28
29{% endif %} 29{% endif %}
30{% if 'network_routing_v6' in zdi %} 30{% if 'network_routing_v6' in znetinfo %}
31Network routing tables v6 31Network routing tables v6
32========================= 32=========================
33{{ zdi.network_routing_v6 }} 33{{ znetinfo.network_routing_v6 }}
34 34
35{% endif %} 35{% endif %}
36{% if 'network_neighbors' in zdi %} 36{% if 'network_neighbors' in znetinfo %}
37Network neighbors 37Network neighbors
38================= 38=================
39{{ zdi.network_neighbors }} 39{{ znetinfo.network_neighbors }}
40 40
41{% endif %} 41{% endif %}
42{% if 'traceroute_v4' in zdi %} 42{% if 'traceroute_v4' in znetinfo %}
43Route to Known Host v4 43Route to Known Host v4
44====================== 44======================
45Known Host: {{ zuul_site_traceroute_host }} 45Known Host: {{ zuul_site_traceroute_host }}
46{{ zdi.traceroute_v4 }} 46{{ znetinfo.traceroute_v4 }}
47 47
48{% endif %} 48{% endif %}
49{% if 'traceroute_v6' in zdi %} 49{% if 'traceroute_v6' in znetinfo %}
50Route to Known Host v6 50Route to Known Host v6
51====================== 51======================
52Known Host: {{ zuul_site_traceroute_host }} 52Known Host: {{ zuul_site_traceroute_host }}
53{{ zdi.traceroute_v6 }} 53{{ znetinfo.traceroute_v6 }}
54{% endif %} 54{% endif %}