summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSimon Westphahl <simon.westphahl@bmw.de>2019-02-06 08:57:42 +0100
committerSimon Westphahl <simon.westphahl@bmw.de>2019-02-06 10:25:56 +0100
commit6e424878ae0669adb2b2950c4e0e06c9c46daaec (patch)
treea6904490db4f7f167c5b3d905fdfe494cecaa7ce
parent85c70008d6a059c82e1bcd28053b392fb39aa6f8 (diff)
Fix error reporting for special task failures
For some tasks the Ansible log will not contain enough information to debug failures (e.g. missing role with include_role). Ansible treats those issues not like an error (exit code 1) but like a failed task, leading to an exit code of 2. Change-Id: Iea754814e3d55be6be1c2de7f2d45ceda757f480
Notes
Notes (review): Code-Review+2: Tobias Henkel <tobias.henkel@bmw.de> Code-Review+2: Monty Taylor <mordred@inaugust.com> Workflow+1: Monty Taylor <mordred@inaugust.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Wed, 06 Feb 2019 13:28:32 +0000 Reviewed-on: https://review.openstack.org/635110 Project: openstack-infra/zuul Branch: refs/heads/master
-rw-r--r--tests/fixtures/config/job-output/git/common-config/playbooks/job-output-missing-role-include.yaml4
-rw-r--r--tests/fixtures/config/job-output/git/common-config/zuul.yaml5
-rw-r--r--tests/unit/test_v3.py10
-rw-r--r--zuul/executor/server.py29
4 files changed, 37 insertions, 11 deletions
diff --git a/tests/fixtures/config/job-output/git/common-config/playbooks/job-output-missing-role-include.yaml b/tests/fixtures/config/job-output/git/common-config/playbooks/job-output-missing-role-include.yaml
new file mode 100644
index 0000000..e8e7f28
--- /dev/null
+++ b/tests/fixtures/config/job-output/git/common-config/playbooks/job-output-missing-role-include.yaml
@@ -0,0 +1,4 @@
1- hosts: all
2 tasks:
3 - include_role:
4 name: not_existing
diff --git a/tests/fixtures/config/job-output/git/common-config/zuul.yaml b/tests/fixtures/config/job-output/git/common-config/zuul.yaml
index 5b44347..a0122a6 100644
--- a/tests/fixtures/config/job-output/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/job-output/git/common-config/zuul.yaml
@@ -30,6 +30,10 @@
30 name: job-output-missing-role 30 name: job-output-missing-role
31 run: playbooks/job-output-missing-role.yaml 31 run: playbooks/job-output-missing-role.yaml
32 32
33- job:
34 name: job-output-missing-role-include
35 run: playbooks/job-output-missing-role-include.yaml
36
33- project: 37- project:
34 name: org/project 38 name: org/project
35 check: 39 check:
@@ -47,3 +51,4 @@
47 check: 51 check:
48 jobs: 52 jobs:
49 - job-output-missing-role 53 - job-output-missing-role
54 - job-output-missing-role-include
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index b9955d2..00875eb 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -4360,11 +4360,15 @@ class TestJobOutput(AnsibleZuulTestCase):
4360 self.assertHistory([ 4360 self.assertHistory([
4361 dict(name='job-output-missing-role', result='FAILURE', 4361 dict(name='job-output-missing-role', result='FAILURE',
4362 changes='1,1'), 4362 changes='1,1'),
4363 dict(name='job-output-missing-role-include', result='FAILURE',
4364 changes='1,1'),
4363 ], ordered=False) 4365 ], ordered=False)
4364 4366
4365 job_output = self._get_file(self.history[0], 4367 for history in self.history:
4366 'work/logs/job-output.txt') 4368 job_output = self._get_file(history,
4367 self.assertIn('the role \'not_existing\' was not found', job_output) 4369 'work/logs/job-output.txt')
4370 self.assertIn('the role \'not_existing\' was not found',
4371 job_output)
4368 4372
4369 def test_job_output_failure_log(self): 4373 def test_job_output_failure_log(self):
4370 logger = logging.getLogger('zuul.AnsibleJob') 4374 logger = logging.getLogger('zuul.AnsibleJob')
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 05c8729..678919c 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -1948,14 +1948,27 @@ class AnsibleJob(object):
1948 now=datetime.datetime.now(), 1948 now=datetime.datetime.now(),
1949 line=line.decode('utf-8').rstrip())) 1949 line=line.decode('utf-8').rstrip()))
1950 elif ret == 2: 1950 elif ret == 2:
1951 # This is a workaround to detect winrm connection failures that are 1951 with open(self.jobdir.job_output_file, 'a') as job_output:
1952 # not detected by ansible. These can be detected if the string 1952 found_marker = False
1953 # 'FATAL ERROR DURING FILE TRANSFER' is in the ansible output. 1953 for line in syntax_buffer:
1954 # In this case we should treat the host as unreachable and retry 1954 # This is a workaround to detect winrm connection failures
1955 # the job. 1955 # that are not detected by ansible. These can be detected
1956 for line in syntax_buffer: 1956 # if the string 'FATAL ERROR DURING FILE TRANSFER' is in
1957 if b'FATAL ERROR DURING FILE TRANSFER' in line: 1957 # the ansible output. In this case we should treat the
1958 return self.RESULT_UNREACHABLE, None 1958 # host as unreachable and retry the job.
1959 if b'FATAL ERROR DURING FILE TRANSFER' in line:
1960 return self.RESULT_UNREACHABLE, None
1961
1962 # Extract errors for special cases that are treated like
1963 # task errors by Ansible (e.g. missing role when using
1964 # 'include_role').
1965 if line.startswith(b'ERROR!'):
1966 found_marker = True
1967 if not found_marker:
1968 continue
1969 job_output.write("{now} | {line}\n".format(
1970 now=datetime.datetime.now(),
1971 line=line.decode('utf-8').rstrip()))
1959 1972
1960 return (self.RESULT_NORMAL, ret) 1973 return (self.RESULT_NORMAL, ret)
1961 1974