Fix wedged scheduler on force-merge of non-existing template

If there is already a post pipeline defined a force-merge of a change
referencing a non-existent project template currently can wedge the
scheduler. In this case an exception is raised during reporting which
throws the run handler completely out of its loop and the loop starts
over from the beginning. However the item to be reported is not
consumed in this case so zuul is stuck in an exception loop [1] and
can only be recovered by a restart.

This can be fixed by catching the exception and continuing the
reporting.

[1] Traceback:
2019-01-28 07:33:57,304 ERROR zuul.Scheduler: Exception in run handler:
Traceback (most recent call last):
  File "/opt/zuul/lib/python3.6/site-packages/zuul/scheduler.py", line 1033, in run
    while (pipeline.manager.processQueue() and
  File "/opt/zuul/lib/python3.6/site-packages/zuul/manager/__init__.py", line 768, in processQueue
    item, nnfi)
  File "/opt/zuul/lib/python3.6/site-packages/zuul/manager/__init__.py", line 735, in _processOneItem
    self.reportItem(item)
  File "/opt/zuul/lib/python3.6/site-packages/zuul/manager/__init__.py", line 880, in reportItem
    item.reported = not self._reportItem(item)
  File "/opt/zuul/lib/python3.6/site-packages/zuul/manager/__init__.py", line 920, in _reportItem
    if not layout.getProjectPipelineConfig(item):
  File "/opt/zuul/lib/python3.6/site-packages/zuul/model.py", line 3435, in getProjectPipelineConfig
    templates = self.getProjectTemplates(template_name)
  File "/opt/zuul/lib/python3.6/site-packages/zuul/model.py", line 3374, in getProjectTemplates
    raise TemplateNotFoundError("Project template %s not found" % name)
zuul.model.TemplateNotFoundError: Project template foo not found

Change-Id: I1a3b59dadbd9337a8ba5b146f09ad093a0123ce8
This commit is contained in:
Tobias Henkel 2019-01-28 15:29:24 +01:00
parent b44b6c532c
commit b999b7c862
No known key found for this signature in database
GPG Key ID: 03750DEC158E5FA2
9 changed files with 117 additions and 1 deletions

View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

View File

@ -0,0 +1,35 @@
- pipeline:
name: check
manager: independent
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
Verified: 1
failure:
gerrit:
Verified: -1
- pipeline:
name: post
manager: independent
trigger:
gerrit:
- event: ref-updated
ref: ^(?!refs/).*$
precedence: low
- job:
name: base
parent: null
- job:
name: post-job
run: playbooks/test.yaml
- job:
name: other-job
run: playbooks/test.yaml

View File

@ -0,0 +1 @@
test

View File

@ -0,0 +1,7 @@
- project:
check:
jobs:
- noop
post:
jobs:
- post-job

View File

@ -0,0 +1 @@
test

View File

@ -0,0 +1,4 @@
- project:
check:
jobs:
- other-job

View File

@ -0,0 +1,17 @@
- tenant:
name: tenant-one
source:
gerrit:
config-projects:
- common-config
untrusted-projects:
- org/project
- tenant:
name: tenant-two
source:
gerrit:
config-projects:
- common-config
untrusted-projects:
- org/project2

View File

@ -5002,3 +5002,47 @@ class TestProvidesRequires(ZuulDBTestCase):
])
self.assertIn('image-user : SKIPPED', B.messages[0])
self.assertIn('not met by build', B.messages[0])
class TestForceMergeMissingTemplate(ZuulTestCase):
tenant_config_file = "config/force-merge-template/main.yaml"
def test_force_merge_missing_template(self):
"""
Tests that force merging a change using a non-existent project
template triggering a post job doesn't wedge zuul on reporting.
"""
# Create change that adds uses a non-existent project template
conf = textwrap.dedent(
"""
- project:
templates:
- non-existent
check:
jobs:
- noop
post:
jobs:
- post-job
""")
file_dict = {'zuul.yaml': conf}
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A',
files=file_dict)
# Now force merge the change
A.setMerged()
self.fake_gerrit.addEvent(A.getChangeMergedEvent())
self.waitUntilSettled()
self.fake_gerrit.addEvent(A.getRefUpdatedEvent())
self.waitUntilSettled()
B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertEqual(B.reported, 1)
self.assertHistory([
dict(name='other-job', result='SUCCESS', changes='2,1'),
])

View File

@ -912,7 +912,12 @@ class PipelineManager(object):
layout = (item.layout or self.pipeline.tenant.layout)
project_in_pipeline = True
if not layout.getProjectPipelineConfig(item):
ppc = None
try:
ppc = layout.getProjectPipelineConfig(item)
except Exception:
self.log.exception("Invalid config for change %s" % item.change)
if not ppc:
self.log.debug("Project %s not in pipeline %s for change %s" % (
item.change.project, self.pipeline, item.change))
project_in_pipeline = False