summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2018-04-20 23:51:16 +0000
committerGerrit Code Review <review@openstack.org>2018-04-20 23:51:16 +0000
commit623533ab85ddf406bcac16452a638a706682775a (patch)
tree67fbb075aea7fa46a0639addf9a37bb49ef566ca
parent0101a37c18fc466f886ca33b46884137b72594fa (diff)
parentc5e6f5cefe3ee33c65e0007a18241ab7e197b1d7 (diff)
Merge "Fix missing semaphore release on zk error"3.0.2
-rw-r--r--tests/fixtures/config/semaphore/git/common-config/zuul.yaml15
-rw-r--r--tests/fixtures/config/semaphore/git/org_project2/README1
-rw-r--r--tests/fixtures/config/semaphore/main.yaml1
-rwxr-xr-xtests/unit/test_scheduler.py30
-rw-r--r--zuul/manager/__init__.py8
5 files changed, 55 insertions, 0 deletions
diff --git a/tests/fixtures/config/semaphore/git/common-config/zuul.yaml b/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
index 52a0e7d..600543c 100644
--- a/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/semaphore/git/common-config/zuul.yaml
@@ -47,6 +47,15 @@
47 semaphore: test-semaphore-two 47 semaphore: test-semaphore-two
48 run: playbooks/semaphore-two-test2.yaml 48 run: playbooks/semaphore-two-test2.yaml
49 49
50- job:
51 name: semaphore-one-test3
52 semaphore: test-semaphore
53 run: playbooks/semaphore-two-test1.yaml
54 nodeset:
55 nodes:
56 - name: controller
57 label: label1
58
50- project: 59- project:
51 name: org/project 60 name: org/project
52 check: 61 check:
@@ -62,3 +71,9 @@
62 - project-test1 71 - project-test1
63 - semaphore-two-test1 72 - semaphore-two-test1
64 - semaphore-two-test2 73 - semaphore-two-test2
74
75- project:
76 name: org/project2
77 check:
78 jobs:
79 - semaphore-one-test3
diff --git a/tests/fixtures/config/semaphore/git/org_project2/README b/tests/fixtures/config/semaphore/git/org_project2/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/semaphore/git/org_project2/README
@@ -0,0 +1 @@
test
diff --git a/tests/fixtures/config/semaphore/main.yaml b/tests/fixtures/config/semaphore/main.yaml
index 5f57245..83ed092 100644
--- a/tests/fixtures/config/semaphore/main.yaml
+++ b/tests/fixtures/config/semaphore/main.yaml
@@ -7,3 +7,4 @@
7 untrusted-projects: 7 untrusted-projects:
8 - org/project 8 - org/project
9 - org/project1 9 - org/project1
10 - org/project2
diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py
index 735244e..d7686c6 100755
--- a/tests/unit/test_scheduler.py
+++ b/tests/unit/test_scheduler.py
@@ -23,6 +23,7 @@ import shutil
23import time 23import time
24from unittest import mock 24from unittest import mock
25from unittest import skip 25from unittest import skip
26from kazoo.exceptions import NoNodeError
26 27
27import git 28import git
28import testtools 29import testtools
@@ -5487,6 +5488,35 @@ class TestSemaphore(ZuulTestCase):
5487 self.assertEqual(A.reported, 1) 5488 self.assertEqual(A.reported, 1)
5488 self.assertEqual(B.reported, 1) 5489 self.assertEqual(B.reported, 1)
5489 5490
5491 def test_semaphore_zk_error(self):
5492 "Test semaphore release with zk error"
5493 tenant = self.sched.abide.tenants.get('tenant-one')
5494
5495 A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A')
5496 self.assertFalse('test-semaphore' in
5497 tenant.semaphore_handler.semaphores)
5498
5499 # Simulate a single zk error in useNodeSet
5500 orig_useNodeSet = self.nodepool.useNodeSet
5501
5502 def broken_use_nodeset(nodeset):
5503 # restore original useNodeSet
5504 self.nodepool.useNodeSet = orig_useNodeSet
5505 raise NoNodeError()
5506
5507 self.nodepool.useNodeSet = broken_use_nodeset
5508
5509 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5510 self.waitUntilSettled()
5511
5512 # The semaphore should be released
5513 self.assertFalse('test-semaphore' in
5514 tenant.semaphore_handler.semaphores)
5515
5516 # cleanup the queue
5517 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
5518 self.waitUntilSettled()
5519
5490 def test_semaphore_abandon(self): 5520 def test_semaphore_abandon(self):
5491 "Test abandon with job semaphores" 5521 "Test abandon with job semaphores"
5492 self.executor_server.hold_jobs_in_build = True 5522 self.executor_server.hold_jobs_in_build = True
diff --git a/zuul/manager/__init__.py b/zuul/manager/__init__.py
index a5cf0ee..bcb70af 100644
--- a/zuul/manager/__init__.py
+++ b/zuul/manager/__init__.py
@@ -399,6 +399,14 @@ class PipelineManager(object):
399 except Exception: 399 except Exception:
400 self.log.exception("Exception while executing job %s " 400 self.log.exception("Exception while executing job %s "
401 "for change %s:" % (job, item.change)) 401 "for change %s:" % (job, item.change))
402 try:
403 # If we hit an exception we don't have a build in the
404 # current item so a potentially aquired semaphore must be
405 # released as it won't be released on dequeue of the item.
406 tenant = item.pipeline.layout.tenant
407 tenant.semaphore_handler.release(item, job)
408 except Exception:
409 self.log.exception("Exception while releasing semaphore")
402 410
403 def executeJobs(self, item): 411 def executeJobs(self, item):
404 # TODO(jeblair): This should return a value indicating a job 412 # TODO(jeblair): This should return a value indicating a job