summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJames E. Blair <jeblair@redhat.com>2019-01-28 13:18:22 -0800
committerJames E. Blair <jeblair@redhat.com>2019-01-30 14:07:42 -0800
commit1317391323bee2d868b4f3ce30a4840fc6754cf1 (patch)
treefbabb3b283a03bb3e724c065867afb6a4dfb9609
parent91e7e680a16dc17469195029750a0a3231586c10 (diff)
Add provides/requires support
Adds support for expressing artifact dependencies between jobs which may run in different projects. Change-Id: If8cce8750d296d607841800e4bbf688a24c40e08
Notes
Notes (review): Code-Review+2: Tobias Henkel <tobias.henkel@bmw.de> Code-Review+2: Monty Taylor <mordred@inaugust.com> Workflow+1: Monty Taylor <mordred@inaugust.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Sat, 02 Feb 2019 07:35:51 +0000 Reviewed-on: https://review.openstack.org/633605 Project: openstack-infra/zuul Branch: refs/heads/master
-rw-r--r--doc/source/user/config.rst51
-rw-r--r--doc/source/user/jobs.rst35
-rw-r--r--releasenotes/notes/provides_requires-4c6b54ede999e86c.yaml7
-rw-r--r--tests/base.py29
-rw-r--r--tests/fixtures/config/provides-requires-pause/git/common-config/zuul.yaml38
-rw-r--r--tests/fixtures/config/provides-requires-pause/git/org_project1/README1
-rw-r--r--tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-builder.yaml10
-rw-r--r--tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-user.yaml4
-rw-r--r--tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml26
-rw-r--r--tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml8
-rw-r--r--tests/fixtures/config/provides-requires-pause/main.yaml8
-rw-r--r--tests/fixtures/layouts/provides-requires-two-jobs.yaml72
-rw-r--r--tests/fixtures/layouts/provides-requires-unshared.yaml58
-rw-r--r--tests/fixtures/layouts/provides-requires.yaml70
-rw-r--r--tests/unit/test_v3.py288
-rwxr-xr-xtests/unit/test_web.py21
-rw-r--r--zuul/configloader.py11
-rw-r--r--zuul/driver/sql/alembic/versions/39d302d34d38_add_provides.py46
-rw-r--r--zuul/driver/sql/sqlconnection.py28
-rw-r--r--zuul/driver/sql/sqlreporter.py55
-rw-r--r--zuul/executor/client.py2
-rw-r--r--zuul/lib/artifacts.py69
-rw-r--r--zuul/model.py136
-rwxr-xr-xzuul/web/__init__.py5
24 files changed, 1019 insertions, 59 deletions
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 04de33e..c2053b3 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -686,6 +686,57 @@ Here is an example of two job definitions:
686 tags from all the jobs and variants used in constructing the 686 tags from all the jobs and variants used in constructing the
687 frozen job, with no duplication. 687 frozen job, with no duplication.
688 688
689 .. attr:: provides
690
691 A list of free-form strings which identifies resources provided
692 by this job which may be used by other jobs for other changes
693 using the :attr:`job.requires` attribute.
694
695 .. attr:: requires
696
697 A list of free-form strings which identify resources which may
698 be provided by other jobs for other changes (via the
699 :attr:`job.provides` attribute) that are used by this job.
700
701 When Zuul encounters a job with a `requires` attribute, it
702 searches for those values in the `provides` attributes of any
703 jobs associated with any queue items ahead of the current
704 change. In this way, if a change uses either git dependencies
705 or a `Depends-On` header to indicate a dependency on another
706 change, Zuul will be able to determine that the parent change
707 affects the run-time environment of the child change. If such a
708 relationship is found, the job with `requires` will not start
709 until all of the jobs with matching `provides` have completed or
710 paused. Additionally, the :ref:`artifacts <return_artifacts>`
711 returned by the `provides` jobs will be made available to the
712 `requires` job.
713
714 For example, a job which produces a builder container image in
715 one project that is then consumed by a container image build job
716 in another project might look like this:
717
718 .. code-block:: yaml
719
720 - job:
721 name: build-builder-image
722 provides: images
723
724 - job:
725 name: build-final-image
726 requires: images
727
728 - project:
729 name: builder-project
730 check:
731 jobs:
732 - build-builder-image
733
734 - project:
735 name: final-project
736 check:
737 jobs:
738 - build-final-image
739
689 .. attr:: secrets 740 .. attr:: secrets
690 741
691 A list of secrets which may be used by the job. A 742 A list of secrets which may be used by the job. A
diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst
index cc195e7..826ff34 100644
--- a/doc/source/user/jobs.rst
+++ b/doc/source/user/jobs.rst
@@ -228,6 +228,41 @@ of item.
228 All items provide the following information as Ansible variables 228 All items provide the following information as Ansible variables
229 under the ``zuul`` key: 229 under the ``zuul`` key:
230 230
231 .. var:: artifacts
232 :type: list
233
234 If the job has a :attr:`job.requires` attribute, and Zuul has
235 found changes ahead of this change in the pipeline with matching
236 :attr:`job.provides` attributes, then information about any
237 :ref:`artifacts returned <return_artifacts>` from those jobs
238 will appear here.
239
240 This value is a list of dictionaries with the following format:
241
242 .. var:: project
243
244 The name of the project which supplied this artifact.
245
246 .. var:: change
247
248 The change number which supplied this artifact.
249
250 .. var:: patchset
251
252 The patchset of the change.
253
254 .. var:: job
255
256 The name of the job which produced the artifact.
257
258 .. var:: name
259
260 The name of the artifact (as supplied to :ref:`return_artifacts`).
261
262 .. var:: url
263
264 The URL of the artifact (as supplied to :ref:`return_artifacts`).
265
231 .. var:: build 266 .. var:: build
232 267
233 The UUID of the build. A build is a single execution of a job. 268 The UUID of the build. A build is a single execution of a job.
diff --git a/releasenotes/notes/provides_requires-4c6b54ede999e86c.yaml b/releasenotes/notes/provides_requires-4c6b54ede999e86c.yaml
new file mode 100644
index 0000000..1990065
--- /dev/null
+++ b/releasenotes/notes/provides_requires-4c6b54ede999e86c.yaml
@@ -0,0 +1,7 @@
1---
2features:
3 - Support for expressing artifact or other resource dependencies
4 between jobs running on different changes with a dependency
5 relationship (e.g., a container image built in one project and
6 consumed in a second project) has been added via the
7 :attr:`job.provides` and :attr:`job.requires` job attributes.
diff --git a/tests/base.py b/tests/base.py
index a2f7c93..a8803c7 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -1355,6 +1355,11 @@ class FakeBuild(object):
1355 items = self.parameters['zuul']['items'] 1355 items = self.parameters['zuul']['items']
1356 self.changes = ' '.join(['%s,%s' % (x['change'], x['patchset']) 1356 self.changes = ' '.join(['%s,%s' % (x['change'], x['patchset'])
1357 for x in items if 'change' in x]) 1357 for x in items if 'change' in x])
1358 if 'change' in items[-1]:
1359 self.change = ' '.join((items[-1]['change'],
1360 items[-1]['patchset']))
1361 else:
1362 self.change = None
1358 1363
1359 def __repr__(self): 1364 def __repr__(self):
1360 waiting = '' 1365 waiting = ''
@@ -1401,6 +1406,8 @@ class FakeBuild(object):
1401 self._wait() 1406 self._wait()
1402 self.log.debug("Build %s continuing" % self.unique) 1407 self.log.debug("Build %s continuing" % self.unique)
1403 1408
1409 self.writeReturnData()
1410
1404 result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success 1411 result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
1405 if self.shouldFail(): 1412 if self.shouldFail():
1406 result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure 1413 result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
@@ -1418,6 +1425,14 @@ class FakeBuild(object):
1418 return True 1425 return True
1419 return False 1426 return False
1420 1427
1428 def writeReturnData(self):
1429 changes = self.executor_server.return_data.get(self.name, {})
1430 data = changes.get(self.change)
1431 if data is None:
1432 return
1433 with open(self.jobdir.result_data_file, 'w') as f:
1434 f.write(json.dumps(data))
1435
1421 def hasChanges(self, *changes): 1436 def hasChanges(self, *changes):
1422 """Return whether this build has certain changes in its git repos. 1437 """Return whether this build has certain changes in its git repos.
1423 1438
@@ -1554,6 +1569,7 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
1554 self.running_builds = [] 1569 self.running_builds = []
1555 self.build_history = [] 1570 self.build_history = []
1556 self.fail_tests = {} 1571 self.fail_tests = {}
1572 self.return_data = {}
1557 self.job_builds = {} 1573 self.job_builds = {}
1558 1574
1559 def failJob(self, name, change): 1575 def failJob(self, name, change):
@@ -1569,6 +1585,19 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
1569 l.append(change) 1585 l.append(change)
1570 self.fail_tests[name] = l 1586 self.fail_tests[name] = l
1571 1587
1588 def returnData(self, name, change, data):
1589 """Instruct the executor to return data for this build.
1590
1591 :arg str name: The name of the job to return data.
1592 :arg Change change: The :py:class:`~tests.base.FakeChange`
1593 instance which should cause the job to return data.
1594 :arg dict data: The data to return
1595
1596 """
1597 changes = self.return_data.setdefault(name, {})
1598 cid = ' '.join((str(change.number), str(change.latest_patchset)))
1599 changes[cid] = data
1600
1572 def release(self, regex=None): 1601 def release(self, regex=None):
1573 """Release a held build. 1602 """Release a held build.
1574 1603
diff --git a/tests/fixtures/config/provides-requires-pause/git/common-config/zuul.yaml b/tests/fixtures/config/provides-requires-pause/git/common-config/zuul.yaml
new file mode 100644
index 0000000..12901dd
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/git/common-config/zuul.yaml
@@ -0,0 +1,38 @@
1- pipeline:
2 name: check
3 manager: independent
4 post-review: true
5 trigger:
6 gerrit:
7 - event: patchset-created
8 success:
9 gerrit:
10 Verified: 1
11 failure:
12 gerrit:
13 Verified: -1
14
15- pipeline:
16 name: gate
17 manager: dependent
18 post-review: True
19 trigger:
20 gerrit:
21 - event: comment-added
22 approval:
23 - Approved: 1
24 success:
25 gerrit:
26 Verified: 2
27 submit: true
28 failure:
29 gerrit:
30 Verified: -2
31 start:
32 gerrit:
33 Verified: 0
34 precedence: high
35
36- job:
37 name: base
38 parent: null
diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/README b/tests/fixtures/config/provides-requires-pause/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/README
@@ -0,0 +1 @@
test
diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-builder.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-builder.yaml
new file mode 100644
index 0000000..7d773b6
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-builder.yaml
@@ -0,0 +1,10 @@
1- hosts: all
2 tasks:
3 - name: Pause and let child run
4 zuul_return:
5 data:
6 zuul:
7 pause: true
8 artifacts:
9 - name: image
10 url: http://example.com/image
diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-user.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-user.yaml
new file mode 100644
index 0000000..583279c
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-user.yaml
@@ -0,0 +1,4 @@
1- hosts: all
2 tasks:
3 - debug:
4 var: zuul.artifacts
diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..412fe2c
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml
@@ -0,0 +1,26 @@
1- job:
2 name: image-builder
3 provides:
4 - image
5 run: playbooks/image-builder.yaml
6
7- job:
8 name: image-user
9 requires:
10 - image
11 run: playbooks/image-user.yaml
12
13- project:
14 check:
15 jobs:
16 - image-builder
17 - image-user:
18 dependencies:
19 - image-builder
20 gate:
21 queue: integrated
22 jobs:
23 - image-builder
24 - image-user:
25 dependencies:
26 - image-builder
diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..e9e6b58
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml
@@ -0,0 +1,8 @@
1- project:
2 check:
3 jobs:
4 - image-user
5 gate:
6 queue: integrated
7 jobs:
8 - image-user
diff --git a/tests/fixtures/config/provides-requires-pause/main.yaml b/tests/fixtures/config/provides-requires-pause/main.yaml
new file mode 100644
index 0000000..3a74155
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/main.yaml
@@ -0,0 +1,8 @@
1- tenant:
2 name: tenant-one
3 source:
4 gerrit:
5 config-projects:
6 - common-config
7 - org/project1
8 - org/project2
diff --git a/tests/fixtures/layouts/provides-requires-two-jobs.yaml b/tests/fixtures/layouts/provides-requires-two-jobs.yaml
new file mode 100644
index 0000000..6d1cc77
--- /dev/null
+++ b/tests/fixtures/layouts/provides-requires-two-jobs.yaml
@@ -0,0 +1,72 @@
1- pipeline:
2 name: check
3 manager: independent
4 trigger:
5 gerrit:
6 - event: patchset-created
7 success:
8 gerrit:
9 Verified: 1
10 resultsdb_mysql: null
11 resultsdb_postgresql: null
12 failure:
13 gerrit:
14 Verified: -1
15 resultsdb_mysql: null
16 resultsdb_postgresql: null
17
18- pipeline:
19 name: gate
20 manager: dependent
21 success-message: Build succeeded (gate).
22 trigger:
23 gerrit:
24 - event: comment-added
25 approval:
26 - Approved: 1
27 success:
28 gerrit:
29 Verified: 2
30 submit: true
31 failure:
32 gerrit:
33 Verified: -2
34 start:
35 gerrit:
36 Verified: 0
37 precedence: high
38
39- job:
40 name: base
41 parent: null
42 run: playbooks/base.yaml
43
44- job:
45 name: image-builder
46 provides: images
47
48- job:
49 name: image-user
50 requires: images
51
52- project:
53 name: org/project1
54 check:
55 jobs:
56 - image-builder
57 gate:
58 queue: integrated
59 jobs:
60 - image-builder
61 - image-user:
62 dependencies: image-builder
63
64- project:
65 name: org/project2
66 check:
67 jobs:
68 - image-user
69 gate:
70 queue: integrated
71 jobs:
72 - image-user
diff --git a/tests/fixtures/layouts/provides-requires-unshared.yaml b/tests/fixtures/layouts/provides-requires-unshared.yaml
new file mode 100644
index 0000000..65df1bc
--- /dev/null
+++ b/tests/fixtures/layouts/provides-requires-unshared.yaml
@@ -0,0 +1,58 @@
1- pipeline:
2 name: check
3 manager: independent
4 trigger:
5 gerrit:
6 - event: patchset-created
7 success:
8 gerrit:
9 Verified: 1
10 failure:
11 gerrit:
12 Verified: -1
13
14- pipeline:
15 name: gate
16 manager: dependent
17 success-message: Build succeeded (gate).
18 trigger:
19 gerrit:
20 - event: comment-added
21 approval:
22 - Approved: 1
23 success:
24 gerrit:
25 Verified: 2
26 submit: true
27 failure:
28 gerrit:
29 Verified: -2
30 start:
31 gerrit:
32 Verified: 0
33 precedence: high
34
35- job:
36 name: base
37 parent: null
38 run: playbooks/base.yaml
39
40- job:
41 name: image-builder
42 provides: images
43
44- job:
45 name: image-user
46 requires: images
47
48- project:
49 name: org/project1
50 gate:
51 jobs:
52 - image-builder
53
54- project:
55 name: org/project2
56 gate:
57 jobs:
58 - image-user
diff --git a/tests/fixtures/layouts/provides-requires.yaml b/tests/fixtures/layouts/provides-requires.yaml
new file mode 100644
index 0000000..3947345
--- /dev/null
+++ b/tests/fixtures/layouts/provides-requires.yaml
@@ -0,0 +1,70 @@
1- pipeline:
2 name: check
3 manager: independent
4 trigger:
5 gerrit:
6 - event: patchset-created
7 success:
8 gerrit:
9 Verified: 1
10 resultsdb_mysql: null
11 resultsdb_postgresql: null
12 failure:
13 gerrit:
14 Verified: -1
15 resultsdb_mysql: null
16 resultsdb_postgresql: null
17
18- pipeline:
19 name: gate
20 manager: dependent
21 success-message: Build succeeded (gate).
22 trigger:
23 gerrit:
24 - event: comment-added
25 approval:
26 - Approved: 1
27 success:
28 gerrit:
29 Verified: 2
30 submit: true
31 failure:
32 gerrit:
33 Verified: -2
34 start:
35 gerrit:
36 Verified: 0
37 precedence: high
38
39- job:
40 name: base
41 parent: null
42 run: playbooks/base.yaml
43
44- job:
45 name: image-builder
46 provides: images
47
48- job:
49 name: image-user
50 requires: images
51
52- project:
53 name: org/project1
54 check:
55 jobs:
56 - image-builder
57 gate:
58 queue: integrated
59 jobs:
60 - image-builder
61
62- project:
63 name: org/project2
64 check:
65 jobs:
66 - image-user
67 gate:
68 queue: integrated
69 jobs:
70 - image-user
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index d6554a0..4ae89fd 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -28,6 +28,7 @@ from zuul.lib import encryption
28from tests.base import ( 28from tests.base import (
29 AnsibleZuulTestCase, 29 AnsibleZuulTestCase,
30 ZuulTestCase, 30 ZuulTestCase,
31 ZuulDBTestCase,
31 FIXTURE_DIR, 32 FIXTURE_DIR,
32 simple_layout, 33 simple_layout,
33) 34)
@@ -4714,3 +4715,290 @@ class TestContainerJobs(AnsibleZuulTestCase):
4714 dict(name='container-machine', result='SUCCESS', changes='1,1'), 4715 dict(name='container-machine', result='SUCCESS', changes='1,1'),
4715 dict(name='container-native', result='SUCCESS', changes='1,1'), 4716 dict(name='container-native', result='SUCCESS', changes='1,1'),
4716 ]) 4717 ])
4718
4719
4720class TestProvidesRequiresPause(AnsibleZuulTestCase):
4721 tenant_config_file = "config/provides-requires-pause/main.yaml"
4722
4723 def test_provides_requires_pause(self):
4724 # Changes share a queue, with both running at the same time.
4725 self.executor_server.hold_jobs_in_build = True
4726 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4727 A.addApproval('Code-Review', 2)
4728 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4729 self.waitUntilSettled()
4730
4731 self.assertEqual(len(self.builds), 1)
4732
4733 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4734 B.addApproval('Code-Review', 2)
4735 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4736 self.waitUntilSettled()
4737
4738 self.assertEqual(len(self.builds), 1)
4739
4740 # Release image-build, it should cause both instances of
4741 # image-user to run.
4742 self.executor_server.hold_jobs_in_build = False
4743 self.executor_server.release()
4744 self.waitUntilSettled()
4745
4746 self.assertHistory([
4747 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4748 dict(name='image-user', result='SUCCESS', changes='1,1'),
4749 dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4750 ], ordered=False)
4751 build = self.getJobFromHistory('image-user', project='org/project2')
4752 self.assertEqual(
4753 build.parameters['zuul']['artifacts'],
4754 [{
4755 'project': 'org/project1',
4756 'change': '1',
4757 'patchset': '1',
4758 'job': 'image-builder',
4759 'url': 'http://example.com/image',
4760 'name': 'image',
4761 }])
4762
4763
4764class TestProvidesRequires(ZuulDBTestCase):
4765 config_file = "zuul-sql-driver.conf"
4766
4767 @simple_layout('layouts/provides-requires.yaml')
4768 def test_provides_requires_shared_queue_fast(self):
4769 # Changes share a queue, but with only one job, the first
4770 # merges before the second starts.
4771 self.executor_server.hold_jobs_in_build = True
4772 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4773 self.executor_server.returnData(
4774 'image-builder', A,
4775 {'zuul':
4776 {'artifacts': [
4777 {'name': 'image', 'url': 'http://example.com/image'},
4778 ]}}
4779 )
4780 A.addApproval('Code-Review', 2)
4781 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4782 self.waitUntilSettled()
4783
4784 self.assertEqual(len(self.builds), 1)
4785
4786 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4787 B.addApproval('Code-Review', 2)
4788 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4789 self.waitUntilSettled()
4790
4791 self.assertEqual(len(self.builds), 1)
4792
4793 self.executor_server.hold_jobs_in_build = False
4794 self.executor_server.release()
4795 self.waitUntilSettled()
4796
4797 self.assertHistory([
4798 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4799 dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4800 ])
4801 # Data are not passed in this instance because the builder
4802 # change merges before the user job runs.
4803 self.assertFalse('artifacts' in self.history[-1].parameters['zuul'])
4804
4805 @simple_layout('layouts/provides-requires-two-jobs.yaml')
4806 def test_provides_requires_shared_queue_slow(self):
4807 # Changes share a queue, with both running at the same time.
4808 self.executor_server.hold_jobs_in_build = True
4809 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4810 self.executor_server.returnData(
4811 'image-builder', A,
4812 {'zuul':
4813 {'artifacts': [
4814 {'name': 'image', 'url': 'http://example.com/image'},
4815 ]}}
4816 )
4817 A.addApproval('Code-Review', 2)
4818 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4819 self.waitUntilSettled()
4820
4821 self.assertEqual(len(self.builds), 1)
4822
4823 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4824 B.addApproval('Code-Review', 2)
4825 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4826 self.waitUntilSettled()
4827
4828 self.assertEqual(len(self.builds), 1)
4829
4830 # Release image-build, it should cause both instances of
4831 # image-user to run.
4832 self.executor_server.release()
4833 self.waitUntilSettled()
4834 self.assertEqual(len(self.builds), 2)
4835 self.assertHistory([
4836 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4837 ])
4838
4839 self.orderedRelease()
4840 self.waitUntilSettled()
4841
4842 self.assertHistory([
4843 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4844 dict(name='image-user', result='SUCCESS', changes='1,1'),
4845 dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4846 ])
4847 self.assertEqual(
4848 self.history[-1].parameters['zuul']['artifacts'],
4849 [{
4850 'project': 'org/project1',
4851 'change': '1',
4852 'patchset': '1',
4853 'job': 'image-builder',
4854 'url': 'http://example.com/image',
4855 'name': 'image',
4856 }])
4857
4858 @simple_layout('layouts/provides-requires-unshared.yaml')
4859 def test_provides_requires_unshared_queue(self):
4860 self.executor_server.hold_jobs_in_build = True
4861 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4862 self.executor_server.returnData(
4863 'image-builder', A,
4864 {'zuul':
4865 {'artifacts': [
4866 {'name': 'image', 'url': 'http://example.com/image'},
4867 ]}}
4868 )
4869 A.addApproval('Code-Review', 2)
4870 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4871 self.waitUntilSettled()
4872
4873 self.assertEqual(len(self.builds), 1)
4874
4875 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4876 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4877 B.subject, A.data['id'])
4878 B.addApproval('Code-Review', 2)
4879 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4880 self.waitUntilSettled()
4881
4882 self.assertEqual(len(self.builds), 1)
4883
4884 self.executor_server.hold_jobs_in_build = False
4885 self.executor_server.release()
4886 self.waitUntilSettled()
4887
4888 self.assertHistory([
4889 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4890 ])
4891
4892 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4893 self.waitUntilSettled()
4894
4895 self.assertHistory([
4896 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4897 dict(name='image-user', result='SUCCESS', changes='2,1'),
4898 ])
4899 # Data are not passed in this instance because the builder
4900 # change merges before the user job runs.
4901 self.assertFalse('artifacts' in self.history[-1].parameters['zuul'])
4902
4903 @simple_layout('layouts/provides-requires.yaml')
4904 def test_provides_requires_check_current(self):
4905 self.executor_server.hold_jobs_in_build = True
4906 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4907 self.executor_server.returnData(
4908 'image-builder', A,
4909 {'zuul':
4910 {'artifacts': [
4911 {'name': 'image', 'url': 'http://example.com/image'},
4912 ]}}
4913 )
4914 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4915 self.waitUntilSettled()
4916
4917 self.assertEqual(len(self.builds), 1)
4918
4919 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4920 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4921 B.subject, A.data['id'])
4922 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4923 self.waitUntilSettled()
4924
4925 self.assertEqual(len(self.builds), 1)
4926
4927 self.executor_server.hold_jobs_in_build = False
4928 self.executor_server.release()
4929 self.waitUntilSettled()
4930
4931 self.assertHistory([
4932 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4933 dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4934 ])
4935 self.assertEqual(
4936 self.history[-1].parameters['zuul']['artifacts'],
4937 [{
4938 'project': 'org/project1',
4939 'change': '1',
4940 'patchset': '1',
4941 'job': 'image-builder',
4942 'url': 'http://example.com/image',
4943 'name': 'image',
4944 }])
4945
4946 @simple_layout('layouts/provides-requires.yaml')
4947 def test_provides_requires_check_old_success(self):
4948 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4949 self.executor_server.returnData(
4950 'image-builder', A,
4951 {'zuul':
4952 {'artifacts': [
4953 {'name': 'image', 'url': 'http://example.com/image'},
4954 ]}}
4955 )
4956 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4957 self.waitUntilSettled()
4958 self.assertHistory([
4959 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4960 ])
4961
4962 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4963 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4964 B.subject, A.data['id'])
4965 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4966 self.waitUntilSettled()
4967
4968 self.assertHistory([
4969 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4970 dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4971 ])
4972 self.assertEqual(
4973 self.history[-1].parameters['zuul']['artifacts'],
4974 [{
4975 'project': 'org/project1',
4976 'change': '1',
4977 'patchset': '1',
4978 'job': 'image-builder',
4979 'url': 'http://example.com/image',
4980 'name': 'image',
4981 }])
4982
4983 @simple_layout('layouts/provides-requires.yaml')
4984 def test_provides_requires_check_old_failure(self):
4985 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4986 self.executor_server.failJob('image-builder', A)
4987 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4988 self.waitUntilSettled()
4989
4990 self.assertHistory([
4991 dict(name='image-builder', result='FAILURE', changes='1,1'),
4992 ])
4993
4994 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4995 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4996 B.subject, A.data['id'])
4997 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4998 self.waitUntilSettled()
4999
5000 self.assertHistory([
5001 dict(name='image-builder', result='FAILURE', changes='1,1'),
5002 ])
5003 self.assertIn('image-user : SKIPPED', B.messages[0])
5004 self.assertIn('not met by build', B.messages[0])
diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py
index 05701a6..e267734 100755
--- a/tests/unit/test_web.py
+++ b/tests/unit/test_web.py
@@ -305,10 +305,13 @@ class TestWeb(BaseTestWeb):
305 'parent': 'base', 305 'parent': 'base',
306 'post_review': None, 306 'post_review': None,
307 'protected': None, 307 'protected': None,
308 'provides': [],
308 'required_projects': [], 309 'required_projects': [],
310 'requires': [],
309 'roles': [common_config_role], 311 'roles': [common_config_role],
310 'semaphore': None, 312 'semaphore': None,
311 'source_context': source_ctx, 313 'source_context': source_ctx,
314 'tags': [],
312 'timeout': None, 315 'timeout': None,
313 'variables': {}, 316 'variables': {},
314 'variant_description': '', 317 'variant_description': '',
@@ -337,10 +340,13 @@ class TestWeb(BaseTestWeb):
337 'parent': 'base', 340 'parent': 'base',
338 'post_review': None, 341 'post_review': None,
339 'protected': None, 342 'protected': None,
343 'provides': [],
340 'required_projects': [], 344 'required_projects': [],
345 'requires': [],
341 'roles': [common_config_role], 346 'roles': [common_config_role],
342 'semaphore': None, 347 'semaphore': None,
343 'source_context': source_ctx, 348 'source_context': source_ctx,
349 'tags': [],
344 'timeout': None, 350 'timeout': None,
345 'variables': {}, 351 'variables': {},
346 'variant_description': 'stable', 352 'variant_description': 'stable',
@@ -363,13 +369,16 @@ class TestWeb(BaseTestWeb):
363 'parent': 'base', 369 'parent': 'base',
364 'post_review': None, 370 'post_review': None,
365 'protected': None, 371 'protected': None,
372 'provides': [],
366 'required_projects': [ 373 'required_projects': [
367 {'override_branch': None, 374 {'override_branch': None,
368 'override_checkout': None, 375 'override_checkout': None,
369 'project_name': 'review.example.com/org/project'}], 376 'project_name': 'review.example.com/org/project'}],
377 'requires': [],
370 'roles': [common_config_role], 378 'roles': [common_config_role],
371 'semaphore': None, 379 'semaphore': None,
372 'source_context': source_ctx, 380 'source_context': source_ctx,
381 'tags': [],
373 'timeout': None, 382 'timeout': None,
374 'variables': {}, 383 'variables': {},
375 'variant_description': '', 384 'variant_description': '',
@@ -434,13 +443,16 @@ class TestWeb(BaseTestWeb):
434 'parent': 'base', 443 'parent': 'base',
435 'post_review': None, 444 'post_review': None,
436 'protected': None, 445 'protected': None,
446 'provides': [],
437 'required_projects': [], 447 'required_projects': [],
448 'requires': [],
438 'roles': [], 449 'roles': [],
439 'semaphore': None, 450 'semaphore': None,
440 'source_context': { 451 'source_context': {
441 'branch': 'master', 452 'branch': 'master',
442 'path': 'zuul.yaml', 453 'path': 'zuul.yaml',
443 'project': 'common-config'}, 454 'project': 'common-config'},
455 'tags': [],
444 'timeout': None, 456 'timeout': None,
445 'variables': {}, 457 'variables': {},
446 'variant_description': '', 458 'variant_description': '',
@@ -458,13 +470,16 @@ class TestWeb(BaseTestWeb):
458 'parent': 'base', 470 'parent': 'base',
459 'post_review': None, 471 'post_review': None,
460 'protected': None, 472 'protected': None,
473 'provides': [],
461 'required_projects': [], 474 'required_projects': [],
475 'requires': [],
462 'roles': [], 476 'roles': [],
463 'semaphore': None, 477 'semaphore': None,
464 'source_context': { 478 'source_context': {
465 'branch': 'master', 479 'branch': 'master',
466 'path': 'zuul.yaml', 480 'path': 'zuul.yaml',
467 'project': 'common-config'}, 481 'project': 'common-config'},
482 'tags': [],
468 'timeout': None, 483 'timeout': None,
469 'variables': {}, 484 'variables': {},
470 'variant_description': '', 485 'variant_description': '',
@@ -482,13 +497,16 @@ class TestWeb(BaseTestWeb):
482 'parent': 'base', 497 'parent': 'base',
483 'post_review': None, 498 'post_review': None,
484 'protected': None, 499 'protected': None,
500 'provides': [],
485 'required_projects': [], 501 'required_projects': [],
502 'requires': [],
486 'roles': [], 503 'roles': [],
487 'semaphore': None, 504 'semaphore': None,
488 'source_context': { 505 'source_context': {
489 'branch': 'master', 506 'branch': 'master',
490 'path': 'zuul.yaml', 507 'path': 'zuul.yaml',
491 'project': 'common-config'}, 508 'project': 'common-config'},
509 'tags': [],
492 'timeout': None, 510 'timeout': None,
493 'variables': {}, 511 'variables': {},
494 'variant_description': '', 512 'variant_description': '',
@@ -506,13 +524,16 @@ class TestWeb(BaseTestWeb):
506 'parent': 'base', 524 'parent': 'base',
507 'post_review': None, 525 'post_review': None,
508 'protected': None, 526 'protected': None,
527 'provides': [],
509 'required_projects': [], 528 'required_projects': [],
529 'requires': [],
510 'roles': [], 530 'roles': [],
511 'semaphore': None, 531 'semaphore': None,
512 'source_context': { 532 'source_context': {
513 'branch': 'master', 533 'branch': 'master',
514 'path': 'zuul.yaml', 534 'path': 'zuul.yaml',
515 'project': 'common-config'}, 535 'project': 'common-config'},
536 'tags': [],
516 'timeout': None, 537 'timeout': None,
517 'variables': {}, 538 'variables': {},
518 'variant_description': '', 539 'variant_description': '',
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 64318e2..5e4afc0 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -545,6 +545,8 @@ class JobParser(object):
545 'final': bool, 545 'final': bool,
546 'abstract': bool, 546 'abstract': bool,
547 'protected': bool, 547 'protected': bool,
548 'requires': to_list(str),
549 'provides': to_list(str),
548 'failure-message': str, 550 'failure-message': str,
549 'success-message': str, 551 'success-message': str,
550 'failure-url': str, 552 'failure-url': str,
@@ -769,11 +771,10 @@ class JobParser(object):
769 semaphore.get('name'), 771 semaphore.get('name'),
770 semaphore.get('resources-first', False)) 772 semaphore.get('resources-first', False))
771 773
772 tags = conf.get('tags') 774 for k in ('tags', 'requires', 'provides', 'dependencies'):
773 if tags: 775 v = frozenset(as_list(conf.get(k)))
774 job.tags = set(tags) 776 if v:
775 777 setattr(job, k, v)
776 job.dependencies = frozenset(as_list(conf.get('dependencies')))
777 778
778 variables = conf.get('vars', None) 779 variables = conf.get('vars', None)
779 if variables: 780 if variables:
diff --git a/zuul/driver/sql/alembic/versions/39d302d34d38_add_provides.py b/zuul/driver/sql/alembic/versions/39d302d34d38_add_provides.py
new file mode 100644
index 0000000..043b0c8
--- /dev/null
+++ b/zuul/driver/sql/alembic/versions/39d302d34d38_add_provides.py
@@ -0,0 +1,46 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13"""add_provides
14
15Revision ID: 39d302d34d38
16Revises: 649ce63b5fe5
17Create Date: 2019-01-28 15:01:07.408072
18
19"""
20
21# revision identifiers, used by Alembic.
22revision = '39d302d34d38'
23down_revision = '649ce63b5fe5'
24branch_labels = None
25depends_on = None
26
27from alembic import op
28import sqlalchemy as sa
29
30
31PROVIDES_TABLE = 'zuul_provides'
32BUILD_TABLE = 'zuul_build'
33
34
35def upgrade(table_prefix=''):
36 op.create_table(
37 table_prefix + PROVIDES_TABLE,
38 sa.Column('id', sa.Integer, primary_key=True),
39 sa.Column('build_id', sa.Integer,
40 sa.ForeignKey(table_prefix + BUILD_TABLE + ".id")),
41 sa.Column('name', sa.String(255)),
42 )
43
44
45def downgrade():
46 raise Exception("Downgrades not supported")
diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py
index 1a0efb5..5cf79e8 100644
--- a/zuul/driver/sql/sqlconnection.py
+++ b/zuul/driver/sql/sqlconnection.py
@@ -28,6 +28,7 @@ from zuul.connection import BaseConnection
28BUILDSET_TABLE = 'zuul_buildset' 28BUILDSET_TABLE = 'zuul_buildset'
29BUILD_TABLE = 'zuul_build' 29BUILD_TABLE = 'zuul_build'
30ARTIFACT_TABLE = 'zuul_artifact' 30ARTIFACT_TABLE = 'zuul_artifact'
31PROVIDES_TABLE = 'zuul_provides'
31 32
32 33
33class DatabaseSession(object): 34class DatabaseSession(object):
@@ -56,17 +57,21 @@ class DatabaseSession(object):
56 def getBuilds(self, tenant=None, project=None, pipeline=None, 57 def getBuilds(self, tenant=None, project=None, pipeline=None,
57 change=None, branch=None, patchset=None, ref=None, 58 change=None, branch=None, patchset=None, ref=None,
58 newrev=None, uuid=None, job_name=None, voting=None, 59 newrev=None, uuid=None, job_name=None, voting=None,
59 node_name=None, result=None, limit=50, offset=0): 60 node_name=None, result=None, provides=None,
61 limit=50, offset=0):
60 62
61 build_table = self.connection.zuul_build_table 63 build_table = self.connection.zuul_build_table
62 buildset_table = self.connection.zuul_buildset_table 64 buildset_table = self.connection.zuul_buildset_table
65 provides_table = self.connection.zuul_provides_table
63 66
64 # contains_eager allows us to perform eager loading on the 67 # contains_eager allows us to perform eager loading on the
65 # buildset *and* use that table in filters (unlike 68 # buildset *and* use that table in filters (unlike
66 # joinedload). 69 # joinedload).
67 q = self.session().query(self.connection.buildModel).\ 70 q = self.session().query(self.connection.buildModel).\
68 join(self.connection.buildSetModel).\ 71 join(self.connection.buildSetModel).\
72 outerjoin(self.connection.providesModel).\
69 options(orm.contains_eager(self.connection.buildModel.buildset), 73 options(orm.contains_eager(self.connection.buildModel.buildset),
74 orm.selectinload(self.connection.buildModel.provides),
70 orm.selectinload(self.connection.buildModel.artifacts)).\ 75 orm.selectinload(self.connection.buildModel.artifacts)).\
71 with_hint(build_table, 'USE INDEX (PRIMARY)', 'mysql') 76 with_hint(build_table, 'USE INDEX (PRIMARY)', 'mysql')
72 77
@@ -83,6 +88,7 @@ class DatabaseSession(object):
83 q = self.listFilter(q, build_table.c.voting, voting) 88 q = self.listFilter(q, build_table.c.voting, voting)
84 q = self.listFilter(q, build_table.c.node_name, node_name) 89 q = self.listFilter(q, build_table.c.node_name, node_name)
85 q = self.listFilter(q, build_table.c.result, result) 90 q = self.listFilter(q, build_table.c.result, result)
91 q = self.listFilter(q, provides_table.c.name, provides)
86 92
87 q = q.order_by(build_table.c.id.desc()).\ 93 q = q.order_by(build_table.c.id.desc()).\
88 limit(limit).\ 94 limit(limit).\
@@ -224,6 +230,15 @@ class SQLConnection(BaseConnection):
224 session.flush() 230 session.flush()
225 return a 231 return a
226 232
233 def createProvides(self, *args, **kw):
234 session = orm.session.Session.object_session(self)
235 p = ProvidesModel(*args, **kw)
236 p.build_id = self.id
237 self.provides.append(p)
238 session.add(p)
239 session.flush()
240 return p
241
227 class ArtifactModel(Base): 242 class ArtifactModel(Base):
228 __tablename__ = self.table_prefix + ARTIFACT_TABLE 243 __tablename__ = self.table_prefix + ARTIFACT_TABLE
229 id = sa.Column(sa.Integer, primary_key=True) 244 id = sa.Column(sa.Integer, primary_key=True)
@@ -233,6 +248,17 @@ class SQLConnection(BaseConnection):
233 url = sa.Column(sa.TEXT()) 248 url = sa.Column(sa.TEXT())
234 build = orm.relationship(BuildModel, backref="artifacts") 249 build = orm.relationship(BuildModel, backref="artifacts")
235 250
251 class ProvidesModel(Base):
252 __tablename__ = self.table_prefix + PROVIDES_TABLE
253 id = sa.Column(sa.Integer, primary_key=True)
254 build_id = sa.Column(sa.Integer, sa.ForeignKey(
255 self.table_prefix + BUILD_TABLE + ".id"))
256 name = sa.Column(sa.String(255))
257 build = orm.relationship(BuildModel, backref="provides")
258
259 self.providesModel = ProvidesModel
260 self.zuul_provides_table = self.providesModel.__table__
261
236 self.artifactModel = ArtifactModel 262 self.artifactModel = ArtifactModel
237 self.zuul_artifact_table = self.artifactModel.__table__ 263 self.zuul_artifact_table = self.artifactModel.__table__
238 264
diff --git a/zuul/driver/sql/sqlreporter.py b/zuul/driver/sql/sqlreporter.py
index 1e148d6..16651e4 100644
--- a/zuul/driver/sql/sqlreporter.py
+++ b/zuul/driver/sql/sqlreporter.py
@@ -16,9 +16,9 @@ import datetime
16import logging 16import logging
17import time 17import time
18import voluptuous as v 18import voluptuous as v
19import urllib.parse
20 19
21from zuul.reporter import BaseReporter 20from zuul.reporter import BaseReporter
21from zuul.lib.artifacts import get_artifacts_from_result_data
22 22
23 23
24class SQLReporter(BaseReporter): 24class SQLReporter(BaseReporter):
@@ -27,26 +27,6 @@ class SQLReporter(BaseReporter):
27 name = 'sql' 27 name = 'sql'
28 log = logging.getLogger("zuul.SQLReporter") 28 log = logging.getLogger("zuul.SQLReporter")
29 29
30 artifact = {
31 'name': str,
32 'url': str,
33 }
34 zuul_data = {
35 'zuul': {
36 'log_url': str,
37 'artifacts': [artifact],
38 v.Extra: object,
39 }
40 }
41 artifact_schema = v.Schema(zuul_data)
42
43 def validateArtifactSchema(self, data):
44 try:
45 self.artifact_schema(data)
46 except Exception:
47 return False
48 return True
49
50 def report(self, item): 30 def report(self, item):
51 """Create an entry into a database.""" 31 """Create an entry into a database."""
52 32
@@ -104,32 +84,13 @@ class SQLReporter(BaseReporter):
104 node_name=build.node_name, 84 node_name=build.node_name,
105 ) 85 )
106 86
107 if self.validateArtifactSchema(build.result_data): 87 for provides in job.provides:
108 artifacts = build.result_data.get('zuul', {}).get( 88 db_build.createProvides(name=provides)
109 'artifacts', []) 89
110 default_url = build.result_data.get('zuul', {}).get( 90 for artifact in get_artifacts_from_result_data(
111 'log_url') 91 build.result_data,
112 if default_url: 92 logger=self.log):
113 if default_url[-1] != '/': 93 db_build.createArtifact(**artifact)
114 default_url += '/'
115 for artifact in artifacts:
116 url = artifact['url']
117 if default_url:
118 # If the artifact url is relative, it will
119 # be combined with the log_url; if it is
120 # absolute, it will replace it.
121 try:
122 url = urllib.parse.urljoin(default_url, url)
123 except Exception:
124 self.log.debug("Error parsing URL:",
125 exc_info=1)
126 db_build.createArtifact(
127 name=artifact['name'],
128 url=url,
129 )
130 else:
131 self.log.debug("Result data did not pass artifact schema "
132 "validation: %s", build.result_data)
133 94
134 95
135def getSchema(): 96def getSchema():
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 00ea8eb..801726d 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -165,6 +165,8 @@ class ExecutorClient(object):
165 timeout=job.timeout, 165 timeout=job.timeout,
166 jobtags=sorted(job.tags), 166 jobtags=sorted(job.tags),
167 _inheritance_path=list(job.inheritance_path)) 167 _inheritance_path=list(job.inheritance_path))
168 if job.artifact_data:
169 zuul_params['artifacts'] = job.artifact_data
168 if job.override_checkout: 170 if job.override_checkout:
169 zuul_params['override_checkout'] = job.override_checkout 171 zuul_params['override_checkout'] = job.override_checkout
170 if hasattr(item.change, 'branch'): 172 if hasattr(item.change, 'branch'):
diff --git a/zuul/lib/artifacts.py b/zuul/lib/artifacts.py
new file mode 100644
index 0000000..c7c2fe0
--- /dev/null
+++ b/zuul/lib/artifacts.py
@@ -0,0 +1,69 @@
1# Copyright 2018-2019 Red Hat, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import voluptuous as v
16import urllib.parse
17
18artifact = {
19 'name': str,
20 'url': str,
21}
22
23zuul_data = {
24 'zuul': {
25 'log_url': str,
26 'artifacts': [artifact],
27 v.Extra: object,
28 }
29}
30
31artifact_schema = v.Schema(zuul_data)
32
33
34def validate_artifact_schema(data):
35 try:
36 artifact_schema(data)
37 except Exception:
38 return False
39 return True
40
41
42def get_artifacts_from_result_data(result_data, logger=None):
43 ret = []
44 if validate_artifact_schema(result_data):
45 artifacts = result_data.get('zuul', {}).get(
46 'artifacts', [])
47 default_url = result_data.get('zuul', {}).get(
48 'log_url')
49 if default_url:
50 if default_url[-1] != '/':
51 default_url += '/'
52 for artifact in artifacts:
53 url = artifact['url']
54 if default_url:
55 # If the artifact url is relative, it will be combined
56 # with the log_url; if it is absolute, it will replace
57 # it.
58 try:
59 url = urllib.parse.urljoin(default_url, url)
60 except Exception:
61 if logger:
62 logger.debug("Error parsing URL:",
63 exc_info=1)
64 ret.append({'name': artifact['name'],
65 'url': url})
66 else:
67 logger.debug("Result data did not pass artifact schema "
68 "validation: %s", result_data)
69 return ret
diff --git a/zuul/model.py b/zuul/model.py
index e7cff08..cefa370 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -28,6 +28,7 @@ import itertools
28 28
29from zuul import change_matcher 29from zuul import change_matcher
30from zuul.lib.config import get_default 30from zuul.lib.config import get_default
31from zuul.lib.artifacts import get_artifacts_from_result_data
31 32
32MERGER_MERGE = 1 # "git merge" 33MERGER_MERGE = 1 # "git merge"
33MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve" 34MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve"
@@ -164,6 +165,11 @@ class TemplateNotFoundError(Exception):
164 pass 165 pass
165 166
166 167
168class RequirementsError(Exception):
169 """A job's requirements were not met."""
170 pass
171
172
167class Attributes(object): 173class Attributes(object):
168 """A class to hold attributes for string formatting.""" 174 """A class to hold attributes for string formatting."""
169 175
@@ -1070,6 +1076,8 @@ class Job(ConfigObject):
1070 file_matcher=None, 1076 file_matcher=None,
1071 irrelevant_file_matcher=None, # skip-if 1077 irrelevant_file_matcher=None, # skip-if
1072 tags=frozenset(), 1078 tags=frozenset(),
1079 provides=frozenset(),
1080 requires=frozenset(),
1073 dependencies=frozenset(), 1081 dependencies=frozenset(),
1074 ) 1082 )
1075 1083
@@ -1111,6 +1119,7 @@ class Job(ConfigObject):
1111 start_mark=None, 1119 start_mark=None,
1112 inheritance_path=(), 1120 inheritance_path=(),
1113 parent_data=None, 1121 parent_data=None,
1122 artifact_data=None,
1114 description=None, 1123 description=None,
1115 variant_description=None, 1124 variant_description=None,
1116 protected_origin=None, 1125 protected_origin=None,
@@ -1161,6 +1170,10 @@ class Job(ConfigObject):
1161 d['protected'] = self.protected 1170 d['protected'] = self.protected
1162 d['voting'] = self.voting 1171 d['voting'] = self.voting
1163 d['timeout'] = self.timeout 1172 d['timeout'] = self.timeout
1173 d['tags'] = list(self.tags)
1174 d['provides'] = list(self.provides)
1175 d['requires'] = list(self.requires)
1176 d['dependencies'] = list(self.dependencies)
1164 d['attempts'] = self.attempts 1177 d['attempts'] = self.attempts
1165 d['roles'] = list(map(lambda x: x.toDict(), self.roles)) 1178 d['roles'] = list(map(lambda x: x.toDict(), self.roles))
1166 d['post_review'] = self.post_review 1179 d['post_review'] = self.post_review
@@ -1170,9 +1183,6 @@ class Job(ConfigObject):
1170 d['parent'] = self.parent 1183 d['parent'] = self.parent
1171 else: 1184 else:
1172 d['parent'] = tenant.default_base_job 1185 d['parent'] = tenant.default_base_job
1173 d['dependencies'] = []
1174 for dependency in self.dependencies:
1175 d['dependencies'].append(dependency)
1176 if isinstance(self.nodeset, str): 1186 if isinstance(self.nodeset, str):
1177 ns = tenant.layout.nodesets.get(self.nodeset) 1187 ns = tenant.layout.nodesets.get(self.nodeset)
1178 else: 1188 else:
@@ -1366,6 +1376,9 @@ class Job(ConfigObject):
1366 self.parent_data = v 1376 self.parent_data = v
1367 self.variables = Job._deepUpdate(self.parent_data, self.variables) 1377 self.variables = Job._deepUpdate(self.parent_data, self.variables)
1368 1378
1379 def updateArtifactData(self, artifact_data):
1380 self.artifact_data = artifact_data
1381
1369 def updateProjectVariables(self, project_vars): 1382 def updateProjectVariables(self, project_vars):
1370 # Merge project/template variables directly into the job 1383 # Merge project/template variables directly into the job
1371 # variables. Job variables override project variables. 1384 # variables. Job variables override project variables.
@@ -1522,11 +1535,12 @@ class Job(ConfigObject):
1522 1535
1523 for k in self.context_attributes: 1536 for k in self.context_attributes:
1524 if (other._get(k) is not None and 1537 if (other._get(k) is not None and
1525 k not in set(['tags'])): 1538 k not in set(['tags', 'requires', 'provides'])):
1526 setattr(self, k, other._get(k)) 1539 setattr(self, k, other._get(k))
1527 1540
1528 if other._get('tags') is not None: 1541 for k in ('tags', 'requires', 'provides'):
1529 self.tags = frozenset(self.tags.union(other.tags)) 1542 if other._get(k) is not None:
1543 setattr(self, k, getattr(self, k).union(other._get(k)))
1530 1544
1531 self.inheritance_path = self.inheritance_path + (repr(other),) 1545 self.inheritance_path = self.inheritance_path + (repr(other),)
1532 1546
@@ -1947,6 +1961,7 @@ class BuildSet(object):
1947 1961
1948 1962
1949class QueueItem(object): 1963class QueueItem(object):
1964
1950 """Represents the position of a Change in a ChangeQueue. 1965 """Represents the position of a Change in a ChangeQueue.
1951 1966
1952 All Changes are enqueued into ChangeQueue in a QueueItem. The QueueItem 1967 All Changes are enqueued into ChangeQueue in a QueueItem. The QueueItem
@@ -1973,6 +1988,7 @@ class QueueItem(object):
1973 self.layout = None 1988 self.layout = None
1974 self.project_pipeline_config = None 1989 self.project_pipeline_config = None
1975 self.job_graph = None 1990 self.job_graph = None
1991 self._cached_sql_results = None
1976 1992
1977 def __repr__(self): 1993 def __repr__(self):
1978 if self.pipeline: 1994 if self.pipeline:
@@ -2169,6 +2185,110 @@ class QueueItem(object):
2169 return False 2185 return False
2170 return self.item_ahead.isHoldingFollowingChanges() 2186 return self.item_ahead.isHoldingFollowingChanges()
2171 2187
2188 def _getRequirementsResultFromSQL(self, requirements):
2189 # This either returns data or raises an exception
2190 if self._cached_sql_results is None:
2191 sql_driver = self.pipeline.manager.sched.connections.drivers['sql']
2192 conn = sql_driver.tenant_connections.get(self.pipeline.tenant.name)
2193 if conn:
2194 builds = conn.getBuilds(
2195 tenant=self.pipeline.tenant.name,
2196 project=self.change.project.name,
2197 pipeline=self.pipeline.name,
2198 change=self.change.number,
2199 branch=self.change.branch,
2200 patchset=self.change.patchset,
2201 provides=list(requirements))
2202 else:
2203 builds = []
2204 # Just look at the most recent buildset.
2205 # TODO: query for a buildset instead of filtering.
2206 builds = [b for b in builds
2207 if b.buildset.uuid == builds[0].buildset.uuid]
2208 self._cached_sql_results = builds
2209
2210 builds = self._cached_sql_results
2211 data = []
2212 if not builds:
2213 return data
2214
2215 for build in builds:
2216 if build.result != 'SUCCESS':
2217 provides = [x.name for x in build.provides]
2218 requirement = list(requirements.intersection(set(provides)))
2219 raise RequirementsError(
2220 "Requirements %s not met by build %s" % (
2221 requirement, build.uuid))
2222 else:
2223 artifacts = [{'name': a.name,
2224 'url': a.url,
2225 'project': build.buildset.project,
2226 'change': str(build.buildset.change),
2227 'patchset': build.buildset.patchset,
2228 'job': build.job_name}
2229 for a in build.artifacts]
2230 data += artifacts
2231 return data
2232
2233 def providesRequirements(self, requirements, data):
2234 # Mutates data and returns true/false if requirements
2235 # satisfied.
2236 if not requirements:
2237 return True
2238 if not self.live:
2239 # Look for this item in other queues in the pipeline.
2240 item = None
2241 found = False
2242 for item in self.pipeline.getAllItems():
2243 if item.live and item.change == self.change:
2244 found = True
2245 break
2246 if found:
2247 if not item.providesRequirements(requirements, data):
2248 return False
2249 else:
2250 # Look for this item in the SQL DB.
2251 data += self._getRequirementsResultFromSQL(requirements)
2252 if self.hasJobGraph():
2253 for job in self.getJobs():
2254 if job.provides.intersection(requirements):
2255 build = self.current_build_set.getBuild(job.name)
2256 if not build:
2257 return False
2258 if build.result and build.result != 'SUCCESS':
2259 return False
2260 if not build.result and not build.paused:
2261 return False
2262 artifacts = get_artifacts_from_result_data(
2263 build.result_data,
2264 logger=self.log)
2265 artifacts = [{'name': a['name'],
2266 'url': a['url'],
2267 'project': self.change.project.name,
2268 'change': self.change.number,
2269 'patchset': self.change.patchset,
2270 'job': build.job.name}
2271 for a in artifacts]
2272 data += artifacts
2273 if not self.item_ahead:
2274 return True
2275 return self.item_ahead.providesRequirements(requirements, data)
2276
2277 def jobRequirementsReady(self, job):
2278 if not self.item_ahead:
2279 return True
2280 try:
2281 data = []
2282 ret = self.item_ahead.providesRequirements(job.requires, data)
2283 job.updateArtifactData(data)
2284 except RequirementsError as e:
2285 self.warning(str(e))
2286 fakebuild = Build(job, None)
2287 fakebuild.result = 'SKIPPED'
2288 self.addBuild(fakebuild)
2289 ret = True
2290 return ret
2291
2172 def findJobsToRun(self, semaphore_handler): 2292 def findJobsToRun(self, semaphore_handler):
2173 torun = [] 2293 torun = []
2174 if not self.live: 2294 if not self.live:
@@ -2196,6 +2316,8 @@ class QueueItem(object):
2196 for job in self.job_graph.getJobs(): 2316 for job in self.job_graph.getJobs():
2197 if job not in jobs_not_started: 2317 if job not in jobs_not_started:
2198 continue 2318 continue
2319 if not self.jobRequirementsReady(job):
2320 continue
2199 all_parent_jobs_successful = True 2321 all_parent_jobs_successful = True
2200 parent_builds_with_data = {} 2322 parent_builds_with_data = {}
2201 for parent_job in self.job_graph.getParentJobsRecursively( 2323 for parent_job in self.job_graph.getParentJobsRecursively(
@@ -2260,6 +2382,8 @@ class QueueItem(object):
2260 for job in self.job_graph.getJobs(): 2382 for job in self.job_graph.getJobs():
2261 if job not in jobs_not_requested: 2383 if job not in jobs_not_requested:
2262 continue 2384 continue
2385 if not self.jobRequirementsReady(job):
2386 continue
2263 all_parent_jobs_successful = True 2387 all_parent_jobs_successful = True
2264 for parent_job in self.job_graph.getParentJobsRecursively( 2388 for parent_job in self.job_graph.getParentJobsRecursively(
2265 job.name): 2389 job.name):
diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py
index fcdae36..dbbb707 100755
--- a/zuul/web/__init__.py
+++ b/zuul/web/__init__.py
@@ -440,6 +440,7 @@ class ZuulWebAPI(object):
440 'newrev': buildset.newrev, 440 'newrev': buildset.newrev,
441 'ref_url': buildset.ref_url, 441 'ref_url': buildset.ref_url,
442 'artifacts': [], 442 'artifacts': [],
443 'provides': [],
443 } 444 }
444 445
445 for artifact in build.artifacts: 446 for artifact in build.artifacts:
@@ -447,6 +448,10 @@ class ZuulWebAPI(object):
447 'name': artifact.name, 448 'name': artifact.name,
448 'url': artifact.url, 449 'url': artifact.url,
449 }) 450 })
451 for provides in build.provides:
452 ret['provides'].append({
453 'name': artifact.name,
454 })
450 return ret 455 return ret
451 456
452 @cherrypy.expose 457 @cherrypy.expose