summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2019-02-02 07:35:51 +0000
committerGerrit Code Review <review@openstack.org>2019-02-02 07:35:51 +0000
commitb44b6c532ce9a999b135d909ae9f9caf0671784d (patch)
tree2dbee7f7a0c46d06a51c8abf52e1b25b98a9f307
parentb4e6247ab5ac1b868e954f4675b1d531600bb667 (diff)
parent1317391323bee2d868b4f3ce30a4840fc6754cf1 (diff)
Merge "Add provides/requires support"
-rw-r--r--doc/source/user/config.rst51
-rw-r--r--doc/source/user/jobs.rst35
-rw-r--r--releasenotes/notes/provides_requires-4c6b54ede999e86c.yaml7
-rw-r--r--tests/base.py29
-rw-r--r--tests/fixtures/config/provides-requires-pause/git/common-config/zuul.yaml38
-rw-r--r--tests/fixtures/config/provides-requires-pause/git/org_project1/README1
-rw-r--r--tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-builder.yaml10
-rw-r--r--tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-user.yaml4
-rw-r--r--tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml26
-rw-r--r--tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml8
-rw-r--r--tests/fixtures/config/provides-requires-pause/main.yaml8
-rw-r--r--tests/fixtures/layouts/provides-requires-two-jobs.yaml72
-rw-r--r--tests/fixtures/layouts/provides-requires-unshared.yaml58
-rw-r--r--tests/fixtures/layouts/provides-requires.yaml70
-rw-r--r--tests/unit/test_v3.py288
-rwxr-xr-xtests/unit/test_web.py21
-rw-r--r--zuul/configloader.py11
-rw-r--r--zuul/driver/sql/alembic/versions/39d302d34d38_add_provides.py46
-rw-r--r--zuul/driver/sql/sqlconnection.py28
-rw-r--r--zuul/driver/sql/sqlreporter.py55
-rw-r--r--zuul/executor/client.py2
-rw-r--r--zuul/lib/artifacts.py69
-rw-r--r--zuul/model.py136
-rwxr-xr-xzuul/web/__init__.py5
24 files changed, 1019 insertions, 59 deletions
diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst
index 04de33e..c2053b3 100644
--- a/doc/source/user/config.rst
+++ b/doc/source/user/config.rst
@@ -686,6 +686,57 @@ Here is an example of two job definitions:
686 tags from all the jobs and variants used in constructing the 686 tags from all the jobs and variants used in constructing the
687 frozen job, with no duplication. 687 frozen job, with no duplication.
688 688
689 .. attr:: provides
690
691 A list of free-form strings which identifies resources provided
692 by this job which may be used by other jobs for other changes
693 using the :attr:`job.requires` attribute.
694
695 .. attr:: requires
696
697 A list of free-form strings which identify resources which may
698 be provided by other jobs for other changes (via the
699 :attr:`job.provides` attribute) that are used by this job.
700
701 When Zuul encounters a job with a `requires` attribute, it
702 searches for those values in the `provides` attributes of any
703 jobs associated with any queue items ahead of the current
704 change. In this way, if a change uses either git dependencies
705 or a `Depends-On` header to indicate a dependency on another
706 change, Zuul will be able to determine that the parent change
707 affects the run-time environment of the child change. If such a
708 relationship is found, the job with `requires` will not start
709 until all of the jobs with matching `provides` have completed or
710 paused. Additionally, the :ref:`artifacts <return_artifacts>`
711 returned by the `provides` jobs will be made available to the
712 `requires` job.
713
714 For example, a job which produces a builder container image in
715 one project that is then consumed by a container image build job
716 in another project might look like this:
717
718 .. code-block:: yaml
719
720 - job:
721 name: build-builder-image
722 provides: images
723
724 - job:
725 name: build-final-image
726 requires: images
727
728 - project:
729 name: builder-project
730 check:
731 jobs:
732 - build-builder-image
733
734 - project:
735 name: final-project
736 check:
737 jobs:
738 - build-final-image
739
689 .. attr:: secrets 740 .. attr:: secrets
690 741
691 A list of secrets which may be used by the job. A 742 A list of secrets which may be used by the job. A
diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst
index cc195e7..826ff34 100644
--- a/doc/source/user/jobs.rst
+++ b/doc/source/user/jobs.rst
@@ -228,6 +228,41 @@ of item.
228 All items provide the following information as Ansible variables 228 All items provide the following information as Ansible variables
229 under the ``zuul`` key: 229 under the ``zuul`` key:
230 230
231 .. var:: artifacts
232 :type: list
233
234 If the job has a :attr:`job.requires` attribute, and Zuul has
235 found changes ahead of this change in the pipeline with matching
236 :attr:`job.provides` attributes, then information about any
237 :ref:`artifacts returned <return_artifacts>` from those jobs
238 will appear here.
239
240 This value is a list of dictionaries with the following format:
241
242 .. var:: project
243
244 The name of the project which supplied this artifact.
245
246 .. var:: change
247
248 The change number which supplied this artifact.
249
250 .. var:: patchset
251
252 The patchset of the change.
253
254 .. var:: job
255
256 The name of the job which produced the artifact.
257
258 .. var:: name
259
260 The name of the artifact (as supplied to :ref:`return_artifacts`).
261
262 .. var:: url
263
264 The URL of the artifact (as supplied to :ref:`return_artifacts`).
265
231 .. var:: build 266 .. var:: build
232 267
233 The UUID of the build. A build is a single execution of a job. 268 The UUID of the build. A build is a single execution of a job.
diff --git a/releasenotes/notes/provides_requires-4c6b54ede999e86c.yaml b/releasenotes/notes/provides_requires-4c6b54ede999e86c.yaml
new file mode 100644
index 0000000..1990065
--- /dev/null
+++ b/releasenotes/notes/provides_requires-4c6b54ede999e86c.yaml
@@ -0,0 +1,7 @@
1---
2features:
3 - Support for expressing artifact or other resource dependencies
4 between jobs running on different changes with a dependency
5 relationship (e.g., a container image built in one project and
6 consumed in a second project) has been added via the
7 :attr:`job.provides` and :attr:`job.requires` job attributes.
diff --git a/tests/base.py b/tests/base.py
index a2f7c93..a8803c7 100644
--- a/tests/base.py
+++ b/tests/base.py
@@ -1355,6 +1355,11 @@ class FakeBuild(object):
1355 items = self.parameters['zuul']['items'] 1355 items = self.parameters['zuul']['items']
1356 self.changes = ' '.join(['%s,%s' % (x['change'], x['patchset']) 1356 self.changes = ' '.join(['%s,%s' % (x['change'], x['patchset'])
1357 for x in items if 'change' in x]) 1357 for x in items if 'change' in x])
1358 if 'change' in items[-1]:
1359 self.change = ' '.join((items[-1]['change'],
1360 items[-1]['patchset']))
1361 else:
1362 self.change = None
1358 1363
1359 def __repr__(self): 1364 def __repr__(self):
1360 waiting = '' 1365 waiting = ''
@@ -1401,6 +1406,8 @@ class FakeBuild(object):
1401 self._wait() 1406 self._wait()
1402 self.log.debug("Build %s continuing" % self.unique) 1407 self.log.debug("Build %s continuing" % self.unique)
1403 1408
1409 self.writeReturnData()
1410
1404 result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success 1411 result = (RecordingAnsibleJob.RESULT_NORMAL, 0) # Success
1405 if self.shouldFail(): 1412 if self.shouldFail():
1406 result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure 1413 result = (RecordingAnsibleJob.RESULT_NORMAL, 1) # Failure
@@ -1418,6 +1425,14 @@ class FakeBuild(object):
1418 return True 1425 return True
1419 return False 1426 return False
1420 1427
1428 def writeReturnData(self):
1429 changes = self.executor_server.return_data.get(self.name, {})
1430 data = changes.get(self.change)
1431 if data is None:
1432 return
1433 with open(self.jobdir.result_data_file, 'w') as f:
1434 f.write(json.dumps(data))
1435
1421 def hasChanges(self, *changes): 1436 def hasChanges(self, *changes):
1422 """Return whether this build has certain changes in its git repos. 1437 """Return whether this build has certain changes in its git repos.
1423 1438
@@ -1554,6 +1569,7 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
1554 self.running_builds = [] 1569 self.running_builds = []
1555 self.build_history = [] 1570 self.build_history = []
1556 self.fail_tests = {} 1571 self.fail_tests = {}
1572 self.return_data = {}
1557 self.job_builds = {} 1573 self.job_builds = {}
1558 1574
1559 def failJob(self, name, change): 1575 def failJob(self, name, change):
@@ -1569,6 +1585,19 @@ class RecordingExecutorServer(zuul.executor.server.ExecutorServer):
1569 l.append(change) 1585 l.append(change)
1570 self.fail_tests[name] = l 1586 self.fail_tests[name] = l
1571 1587
1588 def returnData(self, name, change, data):
1589 """Instruct the executor to return data for this build.
1590
1591 :arg str name: The name of the job to return data.
1592 :arg Change change: The :py:class:`~tests.base.FakeChange`
1593 instance which should cause the job to return data.
1594 :arg dict data: The data to return
1595
1596 """
1597 changes = self.return_data.setdefault(name, {})
1598 cid = ' '.join((str(change.number), str(change.latest_patchset)))
1599 changes[cid] = data
1600
1572 def release(self, regex=None): 1601 def release(self, regex=None):
1573 """Release a held build. 1602 """Release a held build.
1574 1603
diff --git a/tests/fixtures/config/provides-requires-pause/git/common-config/zuul.yaml b/tests/fixtures/config/provides-requires-pause/git/common-config/zuul.yaml
new file mode 100644
index 0000000..12901dd
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/git/common-config/zuul.yaml
@@ -0,0 +1,38 @@
1- pipeline:
2 name: check
3 manager: independent
4 post-review: true
5 trigger:
6 gerrit:
7 - event: patchset-created
8 success:
9 gerrit:
10 Verified: 1
11 failure:
12 gerrit:
13 Verified: -1
14
15- pipeline:
16 name: gate
17 manager: dependent
18 post-review: True
19 trigger:
20 gerrit:
21 - event: comment-added
22 approval:
23 - Approved: 1
24 success:
25 gerrit:
26 Verified: 2
27 submit: true
28 failure:
29 gerrit:
30 Verified: -2
31 start:
32 gerrit:
33 Verified: 0
34 precedence: high
35
36- job:
37 name: base
38 parent: null
diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/README b/tests/fixtures/config/provides-requires-pause/git/org_project1/README
new file mode 100644
index 0000000..9daeafb
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/README
@@ -0,0 +1 @@
test
diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-builder.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-builder.yaml
new file mode 100644
index 0000000..7d773b6
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-builder.yaml
@@ -0,0 +1,10 @@
1- hosts: all
2 tasks:
3 - name: Pause and let child run
4 zuul_return:
5 data:
6 zuul:
7 pause: true
8 artifacts:
9 - name: image
10 url: http://example.com/image
diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-user.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-user.yaml
new file mode 100644
index 0000000..583279c
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/playbooks/image-user.yaml
@@ -0,0 +1,4 @@
1- hosts: all
2 tasks:
3 - debug:
4 var: zuul.artifacts
diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml
new file mode 100644
index 0000000..412fe2c
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/git/org_project1/zuul.yaml
@@ -0,0 +1,26 @@
1- job:
2 name: image-builder
3 provides:
4 - image
5 run: playbooks/image-builder.yaml
6
7- job:
8 name: image-user
9 requires:
10 - image
11 run: playbooks/image-user.yaml
12
13- project:
14 check:
15 jobs:
16 - image-builder
17 - image-user:
18 dependencies:
19 - image-builder
20 gate:
21 queue: integrated
22 jobs:
23 - image-builder
24 - image-user:
25 dependencies:
26 - image-builder
diff --git a/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml b/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml
new file mode 100644
index 0000000..e9e6b58
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/git/org_project2/zuul.yaml
@@ -0,0 +1,8 @@
1- project:
2 check:
3 jobs:
4 - image-user
5 gate:
6 queue: integrated
7 jobs:
8 - image-user
diff --git a/tests/fixtures/config/provides-requires-pause/main.yaml b/tests/fixtures/config/provides-requires-pause/main.yaml
new file mode 100644
index 0000000..3a74155
--- /dev/null
+++ b/tests/fixtures/config/provides-requires-pause/main.yaml
@@ -0,0 +1,8 @@
1- tenant:
2 name: tenant-one
3 source:
4 gerrit:
5 config-projects:
6 - common-config
7 - org/project1
8 - org/project2
diff --git a/tests/fixtures/layouts/provides-requires-two-jobs.yaml b/tests/fixtures/layouts/provides-requires-two-jobs.yaml
new file mode 100644
index 0000000..6d1cc77
--- /dev/null
+++ b/tests/fixtures/layouts/provides-requires-two-jobs.yaml
@@ -0,0 +1,72 @@
1- pipeline:
2 name: check
3 manager: independent
4 trigger:
5 gerrit:
6 - event: patchset-created
7 success:
8 gerrit:
9 Verified: 1
10 resultsdb_mysql: null
11 resultsdb_postgresql: null
12 failure:
13 gerrit:
14 Verified: -1
15 resultsdb_mysql: null
16 resultsdb_postgresql: null
17
18- pipeline:
19 name: gate
20 manager: dependent
21 success-message: Build succeeded (gate).
22 trigger:
23 gerrit:
24 - event: comment-added
25 approval:
26 - Approved: 1
27 success:
28 gerrit:
29 Verified: 2
30 submit: true
31 failure:
32 gerrit:
33 Verified: -2
34 start:
35 gerrit:
36 Verified: 0
37 precedence: high
38
39- job:
40 name: base
41 parent: null
42 run: playbooks/base.yaml
43
44- job:
45 name: image-builder
46 provides: images
47
48- job:
49 name: image-user
50 requires: images
51
52- project:
53 name: org/project1
54 check:
55 jobs:
56 - image-builder
57 gate:
58 queue: integrated
59 jobs:
60 - image-builder
61 - image-user:
62 dependencies: image-builder
63
64- project:
65 name: org/project2
66 check:
67 jobs:
68 - image-user
69 gate:
70 queue: integrated
71 jobs:
72 - image-user
diff --git a/tests/fixtures/layouts/provides-requires-unshared.yaml b/tests/fixtures/layouts/provides-requires-unshared.yaml
new file mode 100644
index 0000000..65df1bc
--- /dev/null
+++ b/tests/fixtures/layouts/provides-requires-unshared.yaml
@@ -0,0 +1,58 @@
1- pipeline:
2 name: check
3 manager: independent
4 trigger:
5 gerrit:
6 - event: patchset-created
7 success:
8 gerrit:
9 Verified: 1
10 failure:
11 gerrit:
12 Verified: -1
13
14- pipeline:
15 name: gate
16 manager: dependent
17 success-message: Build succeeded (gate).
18 trigger:
19 gerrit:
20 - event: comment-added
21 approval:
22 - Approved: 1
23 success:
24 gerrit:
25 Verified: 2
26 submit: true
27 failure:
28 gerrit:
29 Verified: -2
30 start:
31 gerrit:
32 Verified: 0
33 precedence: high
34
35- job:
36 name: base
37 parent: null
38 run: playbooks/base.yaml
39
40- job:
41 name: image-builder
42 provides: images
43
44- job:
45 name: image-user
46 requires: images
47
48- project:
49 name: org/project1
50 gate:
51 jobs:
52 - image-builder
53
54- project:
55 name: org/project2
56 gate:
57 jobs:
58 - image-user
diff --git a/tests/fixtures/layouts/provides-requires.yaml b/tests/fixtures/layouts/provides-requires.yaml
new file mode 100644
index 0000000..3947345
--- /dev/null
+++ b/tests/fixtures/layouts/provides-requires.yaml
@@ -0,0 +1,70 @@
1- pipeline:
2 name: check
3 manager: independent
4 trigger:
5 gerrit:
6 - event: patchset-created
7 success:
8 gerrit:
9 Verified: 1
10 resultsdb_mysql: null
11 resultsdb_postgresql: null
12 failure:
13 gerrit:
14 Verified: -1
15 resultsdb_mysql: null
16 resultsdb_postgresql: null
17
18- pipeline:
19 name: gate
20 manager: dependent
21 success-message: Build succeeded (gate).
22 trigger:
23 gerrit:
24 - event: comment-added
25 approval:
26 - Approved: 1
27 success:
28 gerrit:
29 Verified: 2
30 submit: true
31 failure:
32 gerrit:
33 Verified: -2
34 start:
35 gerrit:
36 Verified: 0
37 precedence: high
38
39- job:
40 name: base
41 parent: null
42 run: playbooks/base.yaml
43
44- job:
45 name: image-builder
46 provides: images
47
48- job:
49 name: image-user
50 requires: images
51
52- project:
53 name: org/project1
54 check:
55 jobs:
56 - image-builder
57 gate:
58 queue: integrated
59 jobs:
60 - image-builder
61
62- project:
63 name: org/project2
64 check:
65 jobs:
66 - image-user
67 gate:
68 queue: integrated
69 jobs:
70 - image-user
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index d6554a0..4ae89fd 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -28,6 +28,7 @@ from zuul.lib import encryption
28from tests.base import ( 28from tests.base import (
29 AnsibleZuulTestCase, 29 AnsibleZuulTestCase,
30 ZuulTestCase, 30 ZuulTestCase,
31 ZuulDBTestCase,
31 FIXTURE_DIR, 32 FIXTURE_DIR,
32 simple_layout, 33 simple_layout,
33) 34)
@@ -4714,3 +4715,290 @@ class TestContainerJobs(AnsibleZuulTestCase):
4714 dict(name='container-machine', result='SUCCESS', changes='1,1'), 4715 dict(name='container-machine', result='SUCCESS', changes='1,1'),
4715 dict(name='container-native', result='SUCCESS', changes='1,1'), 4716 dict(name='container-native', result='SUCCESS', changes='1,1'),
4716 ]) 4717 ])
4718
4719
4720class TestProvidesRequiresPause(AnsibleZuulTestCase):
4721 tenant_config_file = "config/provides-requires-pause/main.yaml"
4722
4723 def test_provides_requires_pause(self):
4724 # Changes share a queue, with both running at the same time.
4725 self.executor_server.hold_jobs_in_build = True
4726 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4727 A.addApproval('Code-Review', 2)
4728 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4729 self.waitUntilSettled()
4730
4731 self.assertEqual(len(self.builds), 1)
4732
4733 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4734 B.addApproval('Code-Review', 2)
4735 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4736 self.waitUntilSettled()
4737
4738 self.assertEqual(len(self.builds), 1)
4739
4740 # Release image-build, it should cause both instances of
4741 # image-user to run.
4742 self.executor_server.hold_jobs_in_build = False
4743 self.executor_server.release()
4744 self.waitUntilSettled()
4745
4746 self.assertHistory([
4747 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4748 dict(name='image-user', result='SUCCESS', changes='1,1'),
4749 dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4750 ], ordered=False)
4751 build = self.getJobFromHistory('image-user', project='org/project2')
4752 self.assertEqual(
4753 build.parameters['zuul']['artifacts'],
4754 [{
4755 'project': 'org/project1',
4756 'change': '1',
4757 'patchset': '1',
4758 'job': 'image-builder',
4759 'url': 'http://example.com/image',
4760 'name': 'image',
4761 }])
4762
4763
4764class TestProvidesRequires(ZuulDBTestCase):
4765 config_file = "zuul-sql-driver.conf"
4766
4767 @simple_layout('layouts/provides-requires.yaml')
4768 def test_provides_requires_shared_queue_fast(self):
4769 # Changes share a queue, but with only one job, the first
4770 # merges before the second starts.
4771 self.executor_server.hold_jobs_in_build = True
4772 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4773 self.executor_server.returnData(
4774 'image-builder', A,
4775 {'zuul':
4776 {'artifacts': [
4777 {'name': 'image', 'url': 'http://example.com/image'},
4778 ]}}
4779 )
4780 A.addApproval('Code-Review', 2)
4781 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4782 self.waitUntilSettled()
4783
4784 self.assertEqual(len(self.builds), 1)
4785
4786 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4787 B.addApproval('Code-Review', 2)
4788 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4789 self.waitUntilSettled()
4790
4791 self.assertEqual(len(self.builds), 1)
4792
4793 self.executor_server.hold_jobs_in_build = False
4794 self.executor_server.release()
4795 self.waitUntilSettled()
4796
4797 self.assertHistory([
4798 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4799 dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4800 ])
4801 # Data are not passed in this instance because the builder
4802 # change merges before the user job runs.
4803 self.assertFalse('artifacts' in self.history[-1].parameters['zuul'])
4804
4805 @simple_layout('layouts/provides-requires-two-jobs.yaml')
4806 def test_provides_requires_shared_queue_slow(self):
4807 # Changes share a queue, with both running at the same time.
4808 self.executor_server.hold_jobs_in_build = True
4809 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4810 self.executor_server.returnData(
4811 'image-builder', A,
4812 {'zuul':
4813 {'artifacts': [
4814 {'name': 'image', 'url': 'http://example.com/image'},
4815 ]}}
4816 )
4817 A.addApproval('Code-Review', 2)
4818 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4819 self.waitUntilSettled()
4820
4821 self.assertEqual(len(self.builds), 1)
4822
4823 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4824 B.addApproval('Code-Review', 2)
4825 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4826 self.waitUntilSettled()
4827
4828 self.assertEqual(len(self.builds), 1)
4829
4830 # Release image-build, it should cause both instances of
4831 # image-user to run.
4832 self.executor_server.release()
4833 self.waitUntilSettled()
4834 self.assertEqual(len(self.builds), 2)
4835 self.assertHistory([
4836 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4837 ])
4838
4839 self.orderedRelease()
4840 self.waitUntilSettled()
4841
4842 self.assertHistory([
4843 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4844 dict(name='image-user', result='SUCCESS', changes='1,1'),
4845 dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4846 ])
4847 self.assertEqual(
4848 self.history[-1].parameters['zuul']['artifacts'],
4849 [{
4850 'project': 'org/project1',
4851 'change': '1',
4852 'patchset': '1',
4853 'job': 'image-builder',
4854 'url': 'http://example.com/image',
4855 'name': 'image',
4856 }])
4857
4858 @simple_layout('layouts/provides-requires-unshared.yaml')
4859 def test_provides_requires_unshared_queue(self):
4860 self.executor_server.hold_jobs_in_build = True
4861 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4862 self.executor_server.returnData(
4863 'image-builder', A,
4864 {'zuul':
4865 {'artifacts': [
4866 {'name': 'image', 'url': 'http://example.com/image'},
4867 ]}}
4868 )
4869 A.addApproval('Code-Review', 2)
4870 self.fake_gerrit.addEvent(A.addApproval('Approved', 1))
4871 self.waitUntilSettled()
4872
4873 self.assertEqual(len(self.builds), 1)
4874
4875 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4876 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4877 B.subject, A.data['id'])
4878 B.addApproval('Code-Review', 2)
4879 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4880 self.waitUntilSettled()
4881
4882 self.assertEqual(len(self.builds), 1)
4883
4884 self.executor_server.hold_jobs_in_build = False
4885 self.executor_server.release()
4886 self.waitUntilSettled()
4887
4888 self.assertHistory([
4889 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4890 ])
4891
4892 self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
4893 self.waitUntilSettled()
4894
4895 self.assertHistory([
4896 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4897 dict(name='image-user', result='SUCCESS', changes='2,1'),
4898 ])
4899 # Data are not passed in this instance because the builder
4900 # change merges before the user job runs.
4901 self.assertFalse('artifacts' in self.history[-1].parameters['zuul'])
4902
4903 @simple_layout('layouts/provides-requires.yaml')
4904 def test_provides_requires_check_current(self):
4905 self.executor_server.hold_jobs_in_build = True
4906 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4907 self.executor_server.returnData(
4908 'image-builder', A,
4909 {'zuul':
4910 {'artifacts': [
4911 {'name': 'image', 'url': 'http://example.com/image'},
4912 ]}}
4913 )
4914 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4915 self.waitUntilSettled()
4916
4917 self.assertEqual(len(self.builds), 1)
4918
4919 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4920 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4921 B.subject, A.data['id'])
4922 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4923 self.waitUntilSettled()
4924
4925 self.assertEqual(len(self.builds), 1)
4926
4927 self.executor_server.hold_jobs_in_build = False
4928 self.executor_server.release()
4929 self.waitUntilSettled()
4930
4931 self.assertHistory([
4932 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4933 dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4934 ])
4935 self.assertEqual(
4936 self.history[-1].parameters['zuul']['artifacts'],
4937 [{
4938 'project': 'org/project1',
4939 'change': '1',
4940 'patchset': '1',
4941 'job': 'image-builder',
4942 'url': 'http://example.com/image',
4943 'name': 'image',
4944 }])
4945
4946 @simple_layout('layouts/provides-requires.yaml')
4947 def test_provides_requires_check_old_success(self):
4948 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4949 self.executor_server.returnData(
4950 'image-builder', A,
4951 {'zuul':
4952 {'artifacts': [
4953 {'name': 'image', 'url': 'http://example.com/image'},
4954 ]}}
4955 )
4956 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4957 self.waitUntilSettled()
4958 self.assertHistory([
4959 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4960 ])
4961
4962 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4963 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4964 B.subject, A.data['id'])
4965 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4966 self.waitUntilSettled()
4967
4968 self.assertHistory([
4969 dict(name='image-builder', result='SUCCESS', changes='1,1'),
4970 dict(name='image-user', result='SUCCESS', changes='1,1 2,1'),
4971 ])
4972 self.assertEqual(
4973 self.history[-1].parameters['zuul']['artifacts'],
4974 [{
4975 'project': 'org/project1',
4976 'change': '1',
4977 'patchset': '1',
4978 'job': 'image-builder',
4979 'url': 'http://example.com/image',
4980 'name': 'image',
4981 }])
4982
4983 @simple_layout('layouts/provides-requires.yaml')
4984 def test_provides_requires_check_old_failure(self):
4985 A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A')
4986 self.executor_server.failJob('image-builder', A)
4987 self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
4988 self.waitUntilSettled()
4989
4990 self.assertHistory([
4991 dict(name='image-builder', result='FAILURE', changes='1,1'),
4992 ])
4993
4994 B = self.fake_gerrit.addFakeChange('org/project2', 'master', 'B')
4995 B.data['commitMessage'] = '%s\n\nDepends-On: %s\n' % (
4996 B.subject, A.data['id'])
4997 self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1))
4998 self.waitUntilSettled()
4999
5000 self.assertHistory([
5001 dict(name='image-builder', result='FAILURE', changes='1,1'),
5002 ])
5003 self.assertIn('image-user : SKIPPED', B.messages[0])
5004 self.assertIn('not met by build', B.messages[0])
diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py
index 05701a6..e267734 100755
--- a/tests/unit/test_web.py
+++ b/tests/unit/test_web.py
@@ -305,10 +305,13 @@ class TestWeb(BaseTestWeb):
305 'parent': 'base', 305 'parent': 'base',
306 'post_review': None, 306 'post_review': None,
307 'protected': None, 307 'protected': None,
308 'provides': [],
308 'required_projects': [], 309 'required_projects': [],
310 'requires': [],
309 'roles': [common_config_role], 311 'roles': [common_config_role],
310 'semaphore': None, 312 'semaphore': None,
311 'source_context': source_ctx, 313 'source_context': source_ctx,
314 'tags': [],
312 'timeout': None, 315 'timeout': None,
313 'variables': {}, 316 'variables': {},
314 'variant_description': '', 317 'variant_description': '',
@@ -337,10 +340,13 @@ class TestWeb(BaseTestWeb):
337 'parent': 'base', 340 'parent': 'base',
338 'post_review': None, 341 'post_review': None,
339 'protected': None, 342 'protected': None,
343 'provides': [],
340 'required_projects': [], 344 'required_projects': [],
345 'requires': [],
341 'roles': [common_config_role], 346 'roles': [common_config_role],
342 'semaphore': None, 347 'semaphore': None,
343 'source_context': source_ctx, 348 'source_context': source_ctx,
349 'tags': [],
344 'timeout': None, 350 'timeout': None,
345 'variables': {}, 351 'variables': {},
346 'variant_description': 'stable', 352 'variant_description': 'stable',
@@ -363,13 +369,16 @@ class TestWeb(BaseTestWeb):
363 'parent': 'base', 369 'parent': 'base',
364 'post_review': None, 370 'post_review': None,
365 'protected': None, 371 'protected': None,
372 'provides': [],
366 'required_projects': [ 373 'required_projects': [
367 {'override_branch': None, 374 {'override_branch': None,
368 'override_checkout': None, 375 'override_checkout': None,
369 'project_name': 'review.example.com/org/project'}], 376 'project_name': 'review.example.com/org/project'}],
377 'requires': [],
370 'roles': [common_config_role], 378 'roles': [common_config_role],
371 'semaphore': None, 379 'semaphore': None,
372 'source_context': source_ctx, 380 'source_context': source_ctx,
381 'tags': [],
373 'timeout': None, 382 'timeout': None,
374 'variables': {}, 383 'variables': {},
375 'variant_description': '', 384 'variant_description': '',
@@ -434,13 +443,16 @@ class TestWeb(BaseTestWeb):
434 'parent': 'base', 443 'parent': 'base',
435 'post_review': None, 444 'post_review': None,
436 'protected': None, 445 'protected': None,
446 'provides': [],
437 'required_projects': [], 447 'required_projects': [],
448 'requires': [],
438 'roles': [], 449 'roles': [],
439 'semaphore': None, 450 'semaphore': None,
440 'source_context': { 451 'source_context': {
441 'branch': 'master', 452 'branch': 'master',
442 'path': 'zuul.yaml', 453 'path': 'zuul.yaml',
443 'project': 'common-config'}, 454 'project': 'common-config'},
455 'tags': [],
444 'timeout': None, 456 'timeout': None,
445 'variables': {}, 457 'variables': {},
446 'variant_description': '', 458 'variant_description': '',
@@ -458,13 +470,16 @@ class TestWeb(BaseTestWeb):
458 'parent': 'base', 470 'parent': 'base',
459 'post_review': None, 471 'post_review': None,
460 'protected': None, 472 'protected': None,
473 'provides': [],
461 'required_projects': [], 474 'required_projects': [],
475 'requires': [],
462 'roles': [], 476 'roles': [],
463 'semaphore': None, 477 'semaphore': None,
464 'source_context': { 478 'source_context': {
465 'branch': 'master', 479 'branch': 'master',
466 'path': 'zuul.yaml', 480 'path': 'zuul.yaml',
467 'project': 'common-config'}, 481 'project': 'common-config'},
482 'tags': [],
468 'timeout': None, 483 'timeout': None,
469 'variables': {}, 484 'variables': {},
470 'variant_description': '', 485 'variant_description': '',
@@ -482,13 +497,16 @@ class TestWeb(BaseTestWeb):
482 'parent': 'base', 497 'parent': 'base',
483 'post_review': None, 498 'post_review': None,
484 'protected': None, 499 'protected': None,
500 'provides': [],
485 'required_projects': [], 501 'required_projects': [],
502 'requires': [],
486 'roles': [], 503 'roles': [],
487 'semaphore': None, 504 'semaphore': None,
488 'source_context': { 505 'source_context': {
489 'branch': 'master', 506 'branch': 'master',
490 'path': 'zuul.yaml', 507 'path': 'zuul.yaml',
491 'project': 'common-config'}, 508 'project': 'common-config'},
509 'tags': [],
492 'timeout': None, 510 'timeout': None,
493 'variables': {}, 511 'variables': {},
494 'variant_description': '', 512 'variant_description': '',
@@ -506,13 +524,16 @@ class TestWeb(BaseTestWeb):
506 'parent': 'base', 524 'parent': 'base',
507 'post_review': None, 525 'post_review': None,
508 'protected': None, 526 'protected': None,
527 'provides': [],
509 'required_projects': [], 528 'required_projects': [],
529 'requires': [],
510 'roles': [], 530 'roles': [],
511 'semaphore': None, 531 'semaphore': None,
512 'source_context': { 532 'source_context': {
513 'branch': 'master', 533 'branch': 'master',
514 'path': 'zuul.yaml', 534 'path': 'zuul.yaml',
515 'project': 'common-config'}, 535 'project': 'common-config'},
536 'tags': [],
516 'timeout': None, 537 'timeout': None,
517 'variables': {}, 538 'variables': {},
518 'variant_description': '', 539 'variant_description': '',
diff --git a/zuul/configloader.py b/zuul/configloader.py
index 64318e2..5e4afc0 100644
--- a/zuul/configloader.py
+++ b/zuul/configloader.py
@@ -545,6 +545,8 @@ class JobParser(object):
545 'final': bool, 545 'final': bool,
546 'abstract': bool, 546 'abstract': bool,
547 'protected': bool, 547 'protected': bool,
548 'requires': to_list(str),
549 'provides': to_list(str),
548 'failure-message': str, 550 'failure-message': str,
549 'success-message': str, 551 'success-message': str,
550 'failure-url': str, 552 'failure-url': str,
@@ -769,11 +771,10 @@ class JobParser(object):
769 semaphore.get('name'), 771 semaphore.get('name'),
770 semaphore.get('resources-first', False)) 772 semaphore.get('resources-first', False))
771 773
772 tags = conf.get('tags') 774 for k in ('tags', 'requires', 'provides', 'dependencies'):
773 if tags: 775 v = frozenset(as_list(conf.get(k)))
774 job.tags = set(tags) 776 if v:
775 777 setattr(job, k, v)
776 job.dependencies = frozenset(as_list(conf.get('dependencies')))
777 778
778 variables = conf.get('vars', None) 779 variables = conf.get('vars', None)
779 if variables: 780 if variables:
diff --git a/zuul/driver/sql/alembic/versions/39d302d34d38_add_provides.py b/zuul/driver/sql/alembic/versions/39d302d34d38_add_provides.py
new file mode 100644
index 0000000..043b0c8
--- /dev/null
+++ b/zuul/driver/sql/alembic/versions/39d302d34d38_add_provides.py
@@ -0,0 +1,46 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13"""add_provides
14
15Revision ID: 39d302d34d38
16Revises: 649ce63b5fe5
17Create Date: 2019-01-28 15:01:07.408072
18
19"""
20
21# revision identifiers, used by Alembic.
22revision = '39d302d34d38'
23down_revision = '649ce63b5fe5'
24branch_labels = None
25depends_on = None
26
27from alembic import op
28import sqlalchemy as sa
29
30
31PROVIDES_TABLE = 'zuul_provides'
32BUILD_TABLE = 'zuul_build'
33
34
35def upgrade(table_prefix=''):
36 op.create_table(
37 table_prefix + PROVIDES_TABLE,
38 sa.Column('id', sa.Integer, primary_key=True),
39 sa.Column('build_id', sa.Integer,
40 sa.ForeignKey(table_prefix + BUILD_TABLE + ".id")),
41 sa.Column('name', sa.String(255)),
42 )
43
44
45def downgrade():
46 raise Exception("Downgrades not supported")
diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py
index 1a0efb5..5cf79e8 100644
--- a/zuul/driver/sql/sqlconnection.py
+++ b/zuul/driver/sql/sqlconnection.py
@@ -28,6 +28,7 @@ from zuul.connection import BaseConnection
28BUILDSET_TABLE = 'zuul_buildset' 28BUILDSET_TABLE = 'zuul_buildset'
29BUILD_TABLE = 'zuul_build' 29BUILD_TABLE = 'zuul_build'
30ARTIFACT_TABLE = 'zuul_artifact' 30ARTIFACT_TABLE = 'zuul_artifact'
31PROVIDES_TABLE = 'zuul_provides'
31 32
32 33
33class DatabaseSession(object): 34class DatabaseSession(object):
@@ -56,17 +57,21 @@ class DatabaseSession(object):
56 def getBuilds(self, tenant=None, project=None, pipeline=None, 57 def getBuilds(self, tenant=None, project=None, pipeline=None,
57 change=None, branch=None, patchset=None, ref=None, 58 change=None, branch=None, patchset=None, ref=None,
58 newrev=None, uuid=None, job_name=None, voting=None, 59 newrev=None, uuid=None, job_name=None, voting=None,
59 node_name=None, result=None, limit=50, offset=0): 60 node_name=None, result=None, provides=None,
61 limit=50, offset=0):
60 62
61 build_table = self.connection.zuul_build_table 63 build_table = self.connection.zuul_build_table
62 buildset_table = self.connection.zuul_buildset_table 64 buildset_table = self.connection.zuul_buildset_table
65 provides_table = self.connection.zuul_provides_table
63 66
64 # contains_eager allows us to perform eager loading on the 67 # contains_eager allows us to perform eager loading on the
65 # buildset *and* use that table in filters (unlike 68 # buildset *and* use that table in filters (unlike
66 # joinedload). 69 # joinedload).
67 q = self.session().query(self.connection.buildModel).\ 70 q = self.session().query(self.connection.buildModel).\
68 join(self.connection.buildSetModel).\ 71 join(self.connection.buildSetModel).\
72 outerjoin(self.connection.providesModel).\
69 options(orm.contains_eager(self.connection.buildModel.buildset), 73 options(orm.contains_eager(self.connection.buildModel.buildset),
74 orm.selectinload(self.connection.buildModel.provides),
70 orm.selectinload(self.connection.buildModel.artifacts)).\ 75 orm.selectinload(self.connection.buildModel.artifacts)).\
71 with_hint(build_table, 'USE INDEX (PRIMARY)', 'mysql') 76 with_hint(build_table, 'USE INDEX (PRIMARY)', 'mysql')
72 77
@@ -83,6 +88,7 @@ class DatabaseSession(object):
83 q = self.listFilter(q, build_table.c.voting, voting) 88 q = self.listFilter(q, build_table.c.voting, voting)
84 q = self.listFilter(q, build_table.c.node_name, node_name) 89 q = self.listFilter(q, build_table.c.node_name, node_name)
85 q = self.listFilter(q, build_table.c.result, result) 90 q = self.listFilter(q, build_table.c.result, result)
91 q = self.listFilter(q, provides_table.c.name, provides)
86 92
87 q = q.order_by(build_table.c.id.desc()).\ 93 q = q.order_by(build_table.c.id.desc()).\
88 limit(limit).\ 94 limit(limit).\
@@ -224,6 +230,15 @@ class SQLConnection(BaseConnection):
224 session.flush() 230 session.flush()
225 return a 231 return a
226 232
233 def createProvides(self, *args, **kw):
234 session = orm.session.Session.object_session(self)
235 p = ProvidesModel(*args, **kw)
236 p.build_id = self.id
237 self.provides.append(p)
238 session.add(p)
239 session.flush()
240 return p
241
227 class ArtifactModel(Base): 242 class ArtifactModel(Base):
228 __tablename__ = self.table_prefix + ARTIFACT_TABLE 243 __tablename__ = self.table_prefix + ARTIFACT_TABLE
229 id = sa.Column(sa.Integer, primary_key=True) 244 id = sa.Column(sa.Integer, primary_key=True)
@@ -233,6 +248,17 @@ class SQLConnection(BaseConnection):
233 url = sa.Column(sa.TEXT()) 248 url = sa.Column(sa.TEXT())
234 build = orm.relationship(BuildModel, backref="artifacts") 249 build = orm.relationship(BuildModel, backref="artifacts")
235 250
251 class ProvidesModel(Base):
252 __tablename__ = self.table_prefix + PROVIDES_TABLE
253 id = sa.Column(sa.Integer, primary_key=True)
254 build_id = sa.Column(sa.Integer, sa.ForeignKey(
255 self.table_prefix + BUILD_TABLE + ".id"))
256 name = sa.Column(sa.String(255))
257 build = orm.relationship(BuildModel, backref="provides")
258
259 self.providesModel = ProvidesModel
260 self.zuul_provides_table = self.providesModel.__table__
261
236 self.artifactModel = ArtifactModel 262 self.artifactModel = ArtifactModel
237 self.zuul_artifact_table = self.artifactModel.__table__ 263 self.zuul_artifact_table = self.artifactModel.__table__
238 264
diff --git a/zuul/driver/sql/sqlreporter.py b/zuul/driver/sql/sqlreporter.py
index 1e148d6..16651e4 100644
--- a/zuul/driver/sql/sqlreporter.py
+++ b/zuul/driver/sql/sqlreporter.py
@@ -16,9 +16,9 @@ import datetime
16import logging 16import logging
17import time 17import time
18import voluptuous as v 18import voluptuous as v
19import urllib.parse
20 19
21from zuul.reporter import BaseReporter 20from zuul.reporter import BaseReporter
21from zuul.lib.artifacts import get_artifacts_from_result_data
22 22
23 23
24class SQLReporter(BaseReporter): 24class SQLReporter(BaseReporter):
@@ -27,26 +27,6 @@ class SQLReporter(BaseReporter):
27 name = 'sql' 27 name = 'sql'
28 log = logging.getLogger("zuul.SQLReporter") 28 log = logging.getLogger("zuul.SQLReporter")
29 29
30 artifact = {
31 'name': str,
32 'url': str,
33 }
34 zuul_data = {
35 'zuul': {
36 'log_url': str,
37 'artifacts': [artifact],
38 v.Extra: object,
39 }
40 }
41 artifact_schema = v.Schema(zuul_data)
42
43 def validateArtifactSchema(self, data):
44 try:
45 self.artifact_schema(data)
46 except Exception:
47 return False
48 return True
49
50 def report(self, item): 30 def report(self, item):
51 """Create an entry into a database.""" 31 """Create an entry into a database."""
52 32
@@ -104,32 +84,13 @@ class SQLReporter(BaseReporter):
104 node_name=build.node_name, 84 node_name=build.node_name,
105 ) 85 )
106 86
107 if self.validateArtifactSchema(build.result_data): 87 for provides in job.provides:
108 artifacts = build.result_data.get('zuul', {}).get( 88 db_build.createProvides(name=provides)
109 'artifacts', []) 89
110 default_url = build.result_data.get('zuul', {}).get( 90 for artifact in get_artifacts_from_result_data(
111 'log_url') 91 build.result_data,
112 if default_url: 92 logger=self.log):
113 if default_url[-1] != '/': 93 db_build.createArtifact(**artifact)
114 default_url += '/'
115 for artifact in artifacts:
116 url = artifact['url']
117 if default_url:
118 # If the artifact url is relative, it will
119 # be combined with the log_url; if it is
120 # absolute, it will replace it.
121 try:
122 url = urllib.parse.urljoin(default_url, url)
123 except Exception:
124 self.log.debug("Error parsing URL:",
125 exc_info=1)
126 db_build.createArtifact(
127 name=artifact['name'],
128 url=url,
129 )
130 else:
131 self.log.debug("Result data did not pass artifact schema "
132 "validation: %s", build.result_data)
133 94
134 95
135def getSchema(): 96def getSchema():
diff --git a/zuul/executor/client.py b/zuul/executor/client.py
index 00ea8eb..801726d 100644
--- a/zuul/executor/client.py
+++ b/zuul/executor/client.py
@@ -165,6 +165,8 @@ class ExecutorClient(object):
165 timeout=job.timeout, 165 timeout=job.timeout,
166 jobtags=sorted(job.tags), 166 jobtags=sorted(job.tags),
167 _inheritance_path=list(job.inheritance_path)) 167 _inheritance_path=list(job.inheritance_path))
168 if job.artifact_data:
169 zuul_params['artifacts'] = job.artifact_data
168 if job.override_checkout: 170 if job.override_checkout:
169 zuul_params['override_checkout'] = job.override_checkout 171 zuul_params['override_checkout'] = job.override_checkout
170 if hasattr(item.change, 'branch'): 172 if hasattr(item.change, 'branch'):
diff --git a/zuul/lib/artifacts.py b/zuul/lib/artifacts.py
new file mode 100644
index 0000000..c7c2fe0
--- /dev/null
+++ b/zuul/lib/artifacts.py
@@ -0,0 +1,69 @@
1# Copyright 2018-2019 Red Hat, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import voluptuous as v
16import urllib.parse
17
18artifact = {
19 'name': str,
20 'url': str,
21}
22
23zuul_data = {
24 'zuul': {
25 'log_url': str,
26 'artifacts': [artifact],
27 v.Extra: object,
28 }
29}
30
31artifact_schema = v.Schema(zuul_data)
32
33
34def validate_artifact_schema(data):
35 try:
36 artifact_schema(data)
37 except Exception:
38 return False
39 return True
40
41
42def get_artifacts_from_result_data(result_data, logger=None):
43 ret = []
44 if validate_artifact_schema(result_data):
45 artifacts = result_data.get('zuul', {}).get(
46 'artifacts', [])
47 default_url = result_data.get('zuul', {}).get(
48 'log_url')
49 if default_url:
50 if default_url[-1] != '/':
51 default_url += '/'
52 for artifact in artifacts:
53 url = artifact['url']
54 if default_url:
55 # If the artifact url is relative, it will be combined
56 # with the log_url; if it is absolute, it will replace
57 # it.
58 try:
59 url = urllib.parse.urljoin(default_url, url)
60 except Exception:
61 if logger:
62 logger.debug("Error parsing URL:",
63 exc_info=1)
64 ret.append({'name': artifact['name'],
65 'url': url})
66 else:
67 logger.debug("Result data did not pass artifact schema "
68 "validation: %s", result_data)
69 return ret
diff --git a/zuul/model.py b/zuul/model.py
index ac589c2..5a842df 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -28,6 +28,7 @@ import itertools
28 28
29from zuul import change_matcher 29from zuul import change_matcher
30from zuul.lib.config import get_default 30from zuul.lib.config import get_default
31from zuul.lib.artifacts import get_artifacts_from_result_data
31 32
32MERGER_MERGE = 1 # "git merge" 33MERGER_MERGE = 1 # "git merge"
33MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve" 34MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve"
@@ -164,6 +165,11 @@ class TemplateNotFoundError(Exception):
164 pass 165 pass
165 166
166 167
168class RequirementsError(Exception):
169 """A job's requirements were not met."""
170 pass
171
172
167class Attributes(object): 173class Attributes(object):
168 """A class to hold attributes for string formatting.""" 174 """A class to hold attributes for string formatting."""
169 175
@@ -1070,6 +1076,8 @@ class Job(ConfigObject):
1070 file_matcher=None, 1076 file_matcher=None,
1071 irrelevant_file_matcher=None, # skip-if 1077 irrelevant_file_matcher=None, # skip-if
1072 tags=frozenset(), 1078 tags=frozenset(),
1079 provides=frozenset(),
1080 requires=frozenset(),
1073 dependencies=frozenset(), 1081 dependencies=frozenset(),
1074 ) 1082 )
1075 1083
@@ -1111,6 +1119,7 @@ class Job(ConfigObject):
1111 start_mark=None, 1119 start_mark=None,
1112 inheritance_path=(), 1120 inheritance_path=(),
1113 parent_data=None, 1121 parent_data=None,
1122 artifact_data=None,
1114 description=None, 1123 description=None,
1115 variant_description=None, 1124 variant_description=None,
1116 protected_origin=None, 1125 protected_origin=None,
@@ -1161,6 +1170,10 @@ class Job(ConfigObject):
1161 d['protected'] = self.protected 1170 d['protected'] = self.protected
1162 d['voting'] = self.voting 1171 d['voting'] = self.voting
1163 d['timeout'] = self.timeout 1172 d['timeout'] = self.timeout
1173 d['tags'] = list(self.tags)
1174 d['provides'] = list(self.provides)
1175 d['requires'] = list(self.requires)
1176 d['dependencies'] = list(self.dependencies)
1164 d['attempts'] = self.attempts 1177 d['attempts'] = self.attempts
1165 d['roles'] = list(map(lambda x: x.toDict(), self.roles)) 1178 d['roles'] = list(map(lambda x: x.toDict(), self.roles))
1166 d['post_review'] = self.post_review 1179 d['post_review'] = self.post_review
@@ -1170,9 +1183,6 @@ class Job(ConfigObject):
1170 d['parent'] = self.parent 1183 d['parent'] = self.parent
1171 else: 1184 else:
1172 d['parent'] = tenant.default_base_job 1185 d['parent'] = tenant.default_base_job
1173 d['dependencies'] = []
1174 for dependency in self.dependencies:
1175 d['dependencies'].append(dependency)
1176 if isinstance(self.nodeset, str): 1186 if isinstance(self.nodeset, str):
1177 ns = tenant.layout.nodesets.get(self.nodeset) 1187 ns = tenant.layout.nodesets.get(self.nodeset)
1178 else: 1188 else:
@@ -1343,6 +1353,9 @@ class Job(ConfigObject):
1343 self.parent_data = v 1353 self.parent_data = v
1344 self.variables = Job._deepUpdate(self.parent_data, self.variables) 1354 self.variables = Job._deepUpdate(self.parent_data, self.variables)
1345 1355
1356 def updateArtifactData(self, artifact_data):
1357 self.artifact_data = artifact_data
1358
1346 def updateProjectVariables(self, project_vars): 1359 def updateProjectVariables(self, project_vars):
1347 # Merge project/template variables directly into the job 1360 # Merge project/template variables directly into the job
1348 # variables. Job variables override project variables. 1361 # variables. Job variables override project variables.
@@ -1499,11 +1512,12 @@ class Job(ConfigObject):
1499 1512
1500 for k in self.context_attributes: 1513 for k in self.context_attributes:
1501 if (other._get(k) is not None and 1514 if (other._get(k) is not None and
1502 k not in set(['tags'])): 1515 k not in set(['tags', 'requires', 'provides'])):
1503 setattr(self, k, other._get(k)) 1516 setattr(self, k, other._get(k))
1504 1517
1505 if other._get('tags') is not None: 1518 for k in ('tags', 'requires', 'provides'):
1506 self.tags = frozenset(self.tags.union(other.tags)) 1519 if other._get(k) is not None:
1520 setattr(self, k, getattr(self, k).union(other._get(k)))
1507 1521
1508 self.inheritance_path = self.inheritance_path + (repr(other),) 1522 self.inheritance_path = self.inheritance_path + (repr(other),)
1509 1523
@@ -1924,6 +1938,7 @@ class BuildSet(object):
1924 1938
1925 1939
1926class QueueItem(object): 1940class QueueItem(object):
1941
1927 """Represents the position of a Change in a ChangeQueue. 1942 """Represents the position of a Change in a ChangeQueue.
1928 1943
1929 All Changes are enqueued into ChangeQueue in a QueueItem. The QueueItem 1944 All Changes are enqueued into ChangeQueue in a QueueItem. The QueueItem
@@ -1950,6 +1965,7 @@ class QueueItem(object):
1950 self.layout = None 1965 self.layout = None
1951 self.project_pipeline_config = None 1966 self.project_pipeline_config = None
1952 self.job_graph = None 1967 self.job_graph = None
1968 self._cached_sql_results = None
1953 1969
1954 def __repr__(self): 1970 def __repr__(self):
1955 if self.pipeline: 1971 if self.pipeline:
@@ -2146,6 +2162,110 @@ class QueueItem(object):
2146 return False 2162 return False
2147 return self.item_ahead.isHoldingFollowingChanges() 2163 return self.item_ahead.isHoldingFollowingChanges()
2148 2164
2165 def _getRequirementsResultFromSQL(self, requirements):
2166 # This either returns data or raises an exception
2167 if self._cached_sql_results is None:
2168 sql_driver = self.pipeline.manager.sched.connections.drivers['sql']
2169 conn = sql_driver.tenant_connections.get(self.pipeline.tenant.name)
2170 if conn:
2171 builds = conn.getBuilds(
2172 tenant=self.pipeline.tenant.name,
2173 project=self.change.project.name,
2174 pipeline=self.pipeline.name,
2175 change=self.change.number,
2176 branch=self.change.branch,
2177 patchset=self.change.patchset,
2178 provides=list(requirements))
2179 else:
2180 builds = []
2181 # Just look at the most recent buildset.
2182 # TODO: query for a buildset instead of filtering.
2183 builds = [b for b in builds
2184 if b.buildset.uuid == builds[0].buildset.uuid]
2185 self._cached_sql_results = builds
2186
2187 builds = self._cached_sql_results
2188 data = []
2189 if not builds:
2190 return data
2191
2192 for build in builds:
2193 if build.result != 'SUCCESS':
2194 provides = [x.name for x in build.provides]
2195 requirement = list(requirements.intersection(set(provides)))
2196 raise RequirementsError(
2197 "Requirements %s not met by build %s" % (
2198 requirement, build.uuid))
2199 else:
2200 artifacts = [{'name': a.name,
2201 'url': a.url,
2202 'project': build.buildset.project,
2203 'change': str(build.buildset.change),
2204 'patchset': build.buildset.patchset,
2205 'job': build.job_name}
2206 for a in build.artifacts]
2207 data += artifacts
2208 return data
2209
2210 def providesRequirements(self, requirements, data):
2211 # Mutates data and returns true/false if requirements
2212 # satisfied.
2213 if not requirements:
2214 return True
2215 if not self.live:
2216 # Look for this item in other queues in the pipeline.
2217 item = None
2218 found = False
2219 for item in self.pipeline.getAllItems():
2220 if item.live and item.change == self.change:
2221 found = True
2222 break
2223 if found:
2224 if not item.providesRequirements(requirements, data):
2225 return False
2226 else:
2227 # Look for this item in the SQL DB.
2228 data += self._getRequirementsResultFromSQL(requirements)
2229 if self.hasJobGraph():
2230 for job in self.getJobs():
2231 if job.provides.intersection(requirements):
2232 build = self.current_build_set.getBuild(job.name)
2233 if not build:
2234 return False
2235 if build.result and build.result != 'SUCCESS':
2236 return False
2237 if not build.result and not build.paused:
2238 return False
2239 artifacts = get_artifacts_from_result_data(
2240 build.result_data,
2241 logger=self.log)
2242 artifacts = [{'name': a['name'],
2243 'url': a['url'],
2244 'project': self.change.project.name,
2245 'change': self.change.number,
2246 'patchset': self.change.patchset,
2247 'job': build.job.name}
2248 for a in artifacts]
2249 data += artifacts
2250 if not self.item_ahead:
2251 return True
2252 return self.item_ahead.providesRequirements(requirements, data)
2253
2254 def jobRequirementsReady(self, job):
2255 if not self.item_ahead:
2256 return True
2257 try:
2258 data = []
2259 ret = self.item_ahead.providesRequirements(job.requires, data)
2260 job.updateArtifactData(data)
2261 except RequirementsError as e:
2262 self.warning(str(e))
2263 fakebuild = Build(job, None)
2264 fakebuild.result = 'SKIPPED'
2265 self.addBuild(fakebuild)
2266 ret = True
2267 return ret
2268
2149 def findJobsToRun(self, semaphore_handler): 2269 def findJobsToRun(self, semaphore_handler):
2150 torun = [] 2270 torun = []
2151 if not self.live: 2271 if not self.live:
@@ -2173,6 +2293,8 @@ class QueueItem(object):
2173 for job in self.job_graph.getJobs(): 2293 for job in self.job_graph.getJobs():
2174 if job not in jobs_not_started: 2294 if job not in jobs_not_started:
2175 continue 2295 continue
2296 if not self.jobRequirementsReady(job):
2297 continue
2176 all_parent_jobs_successful = True 2298 all_parent_jobs_successful = True
2177 parent_builds_with_data = {} 2299 parent_builds_with_data = {}
2178 for parent_job in self.job_graph.getParentJobsRecursively( 2300 for parent_job in self.job_graph.getParentJobsRecursively(
@@ -2237,6 +2359,8 @@ class QueueItem(object):
2237 for job in self.job_graph.getJobs(): 2359 for job in self.job_graph.getJobs():
2238 if job not in jobs_not_requested: 2360 if job not in jobs_not_requested:
2239 continue 2361 continue
2362 if not self.jobRequirementsReady(job):
2363 continue
2240 all_parent_jobs_successful = True 2364 all_parent_jobs_successful = True
2241 for parent_job in self.job_graph.getParentJobsRecursively( 2365 for parent_job in self.job_graph.getParentJobsRecursively(
2242 job.name): 2366 job.name):
diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py
index fcdae36..dbbb707 100755
--- a/zuul/web/__init__.py
+++ b/zuul/web/__init__.py
@@ -440,6 +440,7 @@ class ZuulWebAPI(object):
440 'newrev': buildset.newrev, 440 'newrev': buildset.newrev,
441 'ref_url': buildset.ref_url, 441 'ref_url': buildset.ref_url,
442 'artifacts': [], 442 'artifacts': [],
443 'provides': [],
443 } 444 }
444 445
445 for artifact in build.artifacts: 446 for artifact in build.artifacts:
@@ -447,6 +448,10 @@ class ZuulWebAPI(object):
447 'name': artifact.name, 448 'name': artifact.name,
448 'url': artifact.url, 449 'url': artifact.url,
449 }) 450 })
451 for provides in build.provides:
452 ret['provides'].append({
453 'name': artifact.name,
454 })
450 return ret 455 return ret
451 456
452 @cherrypy.expose 457 @cherrypy.expose