summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJames E. Blair <jeblair@redhat.com>2019-02-04 14:22:08 -0800
committerJames E. Blair <jeblair@redhat.com>2019-02-08 10:19:23 -0800
commitf12453f6cbb452451bcce0cef9c4da4b957fd4a5 (patch)
tree67a098912190c7a06ee51037a99ccf6e563920f4
parent6ea6885e2100943506e4024650c6f54c17732fa5 (diff)
Return artifacts as dicts and add metadata
A recent attempt to use the artifact return feature of zuul_return exposed some rough edges. These two changes should make it much easier to use. First, return artifacts as a dictionary instead of a list. This requires that they have unique names (which is no bad thing -- what would two artifacts named "docs" mean anyway?). But mainly it allows the dict merging behavior of zuul_return to be used, so that one playbook may use zuul_return with some artifacts, and another playbook may do the same, without either needing to load in the values of the other first (assuming, of course, that they use different artifact names). Second, add a metadata field. In the database and API, this is JSON serialized, but in zuul_return and zuul.artifacts, it is exploded into separate fields. This lets jobs do things like associate versions or tags with artifacts without having to abuse the url field. Change-Id: I228687c1bd1c74ebc33b088ffd43f30c7309990d
Notes
Notes (review): Code-Review+2: Tobias Henkel <tobias.henkel@bmw.de> Code-Review+2: Clark Boylan <cboylan@sapwetik.org> Workflow+1: Clark Boylan <cboylan@sapwetik.org> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Mon, 11 Feb 2019 18:07:11 +0000 Reviewed-on: https://review.openstack.org/634825 Project: openstack-infra/zuul Branch: refs/heads/master
-rw-r--r--doc/source/user/jobs.rst18
-rw-r--r--releasenotes/notes/artifact-format-2de4b9c038e28115.yaml12
-rw-r--r--zuul/driver/sql/alembic/versions/c18b1277dfb5_artifact_metadata.py37
-rw-r--r--zuul/driver/sql/sqlconnection.py4
-rw-r--r--zuul/driver/sql/sqlreporter.py3
-rw-r--r--zuul/lib/artifacts.py24
-rw-r--r--zuul/model.py19
7 files changed, 98 insertions, 19 deletions
diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst
index 826ff34..19bf4ea 100644
--- a/doc/source/user/jobs.rst
+++ b/doc/source/user/jobs.rst
@@ -229,7 +229,7 @@ of item.
229 under the ``zuul`` key: 229 under the ``zuul`` key:
230 230
231 .. var:: artifacts 231 .. var:: artifacts
232 :type: list 232 :type: dict
233 233
234 If the job has a :attr:`job.requires` attribute, and Zuul has 234 If the job has a :attr:`job.requires` attribute, and Zuul has
235 found changes ahead of this change in the pipeline with matching 235 found changes ahead of this change in the pipeline with matching
@@ -263,6 +263,10 @@ of item.
263 263
264 The URL of the artifact (as supplied to :ref:`return_artifacts`). 264 The URL of the artifact (as supplied to :ref:`return_artifacts`).
265 265
266 .. var:: metadata
267
268 The metadata of the artifact (as supplied to :ref:`return_artifacts`).
269
266 .. var:: build 270 .. var:: build
267 271
268 The UUID of the build. A build is a single execution of a job. 272 The UUID of the build. A build is a single execution of a job.
@@ -779,7 +783,7 @@ Returning artifact URLs
779 783
780If a build produces artifacts, any number of URLs may be returned to 784If a build produces artifacts, any number of URLs may be returned to
781Zuul and stored in the SQL database. These will then be available via 785Zuul and stored in the SQL database. These will then be available via
782the web interface. 786the web interface and subsequent jobs.
783 787
784To provide artifact URLs for a build, use *zuul_return* to set keys 788To provide artifact URLs for a build, use *zuul_return* to set keys
785under the **zuul.artifacts** dictionary. For example: 789under the **zuul.artifacts** dictionary. For example:
@@ -791,13 +795,17 @@ under the **zuul.artifacts** dictionary. For example:
791 data: 795 data:
792 zuul: 796 zuul:
793 artifacts: 797 artifacts:
794 - name: tarball 798 tarball:
795 url: http://example.com/path/to/package.tar.gz 799 url: http://example.com/path/to/package.tar.gz
796 - name: docs 800 metadata:
801 version: 3.0
802 docs:
797 url: build/docs/ 803 url: build/docs/
798 804
799If the value of **url** is a relative URL, it will be combined with 805If the value of **url** is a relative URL, it will be combined with
800the **zuul.log_url** value if set to create an absolute URL. 806the **zuul.log_url** value if set to create an absolute URL. The
807**metadata** key is optional; if it is provided, it must be a
808dictionary; its keys and values may be anything.
801 809
802Skipping child jobs 810Skipping child jobs
803~~~~~~~~~~~~~~~~~~~ 811~~~~~~~~~~~~~~~~~~~
diff --git a/releasenotes/notes/artifact-format-2de4b9c038e28115.yaml b/releasenotes/notes/artifact-format-2de4b9c038e28115.yaml
new file mode 100644
index 0000000..91c90ef
--- /dev/null
+++ b/releasenotes/notes/artifact-format-2de4b9c038e28115.yaml
@@ -0,0 +1,12 @@
1---
2features:
3 - Artifacts may now include a metadata field for storing arbitrary
4 metadata about the artifacts in the SQL database.
5deprecations:
6 - Artifacts should now be supplied to zuul_return in dictionary form
7 instead of a list. See :ref:`return_artifacts`.
8
9 This is to aid in multiple playbooks providing information back to
10 Zuul without requiring coordination with each other.
11
12 Support for the list format will be removed in a future version.
diff --git a/zuul/driver/sql/alembic/versions/c18b1277dfb5_artifact_metadata.py b/zuul/driver/sql/alembic/versions/c18b1277dfb5_artifact_metadata.py
new file mode 100644
index 0000000..7aec030
--- /dev/null
+++ b/zuul/driver/sql/alembic/versions/c18b1277dfb5_artifact_metadata.py
@@ -0,0 +1,37 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13"""artifact_metadata
14
15Revision ID: c18b1277dfb5
16Revises: 39d302d34d38
17Create Date: 2019-02-04 14:02:44.291890
18
19"""
20
21# revision identifiers, used by Alembic.
22revision = 'c18b1277dfb5'
23down_revision = '39d302d34d38'
24branch_labels = None
25depends_on = None
26
27from alembic import op
28import sqlalchemy as sa
29
30
31def upgrade(table_prefix=''):
32 op.add_column(
33 table_prefix + 'zuul_artifact', sa.Column('metadata', sa.TEXT()))
34
35
36def downgrade():
37 raise Exception("Downgrades not supported")
diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py
index 5cf79e8..170dddf 100644
--- a/zuul/driver/sql/sqlconnection.py
+++ b/zuul/driver/sql/sqlconnection.py
@@ -223,6 +223,9 @@ class SQLConnection(BaseConnection):
223 223
224 def createArtifact(self, *args, **kw): 224 def createArtifact(self, *args, **kw):
225 session = orm.session.Session.object_session(self) 225 session = orm.session.Session.object_session(self)
226 if 'metadata' in kw:
227 kw['meta'] = kw['metadata']
228 del kw['metadata']
226 a = ArtifactModel(*args, **kw) 229 a = ArtifactModel(*args, **kw)
227 a.build_id = self.id 230 a.build_id = self.id
228 self.artifacts.append(a) 231 self.artifacts.append(a)
@@ -246,6 +249,7 @@ class SQLConnection(BaseConnection):
246 self.table_prefix + BUILD_TABLE + ".id")) 249 self.table_prefix + BUILD_TABLE + ".id"))
247 name = sa.Column(sa.String(255)) 250 name = sa.Column(sa.String(255))
248 url = sa.Column(sa.TEXT()) 251 url = sa.Column(sa.TEXT())
252 meta = sa.Column('metadata', sa.TEXT())
249 build = orm.relationship(BuildModel, backref="artifacts") 253 build = orm.relationship(BuildModel, backref="artifacts")
250 254
251 class ProvidesModel(Base): 255 class ProvidesModel(Base):
diff --git a/zuul/driver/sql/sqlreporter.py b/zuul/driver/sql/sqlreporter.py
index 16651e4..03cbff6 100644
--- a/zuul/driver/sql/sqlreporter.py
+++ b/zuul/driver/sql/sqlreporter.py
@@ -13,6 +13,7 @@
13# under the License. 13# under the License.
14 14
15import datetime 15import datetime
16import json
16import logging 17import logging
17import time 18import time
18import voluptuous as v 19import voluptuous as v
@@ -90,6 +91,8 @@ class SQLReporter(BaseReporter):
90 for artifact in get_artifacts_from_result_data( 91 for artifact in get_artifacts_from_result_data(
91 build.result_data, 92 build.result_data,
92 logger=self.log): 93 logger=self.log):
94 if 'metadata' in artifact:
95 artifact['metadata'] = json.dumps(artifact['metadata'])
93 db_build.createArtifact(**artifact) 96 db_build.createArtifact(**artifact)
94 97
95 98
diff --git a/zuul/lib/artifacts.py b/zuul/lib/artifacts.py
index c7c2fe0..81a8338 100644
--- a/zuul/lib/artifacts.py
+++ b/zuul/lib/artifacts.py
@@ -15,15 +15,20 @@
15import voluptuous as v 15import voluptuous as v
16import urllib.parse 16import urllib.parse
17 17
18artifact = { 18old_artifact = {
19 'name': str, 19 'name': str,
20 'url': str, 20 'url': str,
21} 21}
22 22
23new_artifact = {
24 'url': str,
25 'metadata': dict,
26}
27
23zuul_data = { 28zuul_data = {
24 'zuul': { 29 'zuul': {
25 'log_url': str, 30 'log_url': str,
26 'artifacts': [artifact], 31 'artifacts': v.Any([old_artifact], {str: new_artifact}),
27 v.Extra: object, 32 v.Extra: object,
28 } 33 }
29} 34}
@@ -43,13 +48,18 @@ def get_artifacts_from_result_data(result_data, logger=None):
43 ret = [] 48 ret = []
44 if validate_artifact_schema(result_data): 49 if validate_artifact_schema(result_data):
45 artifacts = result_data.get('zuul', {}).get( 50 artifacts = result_data.get('zuul', {}).get(
46 'artifacts', []) 51 'artifacts', {})
52 if isinstance(artifacts, list):
53 new_artifacts = {}
54 for a in artifacts:
55 new_artifacts[a['name']] = {'url': a['url']}
56 artifacts = new_artifacts
47 default_url = result_data.get('zuul', {}).get( 57 default_url = result_data.get('zuul', {}).get(
48 'log_url') 58 'log_url')
49 if default_url: 59 if default_url:
50 if default_url[-1] != '/': 60 if default_url[-1] != '/':
51 default_url += '/' 61 default_url += '/'
52 for artifact in artifacts: 62 for artifact_name, artifact in artifacts.items():
53 url = artifact['url'] 63 url = artifact['url']
54 if default_url: 64 if default_url:
55 # If the artifact url is relative, it will be combined 65 # If the artifact url is relative, it will be combined
@@ -61,8 +71,10 @@ def get_artifacts_from_result_data(result_data, logger=None):
61 if logger: 71 if logger:
62 logger.debug("Error parsing URL:", 72 logger.debug("Error parsing URL:",
63 exc_info=1) 73 exc_info=1)
64 ret.append({'name': artifact['name'], 74 d = artifact.copy()
65 'url': url}) 75 d['name'] = artifact_name
76 d['url'] = url
77 ret.append(d)
66 else: 78 else:
67 logger.debug("Result data did not pass artifact schema " 79 logger.debug("Result data did not pass artifact schema "
68 "validation: %s", result_data) 80 "validation: %s", result_data)
diff --git a/zuul/model.py b/zuul/model.py
index 5a842df..f766032 100644
--- a/zuul/model.py
+++ b/zuul/model.py
@@ -15,6 +15,7 @@
15import abc 15import abc
16from collections import OrderedDict 16from collections import OrderedDict
17import copy 17import copy
18import json
18import logging 19import logging
19import os 20import os
20import re2 21import re2
@@ -2197,14 +2198,16 @@ class QueueItem(object):
2197 "Requirements %s not met by build %s" % ( 2198 "Requirements %s not met by build %s" % (
2198 requirement, build.uuid)) 2199 requirement, build.uuid))
2199 else: 2200 else:
2200 artifacts = [{'name': a.name, 2201 for a in build.artifacts:
2201 'url': a.url, 2202 artifact = {'name': a.name,
2202 'project': build.buildset.project, 2203 'url': a.url,
2203 'change': str(build.buildset.change), 2204 'project': build.buildset.project,
2204 'patchset': build.buildset.patchset, 2205 'change': str(build.buildset.change),
2205 'job': build.job_name} 2206 'patchset': build.buildset.patchset,
2206 for a in build.artifacts] 2207 'job': build.job_name}
2207 data += artifacts 2208 if a.meta:
2209 artifact['metadata'] = json.loads(a.meta)
2210 data.append(artifact)
2208 return data 2211 return data
2209 2212
2210 def providesRequirements(self, requirements, data): 2213 def providesRequirements(self, requirements, data):