summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2018-12-29 14:49:33 +0000
committerGerrit Code Review <review@openstack.org>2018-12-29 14:49:34 +0000
commite46cfe63e04bba058defb00f4b18fec0eb342b1f (patch)
treee5f73f7bc06b86dca25348cc04a244eed4741df1
parent5ae6d549a5d56e2813b5663751b6b2c883486d3c (diff)
parent6d1640d714d300686414f5b727c8e3d593af82c4 (diff)
Merge "Add artifact table"
-rw-r--r--doc/source/user/jobs.rst23
-rw-r--r--releasenotes/notes/add-artifacts-0718af34ca6165aa.yaml6
-rw-r--r--tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test1.yaml11
-rwxr-xr-xtests/unit/test_web.py27
-rw-r--r--zuul/driver/sql/alembic/versions/ea2bae776723_add_artifact_table.py47
-rw-r--r--zuul/driver/sql/sqlconnection.py32
-rw-r--r--zuul/driver/sql/sqlreporter.py29
-rwxr-xr-xzuul/web/__init__.py7
8 files changed, 174 insertions, 8 deletions
diff --git a/doc/source/user/jobs.rst b/doc/source/user/jobs.rst
index fb54975..4322eef 100644
--- a/doc/source/user/jobs.rst
+++ b/doc/source/user/jobs.rst
@@ -733,6 +733,29 @@ To set the log URL for a build, use *zuul_return* to set the
733 zuul: 733 zuul:
734 log_url: http://logs.example.com/path/to/build/logs 734 log_url: http://logs.example.com/path/to/build/logs
735 735
736.. _return_artifacts:
737
738Returning artifact URLs
739~~~~~~~~~~~~~~~~~~~~~~~
740
741If a build produces artifacts, any number of URLs may be returned to
742Zuul and stored in the SQL database. These will then be available via
743the web interface.
744
745To provide artifact URLs for a build, use *zuul_return* to set keys
746under the **zuul.artifacts** dictionary. For example:
747
748.. code-block:: yaml
749
750 tasks:
751 - zuul_return:
752 data:
753 zuul:
754 artifacts:
755 - name: tarball
756 url: http://example.com/path/to/package.tar.gz
757 - name: docs
758 url: http://example.com/path/to/docs
736 759
737Skipping child jobs 760Skipping child jobs
738~~~~~~~~~~~~~~~~~~~ 761~~~~~~~~~~~~~~~~~~~
diff --git a/releasenotes/notes/add-artifacts-0718af34ca6165aa.yaml b/releasenotes/notes/add-artifacts-0718af34ca6165aa.yaml
new file mode 100644
index 0000000..090c4fb
--- /dev/null
+++ b/releasenotes/notes/add-artifacts-0718af34ca6165aa.yaml
@@ -0,0 +1,6 @@
1---
2features:
3 - |
4 Jobs may now return artifact URLs and they will be stored in the
5 SQL database (if configured). See :ref:`return_artifacts` for
6 usage.
diff --git a/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test1.yaml b/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test1.yaml
index f679dce..7f2d48e 100644
--- a/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test1.yaml
+++ b/tests/fixtures/config/sql-driver/git/common-config/playbooks/project-test1.yaml
@@ -1,2 +1,11 @@
1- hosts: all 1- hosts: all
2 tasks: [] 2 tasks:
3 - name: Return artifact data
4 zuul_return:
5 data:
6 zuul:
7 artifacts:
8 - name: tarball
9 url: http://example.com/tarball
10 - name: docs
11 url: http://example.com/docs
diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py
index 73b5cd1..716f2f9 100755
--- a/tests/unit/test_web.py
+++ b/tests/unit/test_web.py
@@ -22,8 +22,8 @@ import requests
22 22
23import zuul.web 23import zuul.web
24 24
25from tests.base import ZuulTestCase, ZuulDBTestCase, FIXTURE_DIR 25from tests.base import ZuulTestCase, ZuulDBTestCase, AnsibleZuulTestCase
26from tests.base import ZuulWebFixture 26from tests.base import ZuulWebFixture, FIXTURE_DIR
27 27
28 28
29class FakeConfig(object): 29class FakeConfig(object):
@@ -685,3 +685,26 @@ class TestBuildInfo(ZuulDBTestCase, BaseTestWeb):
685 685
686 resp = self.get_url("api/tenant/non-tenant/builds") 686 resp = self.get_url("api/tenant/non-tenant/builds")
687 self.assertEqual(404, resp.status_code) 687 self.assertEqual(404, resp.status_code)
688
689
690class TestArtifacts(ZuulDBTestCase, BaseTestWeb, AnsibleZuulTestCase):
691 config_file = 'zuul-sql-driver.conf'
692 tenant_config_file = 'config/sql-driver/main.yaml'
693
694 def test_artifacts(self):
695 # Generate some build records in the db.
696 self.executor_server.hold_jobs_in_build = False
697 self.executor_server.release()
698 self.waitUntilSettled()
699
700 build_query = self.get_url("api/tenant/tenant-one/builds?"
701 "project=org/project&"
702 "job_name=project-test1").json()
703 self.assertEqual(len(build_query), 1)
704 self.assertEqual(len(build_query[0]['artifacts']), 2)
705 self.assertEqual(build_query[0]['artifacts'], [
706 {'url': 'http://example.com/tarball',
707 'name': 'tarball'},
708 {'url': 'http://example.com/docs',
709 'name': 'docs'},
710 ])
diff --git a/zuul/driver/sql/alembic/versions/ea2bae776723_add_artifact_table.py b/zuul/driver/sql/alembic/versions/ea2bae776723_add_artifact_table.py
new file mode 100644
index 0000000..269112a
--- /dev/null
+++ b/zuul/driver/sql/alembic/versions/ea2bae776723_add_artifact_table.py
@@ -0,0 +1,47 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13"""add_artifact_table
14
15Revision ID: ea2bae776723
16Revises: f181b33958c6
17Create Date: 2018-11-26 14:48:54.463512
18
19"""
20
21# revision identifiers, used by Alembic.
22revision = 'ea2bae776723'
23down_revision = 'f181b33958c6'
24branch_labels = None
25depends_on = None
26
27from alembic import op
28import sqlalchemy as sa
29
30
31ARTIFACT_TABLE = 'zuul_artifact'
32BUILD_TABLE = 'zuul_build'
33
34
35def upgrade(table_prefix=''):
36 op.create_table(
37 table_prefix + ARTIFACT_TABLE,
38 sa.Column('id', sa.Integer, primary_key=True),
39 sa.Column('build_id', sa.Integer,
40 sa.ForeignKey(table_prefix + BUILD_TABLE + ".id")),
41 sa.Column('name', sa.String(255)),
42 sa.Column('url', sa.TEXT()),
43 )
44
45
46def downgrade():
47 raise Exception("Downgrades not supported")
diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py
index ffde917..8a9a0f0 100644
--- a/zuul/driver/sql/sqlconnection.py
+++ b/zuul/driver/sql/sqlconnection.py
@@ -27,6 +27,7 @@ from zuul.connection import BaseConnection
27 27
28BUILDSET_TABLE = 'zuul_buildset' 28BUILDSET_TABLE = 'zuul_buildset'
29BUILD_TABLE = 'zuul_build' 29BUILD_TABLE = 'zuul_build'
30ARTIFACT_TABLE = 'zuul_artifact'
30 31
31 32
32class DatabaseSession(object): 33class DatabaseSession(object):
@@ -65,7 +66,8 @@ class DatabaseSession(object):
65 # joinedload). 66 # joinedload).
66 q = self.session().query(self.connection.buildModel).\ 67 q = self.session().query(self.connection.buildModel).\
67 join(self.connection.buildSetModel).\ 68 join(self.connection.buildSetModel).\
68 options(orm.contains_eager(self.connection.buildModel.buildset)).\ 69 options(orm.contains_eager(self.connection.buildModel.buildset),
70 orm.selectinload(self.connection.buildModel.artifacts)).\
69 with_hint(build_table, 'USE INDEX (PRIMARY)', 'mysql') 71 with_hint(build_table, 'USE INDEX (PRIMARY)', 'mysql')
70 72
71 q = self.listFilter(q, buildset_table.c.tenant, tenant) 73 q = self.listFilter(q, buildset_table.c.tenant, tenant)
@@ -115,8 +117,7 @@ class SQLConnection(BaseConnection):
115 117
116 try: 118 try:
117 self.dburi = self.connection_config.get('dburi') 119 self.dburi = self.connection_config.get('dburi')
118 self.zuul_buildset_table, self.zuul_build_table \ 120 self._setup_models()
119 = self._setup_models()
120 121
121 # Recycle connections if they've been idle for more than 1 second. 122 # Recycle connections if they've been idle for more than 1 second.
122 # MySQL connections are lightweight and thus keeping long-lived 123 # MySQL connections are lightweight and thus keeping long-lived
@@ -213,9 +214,32 @@ class SQLConnection(BaseConnection):
213 node_name = sa.Column(sa.String(255)) 214 node_name = sa.Column(sa.String(255))
214 buildset = orm.relationship(BuildSetModel, backref="builds") 215 buildset = orm.relationship(BuildSetModel, backref="builds")
215 216
217 def createArtifact(self, *args, **kw):
218 session = orm.session.Session.object_session(self)
219 a = ArtifactModel(*args, **kw)
220 a.build_id = self.id
221 self.artifacts.append(a)
222 session.add(a)
223 session.flush()
224 return a
225
226 class ArtifactModel(Base):
227 __tablename__ = self.table_prefix + ARTIFACT_TABLE
228 id = sa.Column(sa.Integer, primary_key=True)
229 build_id = sa.Column(sa.Integer, sa.ForeignKey(
230 self.table_prefix + BUILD_TABLE + ".id"))
231 name = sa.Column(sa.String(255))
232 url = sa.Column(sa.TEXT())
233 build = orm.relationship(BuildModel, backref="artifacts")
234
235 self.artifactModel = ArtifactModel
236 self.zuul_artifact_table = self.artifactModel.__table__
237
216 self.buildModel = BuildModel 238 self.buildModel = BuildModel
239 self.zuul_build_table = self.buildModel.__table__
240
217 self.buildSetModel = BuildSetModel 241 self.buildSetModel = BuildSetModel
218 return self.buildSetModel.__table__, self.buildModel.__table__ 242 self.zuul_buildset_table = self.buildSetModel.__table__
219 243
220 def onStop(self): 244 def onStop(self):
221 self.log.debug("Stopping SQL connection %s" % self.connection_name) 245 self.log.debug("Stopping SQL connection %s" % self.connection_name)
diff --git a/zuul/driver/sql/sqlreporter.py b/zuul/driver/sql/sqlreporter.py
index 327bbb9..2a7ae85 100644
--- a/zuul/driver/sql/sqlreporter.py
+++ b/zuul/driver/sql/sqlreporter.py
@@ -26,6 +26,24 @@ class SQLReporter(BaseReporter):
26 name = 'sql' 26 name = 'sql'
27 log = logging.getLogger("zuul.SQLReporter") 27 log = logging.getLogger("zuul.SQLReporter")
28 28
29 artifact = {
30 'name': str,
31 'url': str,
32 }
33 zuul_data = {
34 'zuul': {
35 'artifacts': [artifact]
36 }
37 }
38 artifact_schema = v.Schema(zuul_data)
39
40 def validateArtifactSchema(self, data):
41 try:
42 self.artifact_schema(data)
43 except Exception:
44 return False
45 return True
46
29 def report(self, item): 47 def report(self, item):
30 """Create an entry into a database.""" 48 """Create an entry into a database."""
31 49
@@ -71,7 +89,7 @@ class SQLReporter(BaseReporter):
71 build.end_time, 89 build.end_time,
72 tz=datetime.timezone.utc) 90 tz=datetime.timezone.utc)
73 91
74 db_buildset.createBuild( 92 db_build = db_buildset.createBuild(
75 uuid=build.uuid, 93 uuid=build.uuid,
76 job_name=build.job.name, 94 job_name=build.job.name,
77 result=result, 95 result=result,
@@ -82,6 +100,15 @@ class SQLReporter(BaseReporter):
82 node_name=build.node_name, 100 node_name=build.node_name,
83 ) 101 )
84 102
103 if self.validateArtifactSchema(build.result_data):
104 artifacts = build.result_data.get('zuul', {}).get(
105 'artifacts', [])
106 for artifact in artifacts:
107 db_build.createArtifact(
108 name=artifact['name'],
109 url=artifact['url'],
110 )
111
85 112
86def getSchema(): 113def getSchema():
87 sql_reporter = v.Schema(None) 114 sql_reporter = v.Schema(None)
diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py
index a3a50cb..fcdae36 100755
--- a/zuul/web/__init__.py
+++ b/zuul/web/__init__.py
@@ -439,7 +439,14 @@ class ZuulWebAPI(object):
439 'ref': buildset.ref, 439 'ref': buildset.ref,
440 'newrev': buildset.newrev, 440 'newrev': buildset.newrev,
441 'ref_url': buildset.ref_url, 441 'ref_url': buildset.ref_url,
442 'artifacts': [],
442 } 443 }
444
445 for artifact in build.artifacts:
446 ret['artifacts'].append({
447 'name': artifact.name,
448 'url': artifact.url,
449 })
443 return ret 450 return ret
444 451
445 @cherrypy.expose 452 @cherrypy.expose