Merge "Add artifact table"

This commit is contained in:
Zuul 2018-12-29 14:49:33 +00:00 committed by Gerrit Code Review
commit e46cfe63e0
8 changed files with 174 additions and 8 deletions

View File

@ -733,6 +733,29 @@ To set the log URL for a build, use *zuul_return* to set the
zuul:
log_url: http://logs.example.com/path/to/build/logs
.. _return_artifacts:
Returning artifact URLs
~~~~~~~~~~~~~~~~~~~~~~~
If a build produces artifacts, any number of URLs may be returned to
Zuul and stored in the SQL database. These will then be available via
the web interface.
To provide artifact URLs for a build, use *zuul_return* to set keys
under the **zuul.artifacts** dictionary. For example:
.. code-block:: yaml
tasks:
- zuul_return:
data:
zuul:
artifacts:
- name: tarball
url: http://example.com/path/to/package.tar.gz
- name: docs
url: http://example.com/path/to/docs
Skipping child jobs
~~~~~~~~~~~~~~~~~~~

View File

@ -0,0 +1,6 @@
---
features:
- |
Jobs may now return artifact URLs and they will be stored in the
SQL database (if configured). See :ref:`return_artifacts` for
usage.

View File

@ -1,2 +1,11 @@
- hosts: all
tasks: []
tasks:
- name: Return artifact data
zuul_return:
data:
zuul:
artifacts:
- name: tarball
url: http://example.com/tarball
- name: docs
url: http://example.com/docs

View File

@ -22,8 +22,8 @@ import requests
import zuul.web
from tests.base import ZuulTestCase, ZuulDBTestCase, FIXTURE_DIR
from tests.base import ZuulWebFixture
from tests.base import ZuulTestCase, ZuulDBTestCase, AnsibleZuulTestCase
from tests.base import ZuulWebFixture, FIXTURE_DIR
class FakeConfig(object):
@ -685,3 +685,26 @@ class TestBuildInfo(ZuulDBTestCase, BaseTestWeb):
resp = self.get_url("api/tenant/non-tenant/builds")
self.assertEqual(404, resp.status_code)
class TestArtifacts(ZuulDBTestCase, BaseTestWeb, AnsibleZuulTestCase):
config_file = 'zuul-sql-driver.conf'
tenant_config_file = 'config/sql-driver/main.yaml'
def test_artifacts(self):
# Generate some build records in the db.
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
build_query = self.get_url("api/tenant/tenant-one/builds?"
"project=org/project&"
"job_name=project-test1").json()
self.assertEqual(len(build_query), 1)
self.assertEqual(len(build_query[0]['artifacts']), 2)
self.assertEqual(build_query[0]['artifacts'], [
{'url': 'http://example.com/tarball',
'name': 'tarball'},
{'url': 'http://example.com/docs',
'name': 'docs'},
])

View File

@ -0,0 +1,47 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""add_artifact_table
Revision ID: ea2bae776723
Revises: f181b33958c6
Create Date: 2018-11-26 14:48:54.463512
"""
# revision identifiers, used by Alembic.
revision = 'ea2bae776723'
down_revision = 'f181b33958c6'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
ARTIFACT_TABLE = 'zuul_artifact'
BUILD_TABLE = 'zuul_build'
def upgrade(table_prefix=''):
op.create_table(
table_prefix + ARTIFACT_TABLE,
sa.Column('id', sa.Integer, primary_key=True),
sa.Column('build_id', sa.Integer,
sa.ForeignKey(table_prefix + BUILD_TABLE + ".id")),
sa.Column('name', sa.String(255)),
sa.Column('url', sa.TEXT()),
)
def downgrade():
raise Exception("Downgrades not supported")

View File

@ -27,6 +27,7 @@ from zuul.connection import BaseConnection
BUILDSET_TABLE = 'zuul_buildset'
BUILD_TABLE = 'zuul_build'
ARTIFACT_TABLE = 'zuul_artifact'
class DatabaseSession(object):
@ -65,7 +66,8 @@ class DatabaseSession(object):
# joinedload).
q = self.session().query(self.connection.buildModel).\
join(self.connection.buildSetModel).\
options(orm.contains_eager(self.connection.buildModel.buildset)).\
options(orm.contains_eager(self.connection.buildModel.buildset),
orm.selectinload(self.connection.buildModel.artifacts)).\
with_hint(build_table, 'USE INDEX (PRIMARY)', 'mysql')
q = self.listFilter(q, buildset_table.c.tenant, tenant)
@ -115,8 +117,7 @@ class SQLConnection(BaseConnection):
try:
self.dburi = self.connection_config.get('dburi')
self.zuul_buildset_table, self.zuul_build_table \
= self._setup_models()
self._setup_models()
# Recycle connections if they've been idle for more than 1 second.
# MySQL connections are lightweight and thus keeping long-lived
@ -213,9 +214,32 @@ class SQLConnection(BaseConnection):
node_name = sa.Column(sa.String(255))
buildset = orm.relationship(BuildSetModel, backref="builds")
def createArtifact(self, *args, **kw):
session = orm.session.Session.object_session(self)
a = ArtifactModel(*args, **kw)
a.build_id = self.id
self.artifacts.append(a)
session.add(a)
session.flush()
return a
class ArtifactModel(Base):
__tablename__ = self.table_prefix + ARTIFACT_TABLE
id = sa.Column(sa.Integer, primary_key=True)
build_id = sa.Column(sa.Integer, sa.ForeignKey(
self.table_prefix + BUILD_TABLE + ".id"))
name = sa.Column(sa.String(255))
url = sa.Column(sa.TEXT())
build = orm.relationship(BuildModel, backref="artifacts")
self.artifactModel = ArtifactModel
self.zuul_artifact_table = self.artifactModel.__table__
self.buildModel = BuildModel
self.zuul_build_table = self.buildModel.__table__
self.buildSetModel = BuildSetModel
return self.buildSetModel.__table__, self.buildModel.__table__
self.zuul_buildset_table = self.buildSetModel.__table__
def onStop(self):
self.log.debug("Stopping SQL connection %s" % self.connection_name)

View File

@ -26,6 +26,24 @@ class SQLReporter(BaseReporter):
name = 'sql'
log = logging.getLogger("zuul.SQLReporter")
artifact = {
'name': str,
'url': str,
}
zuul_data = {
'zuul': {
'artifacts': [artifact]
}
}
artifact_schema = v.Schema(zuul_data)
def validateArtifactSchema(self, data):
try:
self.artifact_schema(data)
except Exception:
return False
return True
def report(self, item):
"""Create an entry into a database."""
@ -71,7 +89,7 @@ class SQLReporter(BaseReporter):
build.end_time,
tz=datetime.timezone.utc)
db_buildset.createBuild(
db_build = db_buildset.createBuild(
uuid=build.uuid,
job_name=build.job.name,
result=result,
@ -82,6 +100,15 @@ class SQLReporter(BaseReporter):
node_name=build.node_name,
)
if self.validateArtifactSchema(build.result_data):
artifacts = build.result_data.get('zuul', {}).get(
'artifacts', [])
for artifact in artifacts:
db_build.createArtifact(
name=artifact['name'],
url=artifact['url'],
)
def getSchema():
sql_reporter = v.Schema(None)

View File

@ -439,7 +439,14 @@ class ZuulWebAPI(object):
'ref': buildset.ref,
'newrev': buildset.newrev,
'ref_url': buildset.ref_url,
'artifacts': [],
}
for artifact in build.artifacts:
ret['artifacts'].append({
'name': artifact.name,
'url': artifact.url,
})
return ret
@cherrypy.expose