summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xdoc/source/conf.py2
-rw-r--r--doc/source/docker-image.rst375
-rw-r--r--doc/source/index.rst1
-rw-r--r--doc/source/policy.rst106
-rw-r--r--playbooks/dco-license/run.yaml4
-rw-r--r--playbooks/docker-image/README.rst80
-rw-r--r--playbooks/docker-image/credentials.rst22
-rw-r--r--playbooks/docker-image/pre.yaml3
-rw-r--r--playbooks/docker-image/promote.yaml3
-rw-r--r--playbooks/docker-image/run.yaml3
-rw-r--r--playbooks/docker-image/upload.yaml3
-rw-r--r--roles/add-fileserver/README.rst18
-rw-r--r--roles/build-docker-image/README.rst3
-rw-r--r--roles/build-docker-image/common.rst122
-rw-r--r--roles/build-docker-image/defaults/main.yaml2
-rw-r--r--roles/build-docker-image/tasks/main.yaml28
-rw-r--r--roles/build-docker-image/tasks/push.yaml12
-rw-r--r--roles/deploy-openshift/README.rst1
-rw-r--r--roles/deploy-openshift/tasks/main.yaml24
-rw-r--r--roles/download-artifact/README.rst28
-rw-r--r--roles/download-artifact/defaults/main.yaml2
-rw-r--r--roles/download-artifact/tasks/main.yaml11
-rw-r--r--roles/ensure-twine/tasks/main.yaml2
-rw-r--r--roles/fetch-javascript-output/tasks/main.yaml8
-rw-r--r--roles/fetch-sphinx-tarball/README.rst18
-rw-r--r--roles/fetch-sphinx-tarball/defaults/main.yaml3
-rw-r--r--roles/fetch-sphinx-tarball/tasks/html.yaml43
-rw-r--r--roles/fetch-sphinx-tarball/tasks/main.yaml15
-rw-r--r--roles/install-docker/README.rst7
-rw-r--r--roles/install-docker/defaults/main.yaml1
-rw-r--r--roles/install-docker/tasks/main.yaml34
-rw-r--r--roles/install-docker/templates/daemon.json.j21
-rw-r--r--roles/install-kubernetes/tasks/minikube.yaml6
-rw-r--r--roles/install-nodejs/files/00-nodesource.pref3
-rw-r--r--roles/install-nodejs/tasks/main.yaml6
-rw-r--r--roles/install-openshift/README.rst16
-rw-r--r--roles/install-openshift/defaults/main.yaml2
-rw-r--r--roles/install-openshift/tasks/main.yaml44
-rw-r--r--roles/merge-output-to-logs/tasks/main.yaml3
-rw-r--r--roles/multi-node-bridge/tasks/peer.yaml31
-rw-r--r--roles/multi-node-firewall/tasks/main.yaml5
-rw-r--r--roles/promote-docker-image/README.rst3
-rw-r--r--roles/promote-docker-image/defaults/main.yaml1
-rw-r--r--roles/promote-docker-image/tasks/main.yaml27
-rw-r--r--roles/promote-docker-image/tasks/promote-cleanup.yaml20
-rw-r--r--roles/promote-docker-image/tasks/promote-retag-inner.yaml28
-rw-r--r--roles/promote-docker-image/tasks/promote-retag.yaml13
-rw-r--r--roles/pull-from-intermediate-registry/README.rst62
-rw-r--r--roles/pull-from-intermediate-registry/tasks/main.yaml75
-rw-r--r--roles/push-to-intermediate-registry/README.rst75
-rw-r--r--roles/push-to-intermediate-registry/tasks/main.yaml3
-rw-r--r--roles/push-to-intermediate-registry/tasks/push-image.yaml23
-rw-r--r--roles/push-to-intermediate-registry/tasks/push.yaml72
-rw-r--r--roles/remove-gpgkey/tasks/main.yaml2
-rw-r--r--roles/run-buildset-registry/README.rst43
-rw-r--r--roles/run-buildset-registry/defaults/main.yaml1
-rw-r--r--roles/run-buildset-registry/tasks/main.yaml111
-rw-r--r--roles/stage-output/README.rst10
-rw-r--r--roles/stage-output/tasks/main.yaml19
-rw-r--r--roles/tox/library/tox_install_sibling_packages.py7
-rw-r--r--roles/upload-docker-image/README.rst3
-rw-r--r--roles/upload-docker-image/defaults/main.yaml1
-rw-r--r--roles/upload-docker-image/tasks/main.yaml15
-rw-r--r--roles/upload-docker-image/tasks/push.yaml5
-rw-r--r--roles/upload-forge/README.rst24
-rw-r--r--roles/upload-forge/__init__.py0
-rw-r--r--roles/upload-forge/defaults/main.yaml2
-rw-r--r--roles/upload-forge/library/__init__.py0
-rw-r--r--roles/upload-forge/library/forge_upload.py182
-rw-r--r--roles/upload-forge/library/test_forge_upload.py28
-rw-r--r--roles/upload-forge/tasks/main.yaml13
-rw-r--r--roles/upload-logs/README.rst18
-rw-r--r--roles/upload-pypi/README.rst6
-rw-r--r--roles/upload-pypi/defaults/main.yaml1
-rw-r--r--roles/upload-pypi/tasks/main.yaml12
-rw-r--r--roles/use-buildset-registry/README.rst41
-rw-r--r--roles/use-buildset-registry/tasks/main.yaml77
-rw-r--r--roles/use-buildset-registry/tasks/user-config.yaml43
-rw-r--r--roles/use-docker-mirror/README.rst20
-rw-r--r--roles/use-docker-mirror/tasks/main.yaml11
-rw-r--r--roles/use-docker-mirror/tasks/mirror.yaml (renamed from roles/install-docker/tasks/mirror.yaml)0
-rw-r--r--roles/use-docker-mirror/templates/daemon.json.j24
-rw-r--r--roles/validate-dco-license/README.rst12
-rw-r--r--roles/validate-dco-license/defaults/main.yaml9
-rw-r--r--roles/validate-dco-license/tasks/main.yaml25
-rw-r--r--roles/yarn/tasks/main.yaml4
-rw-r--r--test-requirements.txt2
-rw-r--r--tox.ini3
-rw-r--r--zuul.yaml38
89 files changed, 2258 insertions, 37 deletions
diff --git a/doc/source/conf.py b/doc/source/conf.py
index 71466d9..1cdba5f 100755
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -19,6 +19,8 @@
19extensions = [ 19extensions = [
20 'sphinx.ext.autodoc', 20 'sphinx.ext.autodoc',
21 # 'sphinx.ext.intersphinx', 21 # 'sphinx.ext.intersphinx',
22 'sphinxcontrib.blockdiag',
23 'sphinxcontrib.seqdiag',
22 'zuul_sphinx', 24 'zuul_sphinx',
23] 25]
24 26
diff --git a/doc/source/docker-image.rst b/doc/source/docker-image.rst
new file mode 100644
index 0000000..14cb245
--- /dev/null
+++ b/doc/source/docker-image.rst
@@ -0,0 +1,375 @@
1Container Images
2================
3
4This repo has several jobs which can form the basis of a system
5supporting a full gating process for continuously deployed container
6images. They can be used to build or test images which rely on other
7images using the full power of Zuul's speculative execution.
8
9In order to use these jobs to their full potential, the Zuul site
10administrator will need to run a simple but dedicated container image
11registry, and define local versions of the jobs to use it. The
12following sections describe how to define those jobs and how the
13system is intended to work once the jobs are defined.
14
15Run an Intermediate Container Registry
16--------------------------------------
17
18A dedicated container registry is required for the use of these jobs.
19It is merely used to temporarily hold images so that they can be
20transferred between jobs running in different projects at different
21times. It does not need to be publicly accessible or particularly
22robust. If its backing storage fails and needs to be replaced, the
23only result is that some jobs running in Zuul may fail and may need to
24be re-run. In this system, it is called the "intermediate registry"
25to distinguish it from other registry services.
26
27You may run the registry in whatever manner is appropriate for your
28site. The following docker-compose file may be used as an example
29of a working deployment suitable for production:
30
31.. code-block:: yaml
32
33 services:
34 registry:
35 restart: always
36 image: registry:2
37 network_mode: host
38 environment:
39 REGISTRY_HTTP_TLS_CERTIFICATE: /certs/domain.crt
40 REGISTRY_HTTP_TLS_KEY: /certs/domain.key
41 REGISTRY_AUTH: htpasswd
42 REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
43 REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
44 volumes:
45 - /var/registry/data:/var/lib/registry
46 - /var/registry/certs:/certs
47 - /var/registry/auth:/auth
48
49You will need to provide the SSL certificate and key values, as well
50as the htpassword file with a user and password already present.
51
52Once that service is running, create the following four jobs in a
53Zuul config-project:
54
55.. _yoursite-buildset-registry:
56
57yoursite-buildset-registry
58~~~~~~~~~~~~~~~~~~~~~~~~~~
59
60This job is used to provide a temporary "buildset registry" to jobs
61running in your system; it communicates with the "intermediate"
62registry described above.
63
64.. code-block:: yaml
65 :caption: zuul.yaml
66
67 - secret:
68 name: yoursite-intermediate-registry
69 data:
70 host: insecure-ci-registry.example.org
71 port: 5000
72 username: zuul
73 password: !encrypted/pkcs1-oaep
74 - ...
75
76 - job:
77 name: yoursite-buildset-registry
78 pre-run: playbooks/buildset-registry/pre.yaml
79 run: playbooks/buildset-registry/run.yaml
80 post-run: playbooks/buildset-registry/post.yaml
81 secrets:
82 - secret: yoursite-intermediate-registry
83 name: intermediate_registry
84 requires: docker-image
85
86The credentials in the secret should match those you supplied when
87creating the intermediate registry.
88
89The ``requires: docker-image`` attribute means that whenever this job
90(or any jobs which inherit from it) run, Zuul will search ahead of the
91change in the dependency graph to find any jobs which produce
92docker-images and tell this job about them. This allows the job to
93pull images from the intermediate registry into the buildset registry.
94
95.. code-block:: yaml
96 :caption: playbooks/buildset-registry/pre.yaml
97
98 - hosts: all
99 tasks:
100 - name: Install docker
101 include_role:
102 name: install-docker
103 - name: Run buildset registry (if not already running)
104 when: buildset_registry is not defined
105 include_role:
106 name: run-buildset-registry
107 - name: Use buildset registry
108 include_role:
109 name: use-buildset-registry
110
111 - hosts: localhost
112 roles:
113 - pull-from-intermediate-registry
114
115This playbook runs a buildset registry if one isn't already running.
116It returns the connection information back to Zuul in a variable
117called ``buildset_registry``. Other jobs will use that to learn how
118to connect to the registry, and we can use that here to find out if
119one was already started in a previous job. We will use that facility
120in the :ref:`yoursite-build-docker-image` job below.
121
122.. code-block:: yaml
123 :caption: playbooks/buildset-registry/run.yaml
124
125 - hosts: localhost
126 tasks:
127 - name: Pause the job
128 zuul_return:
129 data:
130 zuul:
131 pause: true
132
133The ``pause`` causes the job to wait until all jobs which depend on
134this one are completed.
135
136.. code-block:: yaml
137 :caption: playbooks/buildset-registry/post.yaml
138
139 - hosts: localhost
140 roles:
141 - push-to-intermediate-registry
142
143.. _yoursite-build-docker-image:
144
145yoursite-build-docker-image
146~~~~~~~~~~~~~~~~~~~~~~~~~~~
147
148This job builds one or more docker images and interacts with the
149buildset and intermediate registries.
150
151.. code-block:: yaml
152 :caption: zuul.yaml
153
154 - job:
155 name: yoursite-build-docker-image
156 parent: yoursite-buildset-registry
157 run: playbooks/docker-image/run.yaml
158 provides: docker-image
159
160Note that the parent of this job is :ref:`yoursite-buildset-registry`.
161This means that a simple repo that only needs to support one image
162building job and doesn't have any other jobs which require a buildset
163registry can just add this job alone and it will run a buildset
164registry on the build host. More complex scenarios would run the
165:ref:`yoursite-buildset-registry` job on its own and construct a job
166graph that depends on it. Because the pre-run playbook in the
167buildset-registry job only runs a buildset registry if one isn't
168already running, it can be used for both cases. And because the run
169playbook which pauses the job is overridden in this job, this job will
170not pause.
171
172.. code-block:: yaml
173 :caption: playbooks/docker-image/run.yaml
174
175 - hosts: all
176 roles:
177 - build-docker-image
178
179.. _yoursite-upload-docker-image:
180
181yoursite-upload-docker-image
182~~~~~~~~~~~~~~~~~~~~~~~~~~~~
183
184This job further builds on the :ref:`yoursite-build-docker-image` job
185and additionally uploads the image to Docker Hub. Depending on the
186situation, you could encode the Docker Hub credentials into this job
187as a secret, or you could allow other users to provide them via the
188`pass-to-parent <https://zuul-ci.org/docs/zuul/user/config.html#attr-job.secrets.pass-to-parent>`_ feature of secrets.
189
190.. code-block:: yaml
191 :caption: zuul.yaml
192
193 - job:
194 name: yoursite-upload-docker-image
195 parent: yoursite-build-docker-image
196 post-run: playbooks/docker-image/upload.yaml
197
198.. code-block:: yaml
199 :caption: playbooks/docker-image/upload.yaml
200
201 - hosts: all
202 roles:
203 - upload-docker-image
204
205.. _yoursite-promote-docker-image:
206
207yoursite-promote-docker-image
208~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
209
210This job does nothing that the :zuul:job:`promote-docker-image` job in
211this repo doesn't already do, but since you created local versions of
212the other two jobs, you should make one of this as well for
213consistency. If you chose to add Docker Hub credentials to the
214:ref:`yoursite-upload-docker-image` job, you should do that here as
215well.
216
217.. code-block:: yaml
218 :caption: zuul.yaml
219
220 - job:
221 name: yoursite-promote-docker-image
222 parent: promote-docker-image
223
224System Architecture
225-------------------
226
227Now that those jobs are defined, this section describes how they work
228together.
229
230There are a few key concepts to keep in mind:
231
232A *buildset* is a group of jobs all running on the same change.
233
234A *buildset registry* is a container image registry which is used to
235store speculatively built images for the use of jobs in a single
236buildset. It holds the differences between the current state of the
237world and the future state if the change in question (and all of its
238dependent changes) were to merge. It must be started by one of the
239jobs in a buildset, and it ceases to exist once that job is complete.
240
241An *intermediate registry* is a long-running registry that is used to
242store images created for unmerged changes for use by other unmerged
243changes. It is not publicly accessible and is intended only to be
244used by Zuul in order to transfer artifacts from one buildset to
245another.
246
247With these concepts in mind, the jobs described above implement the
248following workflow for a single change:
249
250.. _buildset_image_transfer:
251
252.. seqdiag::
253 :caption: Buildset registry image transfer
254
255 seqdiag image_transfer {
256 Ireg [label="Intermediate\nRegistry"];
257 Breg [label="Buildset\nRegistry"];
258 Bjob [label="Image Build Job"];
259 Djob [label="Deployment Test Job"];
260
261 Ireg -> Breg [label='Images from previous changes'];
262 Breg -> Bjob [label='Images from previous changes'];
263 Breg <- Bjob [label='Current image'];
264 Ireg <- Breg [noactivate, label='Current image'];
265 Breg -> Djob [label='Current and previous images'];
266 Breg <- Djob [style=none];
267 Ireg <- Breg [style=none];
268 }
269
270The intermediate registry is always running and the buildset registry
271is started by a job running on a change. The "Image Build" and
272"Deployment Test" jobs are example jobs which might be running on a
273change. Essentially, these are image producer or consumer jobs
274respectively.
275
276There are two ways to use the jobs described above:
277
278A Repository with Producers and Consumers
279-----------------------------------------
280
281The first is in a repository where images are both produced and
282consumed. In this case, we can expect that there will be at least one
283image build job, and at least one job which uses that image (for
284example, by performing a test deployment of the image). In this case
285we need to construct a job graph with dependencies as follows:
286
287.. blockdiag::
288
289 blockdiag dependencies {
290 obr [label='yoursite-\nbuildset-registry'];
291 bi [label='build-image'];
292 ti [label='test-image'];
293
294 obr <- bi <- ti;
295 }
296
297The :ref:`yoursite-buildset-registry` job will run first and
298automatically start a buildset registry populated with images built
299from any changes which appear ahead of the current change. It will
300then return its connection information to Zuul and pause and continue
301running until the completion of the build and test jobs.
302
303The build-image job should inherit from
304:ref:`yoursite-build-docker-image`, which will ensure that it is
305automatically configured to use the buildset registry.
306
307The test-image job is something that you will create yourself. There
308is no standard way to test or deploy an image, that depends on your
309application. However, there is one thing you will need to do in your
310job to take advantage of the buildset registry. In a pre-run playbook,
311use the `use-buildset-registry
312<https://zuul-ci.org/docs/zuul-jobs/roles.html#role-use-buildset-registry>`_
313role:
314
315.. code-block:: yaml
316
317 - hosts: all
318 roles:
319 - use-buildset-registry
320
321That will configure the docker daemon on the host to use the buildset
322registry so that it will use the newly built version of any required
323images.
324
325A Repository with Only Producers
326--------------------------------
327
328The second way to use these jobs is in a repository where an image is
329merely built, but not deployed. In this case, there are no consumers
330of the buildset registry other than the image build job, and so the
331registry can be run on the job itself. In this case, you may omit the
332:ref:`yoursite-buildset-registry` job and run only the
333:ref:`yoursite-build-docker-image` job.
334
335Publishing an Image
336-------------------
337
338So far we've covered the image building process. This system also
339provides two more jobs that are used in publishing images to Docker
340Hub.
341
342The :ref:`yoursite-upload-docker-image` job does everything the
343:ref:`yoursite-build-docker-image` job does, but it also uploads
344the built image to Docker Hub using an automatically-generated and
345temporary tag. The "build" job is designed to be used in the
346*check* pipeline, while the "upload" job is designed to take its
347place in the *gate* pipeline. By front-loading the upload to Docker
348Hub, we reduce the chance that a credential or network error will
349prevent us from publishing an image after a change lands.
350
351The :ref:`yoursite-promote-docker-image` job is designed to be
352used in the *promote* pipeline and simply re-tags the image on Docker
353Hub after the change lands.
354
355Keeping in mind that everything described above in
356:ref:`buildset_image_transfer` applies to the
357:ref:`yoursite-upload-docker-image` job, the following illustrates
358the additional tasks performed by the "upload" and "promote" jobs:
359
360.. seqdiag::
361
362 seqdiag image_transfer {
363 DH [activated, label="Docker Hub"];
364 Ujob [label="upload-image"];
365 Pjob [label="promote-image"];
366
367 DH -> Ujob [style=none];
368 DH <- Ujob [label='Current image with temporary tag'];
369 DH -> Pjob [label='Current image manifest with temporary tag',
370 note='Only the manifest
371 is transferred,
372 not the actual
373 image layers.'];
374 DH <- Pjob [label='Current image manifest with final tag'];
375 }
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 72ca440..31017c3 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -7,6 +7,7 @@
7 policy 7 policy
8 jobs 8 jobs
9 roles 9 roles
10 docker-image
10 11
11Indices and tables 12Indices and tables
12================== 13==================
diff --git a/doc/source/policy.rst b/doc/source/policy.rst
index ef92aac..1c28545 100644
--- a/doc/source/policy.rst
+++ b/doc/source/policy.rst
@@ -52,8 +52,11 @@ Library code should be written to be compatible with both. There are
52some tips on this in `Ansible and Python 3 52some tips on this in `Ansible and Python 3
53<https://docs.ansible.com/ansible/2.5/dev_guide/developing_python_3.html>`__. 53<https://docs.ansible.com/ansible/2.5/dev_guide/developing_python_3.html>`__.
54 54
55Coding guidelines
56-----------------
57
55Role Variable Naming Policy 58Role Variable Naming Policy
56--------------------------- 59***************************
57 60
58Variables referenced by roles from global scope (often intended to be 61Variables referenced by roles from global scope (often intended to be
59set via ``host_vars`` and ``group_vars``, but also set during role 62set via ``host_vars`` and ``group_vars``, but also set during role
@@ -70,6 +73,106 @@ as ``example_role_variable``; e.g.
70 vars: 73 vars:
71 example_role_variable: 'something' 74 example_role_variable: 'something'
72 75
76Support for Multiple Operating Systems
77**************************************
78
79Ideally, roles should be able to run regardless of the OS or the distribution
80flavor of the host. A role can target a specific OS or distribution; in that case
81it should be mentioned in the role's documentation and start with a `fail` task
82if the host does not match the intended environment:
83
84.. code-block:: YAML
85
86 tasks:
87 - name: Make sure the role is run on XXX version Y
88 fail:
89 msg: "This role supports XXX version Y only"
90 when:
91 - ansible_distribution != "XXX"
92 - ansible_distribution_major_version != "Y"
93
94Here are a few guidelines to help make roles OS-independent when possible:
95
96* Use the **package** module instead of **yum**, **apt** or other
97 distribution-specific commands.
98* If more than one specific task is needed for a specific OS, these tasks should
99 be stored in a separate YAML file in a `distros` subdirectory and named after
100 the specific flavor they target. The following boilerplate code can be used to
101 target specific flavors:
102
103.. code-block:: YAML
104
105 tasks:
106 - name: Execute distro-specific tasks
107 include_tasks: "{{ lookup('first_found', params) }}"
108 vars:
109 params:
110 files:
111 - "mytasks-{{ ansible_distribution }}.{{ ansible_distribution_major_version }}.{{ ansible_architecture }}.yaml"
112 - "mytasks-{{ ansible_distribution }}.{{ ansible_distribution_major_version }}.yaml"
113 - "mytasks-{{ ansible_distribution }}.yaml"
114 - "mytasks-{{ ansible_os_family }}.yaml"
115 - "mytasks-default.yaml"
116 paths:
117 - distros
118
119If run on Fedora 29 x86_64, this playbook will attempt to include the first
120playbook found among
121
122* `distros/mytasks-Fedora.29.x86_64.yaml`
123* `distros/mytasks-Fedora.29.yaml`
124* `distros/mytasks-Fedora.yaml`
125* `distros/mytasks-RedHat.yaml`
126* `distros/mytasks-default.yaml`
127
128The default playbook should return a failure explaining the host's environment is
129not supported, or a skip if the tasks were optional.
130
131Handling privileges on hosts
132****************************
133
134Zuul offers great freedom in the types and configurations of hosts on which roles
135are run. Therefore roles should not assume the amount of privileges they will be
136granted on hosts. Some settings may not allow any form of privilege escalation,
137meaning that some tasks such as installing packages will fail.
138
139In order to make a role available to as many hosts as possible, it is good practice
140to avoid privilege escalations:
141
142* Do not use ``become: yes`` in tasks, unless necessary
143* If installing software is required, favor software deployments in user land,
144 like virtualenvs, if possible.
145* Check before executing a task requiring privilege escalation is actually
146 needed (e.g. is the package to install already present, or is the firewall
147 rule already set), and make the task skippable if its effects were already
148 applied to the host.
149
150If privilege escalation is unavoidable, this should be mentioned in the role's
151documentation so that operators can choose or set up their hosts accordingly.
152If relevant, the specific steps where the privilege escalation occurs should be
153documented so that they can be reproduced when configuring hosts. If possible,
154they should be grouped in a separate playbook that can be applied to hosts manually.
155
156Installing Dependencies in Roles
157********************************
158
159Roles should be self-sufficient. This makes it sometimes necessary to pull dependencies
160within a role, in order to execute a task. Since this is usually an action
161requiring elevated privileges on the host, the guidelines in the previous
162paragraph apply. Again, ideally all the installation tasks should be grouped in
163a separate playbook.
164
165Here are the ways to install dependencies in order of preference:
166
167* Use the **package** module to install packages
168* Manage dependencies with `bindep <https://docs.openstack.org/infra/bindep/readme.html>`__
169 and the `bindep` role.
170* Use OS-specific tasks like **apt**, **yum** etc. to support as many OSes as
171 possible.
172
173In any case, the role's documentation should mention which dependencies are
174needed, allowing users to prepare their hosts accordingly.
175
73Testing 176Testing
74------- 177-------
75 178
@@ -108,4 +211,3 @@ which is how it should remain until the next proposed change.
108 211
109.. _zuul-announce: http://lists.zuul-ci.org/cgi-bin/mailman/listinfo/zuul-announce 212.. _zuul-announce: http://lists.zuul-ci.org/cgi-bin/mailman/listinfo/zuul-announce
110.. _zuul-discuss: http://lists.zuul-ci.org/cgi-bin/mailman/listinfo/zuul-discuss 213.. _zuul-discuss: http://lists.zuul-ci.org/cgi-bin/mailman/listinfo/zuul-discuss
111
diff --git a/playbooks/dco-license/run.yaml b/playbooks/dco-license/run.yaml
new file mode 100644
index 0000000..4f617ad
--- /dev/null
+++ b/playbooks/dco-license/run.yaml
@@ -0,0 +1,4 @@
1- hosts: localhost
2 roles:
3 - role: validate-dco-license
4 zuul_work_dir: "{{ zuul.executor.work_root }}/{{ zuul.project.src_dir }}"
diff --git a/playbooks/docker-image/README.rst b/playbooks/docker-image/README.rst
new file mode 100644
index 0000000..91bffd4
--- /dev/null
+++ b/playbooks/docker-image/README.rst
@@ -0,0 +1,80 @@
1This is one of a collection of jobs which are designed to work
2together to build, upload, and promote docker images in a gating
3context:
4
5 * :zuul:job:`build-docker-image`: Build the images.
6 * :zuul:job:`upload-docker-image`: Build and stage the images on dockerhub.
7 * :zuul:job:`promote-docker-image`: Promote previously uploaded images.
8
9The :zuul:job:`build-docker-image` job is designed to be used in
10a `check` pipeline and simply builds the images to verify that
11the build functions.
12
13The :zuul:job:`upload-docker-image` job builds and uploads the images
14to Docker Hub, but only with a single tag corresponding to the
15change ID. This job is designed in a `gate` pipeline so that the
16build produced by the gate is staged and can later be promoted to
17production if the change is successful.
18
19The :zuul:job:`promote-docker-image` job is designed to be used in a
20`promote` pipeline. It requires no nodes and runs very quickly on the
21Zuul executor. It simply re-tags a previously uploaded image for a
22change with whatever tags are supplied by
23:zuul:jobvar:`build-docker-image.docker_images.tags`. It also removes
24the change ID tag from the repository in Docker Hub, and removes any
25similar change ID tags more than 24 hours old. This keeps the
26repository tidy in the case that gated changes fail to merge after
27uploading their staged images.
28
29They all accept the same input data, principally a list of
30dictionaries representing the images to build. YAML anchors_ can be
31used to supply the same data to all three jobs.
32
33**Job Variables**
34
35.. zuul:jobvar:: zuul_work_dir
36 :default: {{ zuul.project.src_dir }}
37
38 The project directory. Serves as the base for
39 :zuul:jobvar:`build-docker-image.docker_images.context`.
40
41.. zuul:jobvar:: docker_images
42 :type: list
43
44 A list of images to build. Each item in the list should have:
45
46 .. zuul:jobvar:: context
47
48 The docker build context; this should be a directory underneath
49 :zuul:jobvar:`build-docker-image.zuul_work_dir`.
50
51 .. zuul:jobvar:: repository
52
53 The name of the target repository in dockerhub for the
54 image. Supply this even if the image is not going to be
55 uploaded (it will be tagged with this in the local
56 registry).
57
58 .. zuul:jobvar:: path
59
60 Optional: the directory that should be passed to docker build.
61 Useful for building images with a Dockerfile in the context
62 directory but a source repository elsewhere.
63
64 .. zuul:jobvar:: build_args
65 :type: list
66
67 Optional: a list of values to pass to the docker ``--build-arg``
68 parameter.
69
70 .. zuul:jobvar:: target
71
72 Optional: the target for a multi-stage build.
73
74 .. zuul:jobvar:: tags
75 :type: list
76 :default: ['latest']
77
78 A list of tags to be added to the image when promoted.
79
80.. _anchors: https://yaml.org/spec/1.2/spec.html#&%20anchor//
diff --git a/playbooks/docker-image/credentials.rst b/playbooks/docker-image/credentials.rst
new file mode 100644
index 0000000..4587dbf
--- /dev/null
+++ b/playbooks/docker-image/credentials.rst
@@ -0,0 +1,22 @@
1.. zuul:jobvar:: docker_credentials
2 :type: dict
3
4 This is expected to be a Zuul Secret with these keys:
5
6 .. zuul:jobvar:: username
7
8 The Docker Hub username.
9
10 .. zuul:jobvar:: password
11
12 The Docker Hub password.
13
14 .. zuul:jobvar:: repository
15
16 Optional; if supplied this is a regular expression which
17 restricts to what repositories the image may be uploaded. The
18 following example allows projects to upload images to
19 repositories within an organization based on their own names::
20
21 repository: "^myorgname/{{ zuul.project.short_name }}.*"
22
diff --git a/playbooks/docker-image/pre.yaml b/playbooks/docker-image/pre.yaml
new file mode 100644
index 0000000..52f5b5b
--- /dev/null
+++ b/playbooks/docker-image/pre.yaml
@@ -0,0 +1,3 @@
1- hosts: all
2 roles:
3 - install-docker
diff --git a/playbooks/docker-image/promote.yaml b/playbooks/docker-image/promote.yaml
new file mode 100644
index 0000000..8160bf4
--- /dev/null
+++ b/playbooks/docker-image/promote.yaml
@@ -0,0 +1,3 @@
1- hosts: localhost
2 roles:
3 - promote-docker-image
diff --git a/playbooks/docker-image/run.yaml b/playbooks/docker-image/run.yaml
new file mode 100644
index 0000000..d3525ad
--- /dev/null
+++ b/playbooks/docker-image/run.yaml
@@ -0,0 +1,3 @@
1- hosts: all
2 roles:
3 - build-docker-image
diff --git a/playbooks/docker-image/upload.yaml b/playbooks/docker-image/upload.yaml
new file mode 100644
index 0000000..712f726
--- /dev/null
+++ b/playbooks/docker-image/upload.yaml
@@ -0,0 +1,3 @@
1- hosts: all
2 roles:
3 - upload-docker-image
diff --git a/roles/add-fileserver/README.rst b/roles/add-fileserver/README.rst
index 8a7b5ec..ae7d741 100644
--- a/roles/add-fileserver/README.rst
+++ b/roles/add-fileserver/README.rst
@@ -7,7 +7,23 @@ in subsequent tasks or roles.
7 7
8 Complex argument which contains the information about the remote 8 Complex argument which contains the information about the remote
9 destination as well as the authentication information needed. It is 9 destination as well as the authentication information needed. It is
10 expected that this argument comes from a `Secret`. 10 expected that this argument comes from a `Secret
11 <https://zuul-ci.org/docs/zuul/user/config.html#secret>`_
12
13 Example:
14
15 .. code-block:: yaml
16
17 - secret:
18 name: site_logs
19 data:
20 fqdn: logs.example.org
21 path: /srv/static/logs
22 ssh_known_hosts: |
23 logs.example.org ssh-rsa ...
24 ssh_username: zuul
25 ssh_private_key: !encrypted/pkcs1-oaep
26 - ...
11 27
12 .. zuul:rolevar:: fqdn 28 .. zuul:rolevar:: fqdn
13 29
diff --git a/roles/build-docker-image/README.rst b/roles/build-docker-image/README.rst
new file mode 100644
index 0000000..f533afa
--- /dev/null
+++ b/roles/build-docker-image/README.rst
@@ -0,0 +1,3 @@
1Build one or more docker images.
2
3.. include:: ../../roles/build-docker-image/common.rst
diff --git a/roles/build-docker-image/common.rst b/roles/build-docker-image/common.rst
new file mode 100644
index 0000000..95d37ad
--- /dev/null
+++ b/roles/build-docker-image/common.rst
@@ -0,0 +1,122 @@
1This is one of a collection of roles which are designed to work
2together to build, upload, and promote docker images in a gating
3context:
4
5* :zuul:role:`build-docker-image`: Build the images.
6* :zuul:role:`upload-docker-image`: Stage the images on dockerhub.
7* :zuul:role:`promote-docker-image`: Promote previously uploaded images.
8
9The :zuul:role:`build-docker-image` role is designed to be used in
10`check` and `gate` pipelines and simply builds the images. It can be
11used to verify that the build functions, or it can be followed by the
12use of subsequent roles to upload the images to Docker Hub.
13
14The :zuul:role:`upload-docker-image` role uploads the images to Docker
15Hub, but only with a single tag corresponding to the change ID. This
16role is designed to be used in a job in a `gate` pipeline so that the
17build produced by the gate is staged and can later be promoted to
18production if the change is successful.
19
20The :zuul:role:`promote-docker-image` role is designed to be used in a
21`promote` pipeline. It requires no nodes and runs very quickly on the
22Zuul executor. It simply re-tags a previously uploaded image for a
23change with whatever tags are supplied by
24:zuul:rolevar:`build-docker-image.docker_images.tags`. It also
25removes the change ID tag from the repository in Docker Hub, and
26removes any similar change ID tags more than 24 hours old. This keeps
27the repository tidy in the case that gated changes fail to merge after
28uploading their staged images.
29
30They all accept the same input data, principally a list of
31dictionaries representing the images to build. YAML anchors_ can be
32used to supply the same data to all three jobs.
33
34Use the :zuul:role:`install-docker` role to install Docker before
35using this role.
36
37**Role Variables**
38
39.. zuul:rolevar:: zuul_work_dir
40 :default: {{ zuul.project.src_dir }}
41
42 The project directory. Serves as the base for
43 :zuul:rolevar:`build-docker-image.docker_images.context`.
44
45.. zuul:rolevar:: docker_dockerfile
46 :default: Dockerfile
47
48 The default Dockerfile name to use. Serves as the base for
49 :zuul:rolevar:`build-docker-image.docker_images.dockerfile`.
50 This allows a global overriding of Dockerfile name, for example
51 when building all images from different folders with similarily
52 named dockerfiles.
53
54.. zuul:rolevar:: docker_credentials
55 :type: dict
56
57 This is only required for the upload and promote roles. This is
58 expected to be a Zuul Secret with two keys:
59
60 .. zuul:rolevar:: username
61
62 The Docker Hub username.
63
64 .. zuul:rolevar:: password
65
66 The Docker Hub password.
67
68 .. zuul:rolevar:: repository
69
70 Optional; if supplied this is a regular expression which
71 restricts to what repositories the image may be uploaded. The
72 following example allows projects to upload images to
73 repositories within an organization based on their own names::
74
75 repository: "^myorgname/{{ zuul.project.short_name }}.*"
76
77.. zuul:rolevar:: docker_images
78 :type: list
79
80 A list of images to build. Each item in the list should have:
81
82 .. zuul:rolevar:: context
83
84 The docker build context; this should be a directory underneath
85 :zuul:rolevar:`build-docker-image.zuul_work_dir`.
86
87 .. zuul:rolevar:: dockerfile
88
89 The filename of the dockerfile, present in the context folder,
90 used for building the image. Provide this if you are using
91 a non-standard filename for a specific image.
92
93 .. zuul:rolevar:: repository
94
95 The name of the target repository in dockerhub for the
96 image. Supply this even if the image is not going to be
97 uploaded (it will be tagged with this in the local
98 registry).
99
100 .. zuul:rolevar:: path
101
102 Optional: the directory that should be passed to docker build.
103 Useful for building images with a Dockerfile in the context
104 directory but a source repository elsewhere.
105
106 .. zuul:rolevar:: build_args
107 :type: list
108
109 Optional: a list of values to pass to the docker ``--build-arg``
110 parameter.
111
112 .. zuul:rolevar:: target
113
114 Optional: the target for a multi-stage build.
115
116 .. zuul:rolevar:: tags
117 :type: list
118 :default: ['latest']
119
120 A list of tags to be added to the image when promoted.
121
122.. _anchors: https://yaml.org/spec/1.2/spec.html#&%20anchor//
diff --git a/roles/build-docker-image/defaults/main.yaml b/roles/build-docker-image/defaults/main.yaml
new file mode 100644
index 0000000..d702500
--- /dev/null
+++ b/roles/build-docker-image/defaults/main.yaml
@@ -0,0 +1,2 @@
1zuul_work_dir: "{{ zuul.project.src_dir }}"
2docker_dockerfile: "Dockerfile"
diff --git a/roles/build-docker-image/tasks/main.yaml b/roles/build-docker-image/tasks/main.yaml
new file mode 100644
index 0000000..4a09643
--- /dev/null
+++ b/roles/build-docker-image/tasks/main.yaml
@@ -0,0 +1,28 @@
1# This can be removed if we add this functionality to Zuul directly
2- name: Load information from zuul_return
3 when: buildset_registry is not defined
4 set_fact:
5 buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
6 ignore_errors: true
7- name: Build a docker image
8 command: >-
9 docker build {{ item.path | default('.') }} -f {{ item.dockerfile | default(docker_dockerfile) }}
10 {% if item.target | default(false) -%}
11 --target {{ item.target }}
12 {% endif -%}
13 {% for build_arg in item.build_args | default([]) -%}
14 --build-arg {{ build_arg }}
15 {% endfor -%}
16 {% for tag in item.tags | default(['latest']) -%}
17 --tag {{ item.repository }}:change_{{ zuul.change }}_{{ tag }}
18 --tag {{ item.repository }}:{{ tag }}
19 {% endfor -%}
20 args:
21 chdir: "{{ zuul_work_dir }}/{{ item.context }}"
22 loop: "{{ docker_images }}"
23- name: Push image to buildset registry
24 when: buildset_registry is defined
25 include_tasks: push.yaml
26 loop: "{{ docker_images }}"
27 loop_control:
28 loop_var: image
diff --git a/roles/build-docker-image/tasks/push.yaml b/roles/build-docker-image/tasks/push.yaml
new file mode 100644
index 0000000..d49edd1
--- /dev/null
+++ b/roles/build-docker-image/tasks/push.yaml
@@ -0,0 +1,12 @@
1- name: Tag image for buildset registry
2 command: >-
3 docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
4 loop: "{{ image.tags | default(['latest']) }}"
5 loop_control:
6 loop_var: image_tag
7- name: Push tag to buildset registry
8 command: >-
9 docker push {{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
10 loop: "{{ image.tags | default(['latest']) }}"
11 loop_control:
12 loop_var: image_tag
diff --git a/roles/deploy-openshift/README.rst b/roles/deploy-openshift/README.rst
new file mode 100644
index 0000000..bb1f498
--- /dev/null
+++ b/roles/deploy-openshift/README.rst
@@ -0,0 +1 @@
Deploy openshift using oc cluster up.
diff --git a/roles/deploy-openshift/tasks/main.yaml b/roles/deploy-openshift/tasks/main.yaml
new file mode 100644
index 0000000..aa41364
--- /dev/null
+++ b/roles/deploy-openshift/tasks/main.yaml
@@ -0,0 +1,24 @@
1---
2- name: Deploy local openshift cluster
3 command: "oc cluster up --insecure-skip-tls-verify=true --public-hostname={{ ansible_hostname }}"
4 become: yes
5
6- name: Create zuul user .kube directory
7 file:
8 path: "{{ ansible_user_dir }}/.kube"
9 state: directory
10
11- name: Setup zuul user kube config
12 copy:
13 src: /root/.kube/config
14 dest: "{{ ansible_env.HOME }}/.kube/config"
15 owner: "{{ ansible_env.USER }}"
16 mode: 0600
17 remote_src: yes
18 become: yes
19
20- name: Login as system:admin
21 command: oc login -u system:admin
22
23- name: Who am i
24 command: oc whoami -c
diff --git a/roles/download-artifact/README.rst b/roles/download-artifact/README.rst
new file mode 100644
index 0000000..4bb2fa6
--- /dev/null
+++ b/roles/download-artifact/README.rst
@@ -0,0 +1,28 @@
1Download an artifact from a completed build of a Zuul job
2
3Given a change downloads an artifact from a previous build (by default
4of the current change) into the work directory.
5
6**Role Variables**
7
8.. zuul:rolevar:: download_artifact_api
9
10 The Zuul API endpoint to use. Example: ``https://zuul.example.org/api/tenant/{{ zuul.tenant }}``
11
12.. zuul:rolevar:: download_artifact_pipeline
13
14 The pipeline in which the previous build ran.
15
16.. zuul:rolevar:: download_artifact_job
17
18 The job of the previous build.
19
20.. zuul:rolevar:: download_artifact_name
21
22 The artifact name.
23
24.. zuul:rolevar:: download_artifact_query
25 :default: change={{ zuul.change }}&patchset={{ zuul.patchset }}&pipeline={{ download_artifact_pipeline }}&job_name={{ download_artifact_job }}
26
27 The query to use to find the build. This should return exactly one
28 result. Normally the default is used.
diff --git a/roles/download-artifact/defaults/main.yaml b/roles/download-artifact/defaults/main.yaml
new file mode 100644
index 0000000..abda6f3
--- /dev/null
+++ b/roles/download-artifact/defaults/main.yaml
@@ -0,0 +1,2 @@
1---
2download_artifact_query: "change={{ zuul.change }}&patchset={{ zuul.patchset }}&pipeline={{ download_artifact_pipeline }}&job_name={{ download_artifact_job }}"
diff --git a/roles/download-artifact/tasks/main.yaml b/roles/download-artifact/tasks/main.yaml
new file mode 100644
index 0000000..fc2e9b8
--- /dev/null
+++ b/roles/download-artifact/tasks/main.yaml
@@ -0,0 +1,11 @@
1- name: Query Zuul API for artifact information
2 uri:
3 url: "{{ download_artifact_api }}/builds?{{ download_artifact_query }}"
4 register: build
5- name: Extract artifact URL
6 set_fact:
7 archive_url: "{{ (build.json[0].artifacts | selectattr('name', 'equalto', download_artifact_name) | list)[0].url }}"
8- name: Download archive
9 uri:
10 url: "{{ archive_url }}"
11 dest: "{{ zuul.executor.work_root }}"
diff --git a/roles/ensure-twine/tasks/main.yaml b/roles/ensure-twine/tasks/main.yaml
index c7c2832..8d96690 100644
--- a/roles/ensure-twine/tasks/main.yaml
+++ b/roles/ensure-twine/tasks/main.yaml
@@ -11,7 +11,7 @@
11- name: Ensure twine is installed 11- name: Ensure twine is installed
12 block: 12 block:
13 - name: Ensure twine is installed 13 - name: Ensure twine is installed
14 command: "{{ twine_python }} -m pip install twine!=1.12.0 readme_renderer[md]!=23.0 --user" 14 command: "{{ twine_python }} -m pip install twine!=1.12.0 readme_renderer[md]!=23.0 requests-toolbelt!=0.9.0 --user"
15 15
16 - name: Set pypi_twine_executable 16 - name: Set pypi_twine_executable
17 set_fact: 17 set_fact:
diff --git a/roles/fetch-javascript-output/tasks/main.yaml b/roles/fetch-javascript-output/tasks/main.yaml
index 13416cb..fdab0a6 100644
--- a/roles/fetch-javascript-output/tasks/main.yaml
+++ b/roles/fetch-javascript-output/tasks/main.yaml
@@ -140,3 +140,11 @@
140 src: "{{ coverage_output_src }}" 140 src: "{{ coverage_output_src }}"
141 verify_host: true 141 verify_host: true
142 when: coverage_report_stat.stat.exists 142 when: coverage_report_stat.stat.exists
143
144- name: Return site artifact location to Zuul
145 zuul_return:
146 data:
147 zuul:
148 artifacts:
149 - name: "site"
150 url: "npm/html/"
diff --git a/roles/fetch-sphinx-tarball/README.rst b/roles/fetch-sphinx-tarball/README.rst
new file mode 100644
index 0000000..a1af64c
--- /dev/null
+++ b/roles/fetch-sphinx-tarball/README.rst
@@ -0,0 +1,18 @@
1Collect output from a sphinx build as a tarball
2
3By default, this copies the output from the sphinx build on the worker
4to the log root of the executor as a tarball, and then extracts the
5archive into the log root for viewing.
6
7**Role Variables**
8
9.. zuul:rolevar:: sphinx_build_dir
10 :default: doc/build
11
12 Directory relative to zuul_work_dir where build output should be
13 found.
14
15.. zuul:rolevar:: zuul_work_dir
16 :default: {{ zuul.project.src_dir }}
17
18 The location of the main working directory of the job.
diff --git a/roles/fetch-sphinx-tarball/defaults/main.yaml b/roles/fetch-sphinx-tarball/defaults/main.yaml
new file mode 100644
index 0000000..4c68b38
--- /dev/null
+++ b/roles/fetch-sphinx-tarball/defaults/main.yaml
@@ -0,0 +1,3 @@
1---
2zuul_work_dir: "{{ zuul.project.src_dir }}"
3sphinx_build_dir: doc/build
diff --git a/roles/fetch-sphinx-tarball/tasks/html.yaml b/roles/fetch-sphinx-tarball/tasks/html.yaml
new file mode 100644
index 0000000..2900bd4
--- /dev/null
+++ b/roles/fetch-sphinx-tarball/tasks/html.yaml
@@ -0,0 +1,43 @@
1- name: Create temporary HTML archive file
2 tempfile:
3 state: file
4 suffix: ".tar.bz2"
5 register: html_archive
6
7- name: Archive HTML
8 command: "tar -f {{ html_archive.path }} -C {{ zuul_work_dir }}/{{ sphinx_build_dir }}/html -cj ."
9 args:
10 warn: false
11
12- name: Fetch archive HTML
13 synchronize:
14 dest: "{{ zuul.executor.log_root }}/docs-html.tar.bz2"
15 mode: pull
16 src: "{{ html_archive.path }}"
17 verify_host: true
18
19- name: Create browseable HTML directory
20 delegate_to: localhost
21 file:
22 path: "{{ zuul.executor.log_root }}/docs"
23 state: directory
24
25- name: Extract archive HTML
26 delegate_to: localhost
27 unarchive:
28 src: "{{ zuul.executor.log_root }}/docs-html.tar.bz2"
29 dest: "{{ zuul.executor.log_root }}/docs"
30
31- name: Return artifact to Zuul
32 zuul_return:
33 data:
34 zuul:
35 artifacts:
36 - name: "docs_archive"
37 url: "docs-html.tar.bz2"
38 metadata:
39 type: docs_archive
40 - name: "docs_site"
41 url: "docs/"
42 metadata:
43 type: docs_site
diff --git a/roles/fetch-sphinx-tarball/tasks/main.yaml b/roles/fetch-sphinx-tarball/tasks/main.yaml
new file mode 100644
index 0000000..5fde05e
--- /dev/null
+++ b/roles/fetch-sphinx-tarball/tasks/main.yaml
@@ -0,0 +1,15 @@
1- name: Inspect sphinx build directory
2 find:
3 file_type: any
4 paths: "{{ zuul_work_dir }}/{{ sphinx_build_dir }}"
5 register: sphinx_dir
6
7- name: Parse sphinx build directory
8 set_fact:
9 sphinx_dir: "{{sphinx_dir.files | map(attribute='path') | map('regex_replace', '^.*/(.*)$', '\\1') | list}}"
10
11- name: Process sphinx HTML
12 when: "'html' in sphinx_dir"
13 include_tasks: html.yaml
14
15# Other sphinx output processing can be added here.
diff --git a/roles/install-docker/README.rst b/roles/install-docker/README.rst
index 7d540a2..e820cb1 100644
--- a/roles/install-docker/README.rst
+++ b/roles/install-docker/README.rst
@@ -32,3 +32,10 @@ An ansible role to install docker and configure it to use mirrors if available.
32 Undefined will install the latest. This will look something like 32 Undefined will install the latest. This will look something like
33 ``18.06.1~ce~3-0~ubuntu``. Only supported when using upstream 33 ``18.06.1~ce~3-0~ubuntu``. Only supported when using upstream
34 docker repos. 34 docker repos.
35
36.. zuul:rolevar:: docker_insecure_registries
37 :default: undefined
38
39 Declare this with a list of insecure registries to define the
40 registries which are allowed to communicate with HTTP only or
41 HTTPS with no valid certificate.
diff --git a/roles/install-docker/defaults/main.yaml b/roles/install-docker/defaults/main.yaml
index 6351680..bef4fbd 100644
--- a/roles/install-docker/defaults/main.yaml
+++ b/roles/install-docker/defaults/main.yaml
@@ -1,4 +1,5 @@
1use_upstream_docker: True 1use_upstream_docker: True
2docker_group: docker
2docker_update_channel: stable 3docker_update_channel: stable
3ubuntu_gpg_key: | 4ubuntu_gpg_key: |
4 -----BEGIN PGP PUBLIC KEY BLOCK----- 5 -----BEGIN PGP PUBLIC KEY BLOCK-----
diff --git a/roles/install-docker/tasks/main.yaml b/roles/install-docker/tasks/main.yaml
index 875d760..45b9684 100644
--- a/roles/install-docker/tasks/main.yaml
+++ b/roles/install-docker/tasks/main.yaml
@@ -1,14 +1,6 @@
1- name: Set mirror_fqdn fact
2 when:
3 - mirror_fqdn is not defined
4 - zuul_site_mirror_fqdn is defined
5 set_fact:
6 mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
7
8- name: Set up docker mirrors 1- name: Set up docker mirrors
9 include: mirror.yaml 2 include_role:
10 when: mirror_fqdn is defined 3 name: use-docker-mirror
11 static: no
12 4
13- name: Install docker-ce from upstream 5- name: Install docker-ce from upstream
14 include: upstream.yaml 6 include: upstream.yaml
@@ -32,8 +24,26 @@
32 user: 24 user:
33 name: "{{ ansible_user }}" 25 name: "{{ ansible_user }}"
34 groups: 26 groups:
35 - docker 27 - "{{ docker_group }}"
36 append: yes 28 append: yes
37 29
38- name: reset ssh connection to pick up docker group 30- name: Assure docker service is running
31 become: yes
32 service:
33 name: docker
34 enabled: yes
35 state: started
36
37- name: Correct group ownership on docker sock
38 become: yes
39 file:
40 path: /var/run/docker.sock
41 group: "{{ docker_group }}"
42
43- name: Reset ssh connection to pick up docker group
39 meta: reset_connection 44 meta: reset_connection
45
46- name: Validate ability to talk with docker
47 command: docker ps
48 args:
49 warn: no
diff --git a/roles/install-docker/templates/daemon.json.j2 b/roles/install-docker/templates/daemon.json.j2
index 1e6f158..2520818 100644
--- a/roles/install-docker/templates/daemon.json.j2
+++ b/roles/install-docker/templates/daemon.json.j2
@@ -1,3 +1,4 @@
1{ 1{
2 {% if docker_insecure_registries is defined -%}"insecure-registries": {{ docker_insecure_registries | to_json }},{% endif %}
2 "registry-mirrors": ["{{ docker_mirror }}"] 3 "registry-mirrors": ["{{ docker_mirror }}"]
3} 4}
diff --git a/roles/install-kubernetes/tasks/minikube.yaml b/roles/install-kubernetes/tasks/minikube.yaml
index 7c3f979..425c001 100644
--- a/roles/install-kubernetes/tasks/minikube.yaml
+++ b/roles/install-kubernetes/tasks/minikube.yaml
@@ -44,5 +44,11 @@
44 MINIKUBE_HOME: "{{ ansible_user_dir }}" 44 MINIKUBE_HOME: "{{ ansible_user_dir }}"
45 KUBECONFIG: "{{ ansible_user_dir }}/.kube/config" 45 KUBECONFIG: "{{ ansible_user_dir }}/.kube/config"
46 46
47- name: Ensure minikube config is owned by ansible_user
48 become: yes
49 file:
50 path: "{{ ansible_user_dir }}/.minikube/client.key"
51 owner: "{{ ansible_user }}"
52
47- name: Get cluster info 53- name: Get cluster info
48 command: kubectl cluster-info 54 command: kubectl cluster-info
diff --git a/roles/install-nodejs/files/00-nodesource.pref b/roles/install-nodejs/files/00-nodesource.pref
new file mode 100644
index 0000000..c830506
--- /dev/null
+++ b/roles/install-nodejs/files/00-nodesource.pref
@@ -0,0 +1,3 @@
1Package: nodejs
2Pin: origin deb.nodesource.com
3Pin-Priority: 900
diff --git a/roles/install-nodejs/tasks/main.yaml b/roles/install-nodejs/tasks/main.yaml
index 84dcd1d..44b7dde 100644
--- a/roles/install-nodejs/tasks/main.yaml
+++ b/roles/install-nodejs/tasks/main.yaml
@@ -9,6 +9,12 @@
9 state: present 9 state: present
10 become: yes 10 become: yes
11 11
12- name: Pin nodejs installs to nodesource
13 copy:
14 src: 00-nodesource.pref
15 dest: /etc/apt/preferences.d/00-nodesource.pref
16 become: yes
17
12- name: Add nodesource repository key 18- name: Add nodesource repository key
13 apt_key: 19 apt_key:
14 url: "https://deb.nodesource.com/gpgkey/nodesource.gpg.key" 20 url: "https://deb.nodesource.com/gpgkey/nodesource.gpg.key"
diff --git a/roles/install-openshift/README.rst b/roles/install-openshift/README.rst
new file mode 100644
index 0000000..758245c
--- /dev/null
+++ b/roles/install-openshift/README.rst
@@ -0,0 +1,16 @@
1Setup openshift requirements and pull the container images.
2The deploy-openshift role can be used to start the services.
3
4This role only works on CentOS.
5
6**Role Variables**
7
8.. zuul:rolevar:: origin_repo
9 :default: centos-release-openshift-origin39
10
11 The origin repository.
12
13.. zuul:rolevar:: origin_version
14 :default: v3.9.0
15
16 The origin version.
diff --git a/roles/install-openshift/defaults/main.yaml b/roles/install-openshift/defaults/main.yaml
new file mode 100644
index 0000000..6f77c75
--- /dev/null
+++ b/roles/install-openshift/defaults/main.yaml
@@ -0,0 +1,2 @@
1origin_repo: centos-release-openshift-origin39
2origin_version: v3.9.0
diff --git a/roles/install-openshift/tasks/main.yaml b/roles/install-openshift/tasks/main.yaml
new file mode 100644
index 0000000..3b5497d
--- /dev/null
+++ b/roles/install-openshift/tasks/main.yaml
@@ -0,0 +1,44 @@
1- name: Install origin repository
2 yum:
3 name: "{{ origin_repo }}"
4 become: yes
5
6- name: Install requirements
7 yum:
8 name: "{{ item }}"
9 with_items:
10 - origin
11 - docker
12 become: yes
13
14- name: Fix docker start options
15 lineinfile:
16 dest: /etc/sysconfig/docker
17 regexp: "^OPTIONS="
18 line: "OPTIONS='--selinux-enabled --log-driver=journald --signature-verification=false --insecure-registry 172.30.0.0/16'"
19 become: yes
20
21# See: https://github.com/openshift/origin/issues/15038
22- name: Fix rhel secret issue
23 file:
24 path: /usr/share/rhel/secrets
25 state: absent
26 become: yes
27
28- name: Start docker service
29 service:
30 name: docker
31 state: started
32 become: yes
33
34- name: Pull origin images
35 command: "docker pull docker.io/openshift/{{ item }}:{{ origin_version }}"
36 with_items:
37 - origin-web-console
38 - origin-docker-registry
39 - origin-haproxy-router
40 - origin-deployer
41 - origin-sti-builder
42 - origin-pod
43 - origin
44 become: yes
diff --git a/roles/merge-output-to-logs/tasks/main.yaml b/roles/merge-output-to-logs/tasks/main.yaml
index d1fa111..504c87a 100644
--- a/roles/merge-output-to-logs/tasks/main.yaml
+++ b/roles/merge-output-to-logs/tasks/main.yaml
@@ -2,7 +2,7 @@
2 when: zuul.change is defined 2 when: zuul.change is defined
3 delegate_to: localhost 3 delegate_to: localhost
4 shell: | 4 shell: |
5 if ! $(ls {{ zuul.executor.work_root }}/{{ item }}) ; then 5 if [ -n "$(find {{ zuul.executor.work_root }}/{{ item }} -mindepth 1)" ] ; then
6 # Only create target directory if it is needed. 6 # Only create target directory if it is needed.
7 # Do not fail if it is already there. 7 # Do not fail if it is already there.
8 mkdir -p {{ zuul.executor.log_root }}/{{ item }} 8 mkdir -p {{ zuul.executor.log_root }}/{{ item }}
@@ -13,3 +13,4 @@
13 loop: 13 loop:
14 - artifacts 14 - artifacts
15 - docs 15 - docs
16 run_once: true
diff --git a/roles/multi-node-bridge/tasks/peer.yaml b/roles/multi-node-bridge/tasks/peer.yaml
index 65f8f9a..8eeba12 100644
--- a/roles/multi-node-bridge/tasks/peer.yaml
+++ b/roles/multi-node-bridge/tasks/peer.yaml
@@ -9,19 +9,30 @@
9 vni: "{{ offset | int + bridge_vni_offset | int }}" 9 vni: "{{ offset | int + bridge_vni_offset | int }}"
10 10
11# To make things more readable in the following tasks 11# To make things more readable in the following tasks
12- name: Set ip address when the node private IP is not set
13 set_fact:
14 nodepool_ip: |
15 {{ nodepool.private_ipv4 | default(nodepool.public_ipv4) }}
16
17- name: Select the switch from group
18 set_fact:
19 switch: "{{ groups['switch'][0] }}"
20
12- name: Alias the primary node private IP 21- name: Alias the primary node private IP
13 set_fact: 22 set_fact:
14 switch_private_ip: "{{ hostvars[groups['switch'][0]]['nodepool']['private_ipv4'] }}" 23 switch_ip: |
24 {{ hostvars[switch].nodepool.private_ipv4 |
25 default(hostvars[switch].nodepool.public_ipv4) }}
15 26
16- name: Add port to bridge on switch node 27- name: Add port to bridge on switch node
17 become: yes 28 become: yes
18 command: >- 29 command: >-
19 ovs-vsctl --may-exist add-port {{ bridge_name }} 30 ovs-vsctl --may-exist add-port {{ bridge_name }}
20 {{ bridge_name }}_{{ nodepool['private_ipv4'] }} 31 {{ bridge_name }}_{{ nodepool_ip }}
21 -- set interface {{ bridge_name }}_{{ nodepool['private_ipv4'] }} 32 -- set interface {{ bridge_name }}_{{ nodepool_ip}}
22 type=vxlan options:remote_ip={{ nodepool['private_ipv4'] }} options:key={{ vni }} 33 type=vxlan options:remote_ip={{ nodepool_ip }} options:key={{ vni }}
23 options:local_ip={{ switch_private_ip }} 34 options:local_ip={{ switch_ip }}
24 delegate_to: "{{ groups['switch'][0] }}" 35 delegate_to: "{{ switch }}"
25 36
26- name: Create bridge on peer node 37- name: Create bridge on peer node
27 become: yes 38 become: yes
@@ -36,10 +47,10 @@
36 become: yes 47 become: yes
37 command: >- 48 command: >-
38 ovs-vsctl --may-exist add-port {{ bridge_name }} 49 ovs-vsctl --may-exist add-port {{ bridge_name }}
39 {{ bridge_name }}_{{ switch_private_ip }} 50 {{ bridge_name }}_{{ switch_ip }}
40 -- set interface {{ bridge_name }}_{{ switch_private_ip }} 51 -- set interface {{ bridge_name }}_{{ switch_ip }}
41 type=vxlan options:remote_ip={{ switch_private_ip }} options:key={{ vni }} 52 type=vxlan options:remote_ip={{ switch_ip }} options:key={{ vni }}
42 options:local_ip={{ nodepool['private_ipv4'] }} 53 options:local_ip={{ nodepool_ip }}
43 54
44- when: bridge_configure_address 55- when: bridge_configure_address
45 block: 56 block:
diff --git a/roles/multi-node-firewall/tasks/main.yaml b/roles/multi-node-firewall/tasks/main.yaml
index 304fd88..3a43b0a 100644
--- a/roles/multi-node-firewall/tasks/main.yaml
+++ b/roles/multi-node-firewall/tasks/main.yaml
@@ -1,3 +1,8 @@
1- name: Ensure iptables
2 become: true
3 package:
4 name: iptables
5
1- name: Set up the host ip addresses 6- name: Set up the host ip addresses
2 set_fact: 7 set_fact:
3 ipv4_addresses: > 8 ipv4_addresses: >
diff --git a/roles/promote-docker-image/README.rst b/roles/promote-docker-image/README.rst
new file mode 100644
index 0000000..abce78f
--- /dev/null
+++ b/roles/promote-docker-image/README.rst
@@ -0,0 +1,3 @@
1Promote one or more previously uploaded docker images.
2
3.. include:: ../../roles/build-docker-image/common.rst
diff --git a/roles/promote-docker-image/defaults/main.yaml b/roles/promote-docker-image/defaults/main.yaml
new file mode 100644
index 0000000..9739eb1
--- /dev/null
+++ b/roles/promote-docker-image/defaults/main.yaml
@@ -0,0 +1 @@
zuul_work_dir: "{{ zuul.project.src_dir }}"
diff --git a/roles/promote-docker-image/tasks/main.yaml b/roles/promote-docker-image/tasks/main.yaml
new file mode 100644
index 0000000..80ad09a
--- /dev/null
+++ b/roles/promote-docker-image/tasks/main.yaml
@@ -0,0 +1,27 @@
1- name: Verify repository names
2 when: |
3 docker_credentials.repository is defined
4 and not item.repository | regex_search(docker_credentials.repository)
5 loop: "{{ docker_images }}"
6 fail:
7 msg: "{{ item.repository }} not permitted by {{ docker_credentials.repository }}"
8# This is used by the delete tasks
9- name: Get dockerhub JWT token
10 no_log: true
11 uri:
12 url: "https://hub.docker.com/v2/users/login/"
13 body_format: json
14 body:
15 username: "{{ docker_credentials.username }}"
16 password: "{{ docker_credentials.password }}"
17 register: jwt_token
18- name: Promote image
19 loop: "{{ docker_images }}"
20 loop_control:
21 loop_var: image
22 include_tasks: promote-retag.yaml
23- name: Delete obsolete tags
24 loop: "{{ docker_images }}"
25 loop_control:
26 loop_var: image
27 include_tasks: promote-cleanup.yaml
diff --git a/roles/promote-docker-image/tasks/promote-cleanup.yaml b/roles/promote-docker-image/tasks/promote-cleanup.yaml
new file mode 100644
index 0000000..d8435b4
--- /dev/null
+++ b/roles/promote-docker-image/tasks/promote-cleanup.yaml
@@ -0,0 +1,20 @@
1- name: List tags
2 uri:
3 url: "https://hub.docker.com/v2/repositories/{{ image.repository }}/tags?page_size=1000"
4 status_code: 200
5 register: tags
6- name: Set cutoff timestamp to 24 hours ago
7 command: "python3 -c \"import datetime; print((datetime.datetime.utcnow()-datetime.timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%fZ'))\""
8 register: cutoff
9- name: Delete all change tags older than the cutoff
10 no_log: true
11 loop: "{{ tags.json.results }}"
12 loop_control:
13 loop_var: docker_tag
14 when: docker_tag.last_updated < cutoff.stdout and docker_tag.name.startswith('change_')
15 uri:
16 url: "https://hub.docker.com/v2/repositories/{{ image.repository }}/tags/{{ docker_tag.name }}/"
17 method: DELETE
18 status_code: 204
19 headers:
20 Authorization: "JWT {{ jwt_token.json.token }}"
diff --git a/roles/promote-docker-image/tasks/promote-retag-inner.yaml b/roles/promote-docker-image/tasks/promote-retag-inner.yaml
new file mode 100644
index 0000000..ed947d8
--- /dev/null
+++ b/roles/promote-docker-image/tasks/promote-retag-inner.yaml
@@ -0,0 +1,28 @@
1- name: Get manifest
2 no_log: true
3 uri:
4 url: "https://registry.hub.docker.com/v2/{{ image.repository }}/manifests/change_{{ zuul.change }}_{{ image_tag }}"
5 status_code: 200
6 headers:
7 Accept: "application/vnd.docker.distribution.manifest.v2+json"
8 Authorization: "Bearer {{ token.json.token }}"
9 return_content: true
10 register: manifest
11- name: Put manifest
12 no_log: true
13 uri:
14 url: "https://registry.hub.docker.com/v2/{{ image.repository }}/manifests/{{ image_tag }}"
15 method: PUT
16 status_code: 201
17 body: "{{ manifest.content | string }}"
18 headers:
19 Content-Type: "application/vnd.docker.distribution.manifest.v2+json"
20 Authorization: "Bearer {{ token.json.token }}"
21- name: Delete the current change tag
22 no_log: true
23 uri:
24 url: "https://hub.docker.com/v2/repositories/{{ image.repository }}/tags/change_{{ zuul.change }}_{{ image_tag }}/"
25 method: DELETE
26 status_code: 204
27 headers:
28 Authorization: "JWT {{ jwt_token.json.token }}"
diff --git a/roles/promote-docker-image/tasks/promote-retag.yaml b/roles/promote-docker-image/tasks/promote-retag.yaml
new file mode 100644
index 0000000..8237dab
--- /dev/null
+++ b/roles/promote-docker-image/tasks/promote-retag.yaml
@@ -0,0 +1,13 @@
1- name: Get dockerhub token
2 no_log: true
3 uri:
4 url: "https://auth.docker.io/token?service=registry.docker.io&scope=repository:{{ image.repository }}:pull,push"
5 user: "{{ docker_credentials.username }}"
6 password: "{{ docker_credentials.password }}"
7 force_basic_auth: true
8 register: token
9- name: Retag image
10 loop: "{{ image.tags | default(['latest']) }}"
11 loop_control:
12 loop_var: image_tag
13 include_tasks: promote-retag-inner.yaml
diff --git a/roles/pull-from-intermediate-registry/README.rst b/roles/pull-from-intermediate-registry/README.rst
new file mode 100644
index 0000000..435d345
--- /dev/null
+++ b/roles/pull-from-intermediate-registry/README.rst
@@ -0,0 +1,62 @@
1Pull artifacts from the intermediate registry
2
3This role will pull any artifacts built for changes ahead of this
4change which have been placed in an intermediate registry into the
5buildset registry for this buildset.
6
7Run this in a trusted pre-playbook at the start of a job (which, in
8the case of multiple dependent jobs in a buildset, should be at the
9root of the job dependency graph).
10
11This requires the :zuul:role:`run-buildset-registry` role already
12applied. It also requires an externally managed "intermediate"
13registry operating for the use of Zuul, and it requires "skopeo" to be
14installed on the Zuul executors.
15
16**Role Variables**
17
18.. zuul:rolevar:: buildset_registry
19
20 Information about the registry, as returned by
21 :zuul:role:`run-buildset-registry`.
22
23 .. zuul:rolevar:: host
24
25 The host (IP address) of the registry.
26
27 .. zuul:rolevar:: port
28
29 The port on which the registry is listening.
30
31 .. zuul:rolevar:: username
32
33 The username used to access the registry via HTTP basic auth.
34
35 .. zuul:rolevar:: password
36
37 The password used to access the registry via HTTP basic auth.
38
39 .. zuul:rolevar:: cert
40
41 The (self-signed) certificate used by the registry.
42
43.. zuul:rolevar:: intermediate_registry
44
45 Information about the registry. This is expected to be provided as
46 a secret.
47
48 .. zuul:rolevar:: host
49
50 The host (IP address) of the registry.
51
52 .. zuul:rolevar:: port
53
54 The port on which the registry is listening.
55
56 .. zuul:rolevar:: username
57
58 The username used to access the registry via HTTP basic auth.
59
60 .. zuul:rolevar:: password
61
62 The password used to access the registry via HTTP basic auth.
diff --git a/roles/pull-from-intermediate-registry/tasks/main.yaml b/roles/pull-from-intermediate-registry/tasks/main.yaml
new file mode 100644
index 0000000..c143b1a
--- /dev/null
+++ b/roles/pull-from-intermediate-registry/tasks/main.yaml
@@ -0,0 +1,75 @@
1# This can be removed if we add this functionality to Zuul directly
2- name: Load information from zuul_return
3 when: buildset_registry is not defined
4 set_fact:
5 buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
6- name: Ensure registry cert directory exists
7 file:
8 path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/"
9 state: directory
10- name: Write registry TLS certificate
11 copy:
12 content: "{{ buildset_registry.cert }}"
13 dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"
14
15
16# Update user config for intermediate and buildset registries
17- name: Ensure docker user directory exists
18 file:
19 state: directory
20 path: "~/.docker"
21 mode: 0700
22- name: Check if docker user configuration exists
23 stat:
24 path: "~/.docker/config.json"
25 register: docker_config_stat
26- name: Load docker user configuration
27 when: docker_config_stat.stat.exists
28 slurp:
29 path: "~/.docker/config.json"
30 register: docker_config
31- name: Parse docker user configuration
32 when: docker_config_stat.stat.exists
33 set_fact:
34 docker_config: "{{ docker_config.content | b64decode | from_json }}"
35- name: Set default docker user configuration
36 when: not docker_config_stat.stat.exists
37 set_fact:
38 docker_config:
39 auths: {}
40- name: Add registry to docker user configuration
41 vars:
42 new_config:
43 auths: |
44 {
45 "{{ intermediate_registry.host }}:{{ intermediate_registry.port }}":
46 {"auth": "{{ (intermediate_registry.username + ":" + intermediate_registry.password) | b64encode }}"},
47 "{{ buildset_registry.host }}:{{ buildset_registry.port }}":
48 {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
49 }
50 set_fact:
51 new_docker_config: "{{ docker_config | combine(new_config, recursive=True) }}"
52- name: Save docker user configuration
53 copy:
54 content: "{{ new_docker_config | to_nice_json }}"
55 dest: "~/.docker/config.json"
56 mode: 0600
57
58# Pull the images
59- name: Pull artifacts from intermediate registry
60 block:
61 - name: Pull artifacts from intermediate registry
62 command: >-
63 skopeo --insecure-policy copy
64 {{ item.url }}
65 docker://{{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }}
66 when: "item.metadata.type | default('') == 'container_image'"
67 loop: "{{ zuul.artifacts | default([]) }}"
68 always:
69 - name: Remove docker user config
70 command: "shred ~/.docker/config.json"
71 - name: Replace docker user configuration
72 copy:
73 content: "{{ docker_config | to_nice_json }}"
74 dest: "~/.docker/config.json"
75 mode: 0600
diff --git a/roles/push-to-intermediate-registry/README.rst b/roles/push-to-intermediate-registry/README.rst
new file mode 100644
index 0000000..ee3939f
--- /dev/null
+++ b/roles/push-to-intermediate-registry/README.rst
@@ -0,0 +1,75 @@
1Push artifacts to the intermediate registry
2
3This role will push any images built by
4:zuul:role:`build-docker-image` into an intermediate registry.
5
6Run this in a trusted post-playbook at the end of a job after the
7image build.
8
9This requires the :zuul:role:`run-buildset-registry` role already
10applied. It also requires an externally managed "intermediate"
11registry operating for the use of Zuul, and it requires "skopeo" to be
12installed on the Zuul executors.
13
14**Role Variables**
15
16.. zuul:rolevar:: buildset_registry
17
18 Information about the registry, as returned by
19 :zuul:role:`run-buildset-registry`.
20
21 .. zuul:rolevar:: host
22
23 The host (IP address) of the registry.
24
25 .. zuul:rolevar:: port
26
27 The port on which the registry is listening.
28
29 .. zuul:rolevar:: username
30
31 The username used to access the registry via HTTP basic auth.
32
33 .. zuul:rolevar:: password
34
35 The password used to access the registry via HTTP basic auth.
36
37 .. zuul:rolevar:: cert
38
39 The (self-signed) certificate used by the registry.
40
41.. zuul:rolevar:: intermediate_registry
42
43 Information about the registry. This is expected to be provided as
44 a secret.
45
46 .. zuul:rolevar:: host
47
48 The host (IP address) of the registry.
49
50 .. zuul:rolevar:: port
51
52 The port on which the registry is listening.
53
54 .. zuul:rolevar:: username
55
56 The username used to access the registry via HTTP basic auth.
57
58 .. zuul:rolevar:: password
59
60 The password used to access the registry via HTTP basic auth.
61
62.. zuul:rolevar:: docker_images
63 :type: list
64
65 A list of images built. Each item in the list should have:
66
67 .. zuul:rolevar:: repository
68
69 The name of the target repository for the image.
70
71 .. zuul:rolevar:: tags
72 :type: list
73 :default: ['latest']
74
75 A list of tags to be added to the image.
diff --git a/roles/push-to-intermediate-registry/tasks/main.yaml b/roles/push-to-intermediate-registry/tasks/main.yaml
new file mode 100644
index 0000000..0d991a0
--- /dev/null
+++ b/roles/push-to-intermediate-registry/tasks/main.yaml
@@ -0,0 +1,3 @@
1- name: Push images to intermediate registry
2 when: docker_images is defined
3 include_tasks: push.yaml
diff --git a/roles/push-to-intermediate-registry/tasks/push-image.yaml b/roles/push-to-intermediate-registry/tasks/push-image.yaml
new file mode 100644
index 0000000..0721a77
--- /dev/null
+++ b/roles/push-to-intermediate-registry/tasks/push-image.yaml
@@ -0,0 +1,23 @@
1- name: Push tag to intermediate registry
2 command: >-
3 skopeo --insecure-policy copy
4 docker://{{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
5 docker://{{ intermediate_registry.host }}:{{ intermediate_registry.port}}/{{ image.repository }}:{{ zuul.build }}_{{ image_tag }}
6 loop: "{{ image.tags | default(['latest']) }}"
7 loop_control:
8 loop_var: image_tag
9
10- name: Return artifact to Zuul
11 zuul_return:
12 data:
13 zuul:
14 artifacts:
15 - name: "image_{{ image.repository }}:{{ image_tag }}"
16 url: "docker://{{ intermediate_registry.host }}:{{ intermediate_registry.port}}/{{ image.repository }}:{{ zuul.build }}_{{ image_tag}}"
17 metadata:
18 type: container_image
19 repository: "{{ image.repository }}"
20 tag: "{{ image_tag }}"
21 loop: "{{ image.tags | default(['latest']) }}"
22 loop_control:
23 loop_var: image_tag
diff --git a/roles/push-to-intermediate-registry/tasks/push.yaml b/roles/push-to-intermediate-registry/tasks/push.yaml
new file mode 100644
index 0000000..36498ff
--- /dev/null
+++ b/roles/push-to-intermediate-registry/tasks/push.yaml
@@ -0,0 +1,72 @@
1# This can be removed if we add this functionality to Zuul directly
2- name: Load information from zuul_return
3 when: buildset_registry is not defined
4 set_fact:
5 buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
6- name: Ensure registry cert directory exists
7 file:
8 path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/"
9 state: directory
10- name: Write registry TLS certificate
11 copy:
12 content: "{{ buildset_registry.cert }}"
13 dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"
14
15# Update user config for intermediate and buildset registries
16- name: Ensure docker user directory exists
17 file:
18 state: directory
19 path: "~/.docker"
20 mode: 0700
21- name: Check if docker user configuration exists
22 stat:
23 path: "~/.docker/config.json"
24 register: docker_config_stat
25- name: Load docker user configuration
26 when: docker_config_stat.stat.exists
27 slurp:
28 path: "~/.docker/config.json"
29 register: docker_config
30- name: Parse docker user configuration
31 when: docker_config_stat.stat.exists
32 set_fact:
33 docker_config: "{{ docker_config.content | b64decode | from_json }}"
34- name: Set default docker user configuration
35 when: not docker_config_stat.stat.exists
36 set_fact:
37 docker_config:
38 auths: {}
39- name: Add registry to docker user configuration
40 vars:
41 new_config:
42 auths: |
43 {
44 "{{ intermediate_registry.host }}:{{ intermediate_registry.port }}":
45 {"auth": "{{ (intermediate_registry.username + ":" + intermediate_registry.password) | b64encode }}"},
46 "{{ buildset_registry.host }}:{{ buildset_registry.port }}":
47 {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
48 }
49 set_fact:
50 new_docker_config: "{{ docker_config | combine(new_config, recursive=True) }}"
51- name: Save docker user configuration
52 copy:
53 content: "{{ new_docker_config | to_nice_json }}"
54 dest: "~/.docker/config.json"
55 mode: 0600
56
57# Push the images
58- name: Push images to intermediate registry
59 block:
60 - name: Push image to intermediate registry
61 include_tasks: push-image.yaml
62 loop: "{{ docker_images }}"
63 loop_control:
64 loop_var: image
65 always:
66 - name: Remove docker user config
67 command: "shred ~/.docker/config.json"
68 - name: Replace docker user configuration
69 copy:
70 content: "{{ docker_config | to_nice_json }}"
71 dest: "~/.docker/config.json"
72 mode: 0600
diff --git a/roles/remove-gpgkey/tasks/main.yaml b/roles/remove-gpgkey/tasks/main.yaml
index e36f111..e4d72ca 100644
--- a/roles/remove-gpgkey/tasks/main.yaml
+++ b/roles/remove-gpgkey/tasks/main.yaml
@@ -1,2 +1,2 @@
1- name: Remove GPG key 1- name: Remove GPG key
2 command: "sh -c 'shred -u ~/.gnupg/*'" 2 shell: find ~/.gnupg/ -type f -exec shred -u {} \;
diff --git a/roles/run-buildset-registry/README.rst b/roles/run-buildset-registry/README.rst
new file mode 100644
index 0000000..4f93764
--- /dev/null
+++ b/roles/run-buildset-registry/README.rst
@@ -0,0 +1,43 @@
1Runs a docker registry for the use of this buildset.
2
3This may be used for a single job running on a single node, or it may
4be used at the root of a job graph so that multiple jobs running for a
5single change can share the registry. Two registry endpoints are
6provided -- one is a local registry, the second is an upstream proxy.
7
8**Role Variables**
9
10.. zuul:rolevar:: buildset_registry_root
11 :default: {{ ansible_user_dir }}/buildset_registry
12
13 Path for the registry volumes.
14
15**Return Values**
16
17.. zuul:rolevar:: buildset_registry
18
19 Information about the registry.
20
21 .. zuul:rolevar:: host
22
23 The host (IP address) of the registry.
24
25 .. zuul:rolevar:: port
26
27 The port on which the registry is listening.
28
29 .. zuul:rolevar:: proxy_port
30
31 The port on which the proxy is listening.
32
33 .. zuul:rolevar:: username
34
35 The username used to access the registry via HTTP basic auth.
36
37 .. zuul:rolevar:: password
38
39 The password used to access the registry via HTTP basic auth.
40
41 .. zuul:rolevar:: cert
42
43 The (self-signed) certificate used by the registry.
diff --git a/roles/run-buildset-registry/defaults/main.yaml b/roles/run-buildset-registry/defaults/main.yaml
new file mode 100644
index 0000000..37c0730
--- /dev/null
+++ b/roles/run-buildset-registry/defaults/main.yaml
@@ -0,0 +1 @@
buildset_registry_root: "{{ ansible_user_dir }}/buildset_registry"
diff --git a/roles/run-buildset-registry/tasks/main.yaml b/roles/run-buildset-registry/tasks/main.yaml
new file mode 100644
index 0000000..3f7c858
--- /dev/null
+++ b/roles/run-buildset-registry/tasks/main.yaml
@@ -0,0 +1,111 @@
1- name: Install packages
2 become: yes
3 package:
4 name:
5 - python-docker
6 - python-openssl
7 - python-passlib
8 - python-bcrypt
9 state: present
10 when: "'python3' not in ansible_python_interpreter"
11- name: Install packages
12 become: yes
13 package:
14 name:
15 - python3-docker
16 - python3-openssl
17 - python3-passlib
18 - python3-bcrypt
19 state: present
20 when: "'python3' in ansible_python_interpreter"
21- name: Ensure Docker registry volume directories exists
22 file:
23 state: directory
24 path: "{{ buildset_registry_root}}/{{ item }}"
25 loop:
26 - certs
27 - auth
28# TODO: use password lookup after allowing access to it in Zuul
29- name: Generate registry password
30 set_fact:
31 registry_password: "{{ (ansible_date_time.iso8601_micro | password_hash('sha256'))[-20:] }}"
32- name: Write htpassword file
33 htpasswd:
34 create: true
35 crypt_scheme: bcrypt
36 path: "{{ buildset_registry_root}}/auth/htpasswd"
37 name: "zuul"
38 password: "{{ registry_password }}"
39- name: Generate a TLS key for the Docker registry
40 openssl_privatekey:
41 path: "{{ buildset_registry_root}}/certs/domain.key"
42- name: Generate a TLS CSR for the Docker registry
43 openssl_csr:
44 path: "{{ buildset_registry_root}}/certs/domain.csr"
45 privatekey_path: "{{ buildset_registry_root}}/certs/domain.key"
46 common_name: "{{ ansible_host }}"
47 subject_alt_name: "DNS:{{ ansible_host }},IP:{{ ansible_host }}"
48- name: Generate a TLS cert for the Docker registry
49 openssl_certificate:
50 path: "{{ buildset_registry_root}}/certs/domain.crt"
51 csr_path: "{{ buildset_registry_root}}/certs/domain.csr"
52 privatekey_path: "{{ buildset_registry_root}}/certs/domain.key"
53 provider: selfsigned
54 register: generated_cert
55- name: Read TLS certificate
56 slurp:
57 src: "{{ generated_cert.filename }}"
58 register: certificate
59- name: Decode TLS certificate
60 set_fact:
61 certificate: "{{ certificate.content | b64decode }}"
62- name: Start a docker registry
63 docker_container:
64 name: buildset_registry
65 image: registry:2
66 state: started
67 restart_policy: always
68 ports:
69 - "5000:5000"
70 env:
71 REGISTRY_HTTP_TLS_CERTIFICATE: /certs/domain.crt
72 REGISTRY_HTTP_TLS_KEY: /certs/domain.key
73 REGISTRY_AUTH: htpasswd
74 REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
75 REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
76 volumes:
77 - "{{ buildset_registry_root}}/certs:/certs"
78 - "{{ buildset_registry_root}}/auth:/auth"
79- name: Start a docker proxy
80 docker_container:
81 name: buildset_proxy
82 image: registry:2
83 state: started
84 restart_policy: always
85 ports:
86 - "5001:5000"
87 env:
88 REGISTRY_HTTP_TLS_CERTIFICATE: /certs/domain.crt
89 REGISTRY_HTTP_TLS_KEY: /certs/domain.key
90 REGISTRY_AUTH: htpasswd
91 REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
92 REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
93 REGISTRY_PROXY_REMOTEURL: https://registry-1.docker.io
94 REGISTRY_PROXY_USERNAME: ''
95 REGISTRY_PROXY_PASSWORD: ''
96 volumes:
97 - "{{ buildset_registry_root}}/certs:/certs"
98 - "{{ buildset_registry_root}}/auth:/auth"
99- name: Set registry information fact
100 set_fact:
101 buildset_registry:
102 host: "{{ ansible_host }}"
103 port: 5000
104 proxy_port: 5001
105 username: zuul
106 password: "{{ registry_password }}"
107 cert: "{{ certificate }}"
108- name: Return registry information to Zuul
109 zuul_return:
110 data:
111 buildset_registry: "{{ buildset_registry }}"
diff --git a/roles/stage-output/README.rst b/roles/stage-output/README.rst
index 9da1b4d..df74640 100644
--- a/roles/stage-output/README.rst
+++ b/roles/stage-output/README.rst
@@ -23,6 +23,16 @@ intended to be used before output fetching in a base job's post-playbook.
23 null overrides the will of a parent job to copy something instructing 23 null overrides the will of a parent job to copy something instructing
24 not to copy. 24 not to copy.
25 25
26 If the type is suffixed with ``_txt``, then the item will have
27 ``.txt`` appended to its name. For example::
28
29 .. code-block:: yaml
30
31 zuul_copy_output:
32 /var/log/syslog: logs_txt
33
34 Will copy ``/var/log/syslog`` to ``logs/syslog.txt``.
35
26.. zuul:rolevar:: stage_dir 36.. zuul:rolevar:: stage_dir
27 :default: {{ ansible_user_dir }} 37 :default: {{ ansible_user_dir }}
28 38
diff --git a/roles/stage-output/tasks/main.yaml b/roles/stage-output/tasks/main.yaml
index 4b59242..76d6f14 100644
--- a/roles/stage-output/tasks/main.yaml
+++ b/roles/stage-output/tasks/main.yaml
@@ -27,12 +27,13 @@
27# NOTE(andreaf) Files or folders that start with a '.' are renamed to starting 27# NOTE(andreaf) Files or folders that start with a '.' are renamed to starting
28# with an '_' else they would not be visible in the logs folder once uploaded. 28# with an '_' else they would not be visible in the logs folder once uploaded.
29# Extension changes are handled later via find as we want to rename files 29# Extension changes are handled later via find as we want to rename files
30# included of folders specified in `zuul_copy_output`. 30# included of folders specified in `zuul_copy_output` (except for logs_txt,
31# which is handled here).
31- name: Set source and destination for files and folders 32- name: Set source and destination for files and folders
32 set_fact: 33 set_fact:
33 source: "{{ item.stat.path }}" 34 source: "{{ item.stat.path }}"
34 dest: "{{ item.item.value }}/{{ item.stat.path|basename|regex_replace('^(\\..*)$', '_\\1') }}" 35 dest: "{{ item.item.value.split('_')[0] }}/{{ item.stat.path|basename|regex_replace('^(\\..*)$', '_\\1') }}{% if item.item.value.endswith('_txt') %}.txt{% endif %}"
35 type: "{{ item.item.value }}" 36 type: "{{ item.item.value.split('_')[0] }}"
36 with_items: "{{ sources.results }}" 37 with_items: "{{ sources.results }}"
37 when: 38 when:
38 - item.stat.exists 39 - item.stat.exists
@@ -59,8 +60,17 @@
59 # remote_src copy does not work recursively, synchronise is restricted by 60 # remote_src copy does not work recursively, synchronise is restricted by
60 # zuul, using command 61 # zuul, using command
61 command: cp -pRL {{ item.source}} {{ stage_dir }}/{{ item.dest }} 62 command: cp -pRL {{ item.source}} {{ stage_dir }}/{{ item.dest }}
63 become: true
62 with_items: "{{ all_sources }}" 64 with_items: "{{ all_sources }}"
63 65
66- name: Make all log files readable
67 file:
68 state: directory
69 dest: "{{ stage_dir }}/logs"
70 mode: u=rwX,g=rX,o=rX
71 recurse: yes
72 become: yes
73
64- name: Discover log files that match extension_list 74- name: Discover log files that match extension_list
65 find: 75 find:
66 paths: "{{ stage_dir }}/logs" 76 paths: "{{ stage_dir }}/logs"
@@ -73,7 +83,8 @@
73- name: Rename log files that match extension_list 83- name: Rename log files that match extension_list
74 shell: "mv {{ item.path }} {{ item.path | regex_replace(extensions_regex, '\\1_\\2.txt') }}" 84 shell: "mv {{ item.path }} {{ item.path | regex_replace(extensions_regex, '\\1_\\2.txt') }}"
75 with_items: "{{ log_files_to_rename.files }}" 85 with_items: "{{ log_files_to_rename.files }}"
76 chdir: "{{ stage_dir }}/logs" 86 args:
87 chdir: "{{ stage_dir }}/logs"
77 tags: 88 tags:
78 - skip_ansible_lint 89 - skip_ansible_lint
79 90
diff --git a/roles/tox/library/tox_install_sibling_packages.py b/roles/tox/library/tox_install_sibling_packages.py
index 4dd23b0..c808114 100644
--- a/roles/tox/library/tox_install_sibling_packages.py
+++ b/roles/tox/library/tox_install_sibling_packages.py
@@ -106,7 +106,12 @@ def get_installed_packages(tox_python):
106 frozen_pkgs = subprocess.check_output( 106 frozen_pkgs = subprocess.check_output(
107 [tox_python, '-m', 'pip', '-qqq', 'freeze'] 107 [tox_python, '-m', 'pip', '-qqq', 'freeze']
108 ) 108 )
109 return [x.split('==')[0] for x in frozen_pkgs.split('\n') if '==' in x] 109 # Matches strings of the form:
110 # 1. '<package_name>==<version>'
111 # 2. '# Editable Git install with no remote (<package_name>==<version>)'
112 # both results: <package_name>
113 return [x[x.find('(') + 1:].split('==')[0]
114 for x in frozen_pkgs.split('\n') if '==' in x]
110 115
111 116
112def write_new_constraints_file(constraints, packages): 117def write_new_constraints_file(constraints, packages):
diff --git a/roles/upload-docker-image/README.rst b/roles/upload-docker-image/README.rst
new file mode 100644
index 0000000..2b04c2e
--- /dev/null
+++ b/roles/upload-docker-image/README.rst
@@ -0,0 +1,3 @@
1Upload one or more docker images.
2
3.. include:: ../../roles/build-docker-image/common.rst
diff --git a/roles/upload-docker-image/defaults/main.yaml b/roles/upload-docker-image/defaults/main.yaml
new file mode 100644
index 0000000..9739eb1
--- /dev/null
+++ b/roles/upload-docker-image/defaults/main.yaml
@@ -0,0 +1 @@
zuul_work_dir: "{{ zuul.project.src_dir }}"
diff --git a/roles/upload-docker-image/tasks/main.yaml b/roles/upload-docker-image/tasks/main.yaml
new file mode 100644
index 0000000..1549090
--- /dev/null
+++ b/roles/upload-docker-image/tasks/main.yaml
@@ -0,0 +1,15 @@
1- name: Verify repository names
2 when: |
3 docker_credentials.repository is defined
4 and not item.repository | regex_search(docker_credentials.repository)
5 loop: "{{ docker_images }}"
6 fail:
7 msg: "{{ item.repository }} not permitted by {{ docker_credentials.repository }}"
8- name: Log in to dockerhub
9 command: "docker login -u {{ docker_credentials.username }} -p {{ docker_credentials.password }}"
10 no_log: true
11- name: Upload image to dockerhub
12 loop: "{{ docker_images }}"
13 loop_control:
14 loop_var: image
15 include_tasks: push.yaml
diff --git a/roles/upload-docker-image/tasks/push.yaml b/roles/upload-docker-image/tasks/push.yaml
new file mode 100644
index 0000000..878a8bb
--- /dev/null
+++ b/roles/upload-docker-image/tasks/push.yaml
@@ -0,0 +1,5 @@
1- name: Upload tag to dockerhub
2 command: "docker push {{ image.repository }}:change_{{ zuul.change }}_{{ image_tag }}"
3 loop: "{{ image.tags | default(['latest']) }}"
4 loop_control:
5 loop_var: image_tag
diff --git a/roles/upload-forge/README.rst b/roles/upload-forge/README.rst
new file mode 100644
index 0000000..a71d2ff
--- /dev/null
+++ b/roles/upload-forge/README.rst
@@ -0,0 +1,24 @@
1Upload puppet module tarball to a Forge server
2
3This role requires the python requests module to be
4installed where Ansible is executing this role.
5
6**Role Variables**
7
8 .. zuul:rolevar:: forge_url
9 :default: https://forgeapi.puppet.com
10
11 The URL to the Puppet Forge API.
12
13 .. zuul:rolevar:: forge_username
14
15 Username to use to log in to Puppet Forge.
16
17 .. zuul:rolevar:: forge_password
18
19 Password to use to log in to Puppet Forge.
20
21 .. zuul:rolevar:: forge_tarball
22
23 Absolute path to the module tarball that should be
24 uploaded.
diff --git a/roles/upload-forge/__init__.py b/roles/upload-forge/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/roles/upload-forge/__init__.py
diff --git a/roles/upload-forge/defaults/main.yaml b/roles/upload-forge/defaults/main.yaml
new file mode 100644
index 0000000..b732452
--- /dev/null
+++ b/roles/upload-forge/defaults/main.yaml
@@ -0,0 +1,2 @@
1---
2forge_url: "https://forgeapi.puppet.com"
diff --git a/roles/upload-forge/library/__init__.py b/roles/upload-forge/library/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/roles/upload-forge/library/__init__.py
diff --git a/roles/upload-forge/library/forge_upload.py b/roles/upload-forge/library/forge_upload.py
new file mode 100644
index 0000000..d477c8e
--- /dev/null
+++ b/roles/upload-forge/library/forge_upload.py
@@ -0,0 +1,182 @@
1#!/usr/bin/env python
2
3# Copyright (c) 2019 Binero
4# Author: Tobias Urdin <tobias.urdin@binero.se>
5#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
18
19ANSIBLE_METADATA = {
20 'metadata_version': '1.0',
21 'status': ['preview'],
22 'supported_by': 'community'
23}
24
25
26DOCUMENTATION = '''
27---
28module: forge_upload
29
30short_description: Uploads a puppet module tarball to a Forge server.
31
32description:
33 - "Uploads a puppet module tarball to a Forge server."
34
35options:
36 username:
37 description:
38 - The username to the Forge account
39 required: true
40 password:
41 description:
42 - The password to the Forge account
43 required: true
44 tarball:
45 description:
46 - The absolute path to the tarball of the puppet module
47 that should be uploaded
48 required: true
49 forgeapi:
50 description:
51 - This base url to the Forge server API, defaults to
52 https://forgeapi.puppet.com
53 required: false
54
55author:
56 - Tobias Urdin (@tobias-urdin)
57'''
58
59
60EXAMPLES = '''
61- name: Upload module
62 forge_upload:
63 username: 'myuser'
64 password: 'mypass'
65 tarball: '/home/myuser/test/pkg/myuser-test-0.1.0.tar.gz'
66'''
67
68
69RETURN = '''
70msg:
71 description: The output message from the module.
72'''
73
74
75from ansible.module_utils.basic import AnsibleModule # noqa
76import os # noqa
77import requests # noqa
78
79
80# Client ID and secret from puppet-blacksmith
81CLIENT_ID = 'b93eb708fd942cfc7b4ed71db6ce219b814954619dbe537ddfd208584e8cff8d'
82CLIENT_SECRET = '216648059ad4afec3e4d77bd9e67817c095b2dcf94cdec18ac3d00584f863180' # noqa
83
84FORGEAPI = 'https://forgeapi.puppet.com'
85
86
87def _get_url(forgeapi, path):
88 path = path[1:] if path.startswith('/') else path
89 return '%s/%s' % (forgeapi, path)
90
91
92def _forge_auth(forgeapi, username, password):
93 url = _get_url(forgeapi, '/oauth/token')
94 data = {
95 'client_id': CLIENT_ID,
96 'client_secret': CLIENT_SECRET,
97 'username': username,
98 'password': password,
99 'grant_type': 'password',
100 }
101 headers = {
102 'User-Agent': 'forge_upload-ansible-module/1.0',
103 }
104 response = requests.post(url, json=data, headers=headers)
105 return response
106
107
108def _forge_upload(forgeapi, token, tarball):
109 url = _get_url(forgeapi, '/v2/releases')
110 data = {
111 'file': open(tarball, 'rb').read(),
112 }
113 headers = {
114 'User-Agent': 'forge_upload-ansible-module/1.0',
115 'Authorization': 'Bearer %s' % token,
116 }
117 response = requests.post(url, files=data, headers=headers)
118 return response
119
120
121def run_module():
122 module_args = dict(
123 username=dict(type='str', required=True),
124 password=dict(type='str', required=True, no_log=True),
125 tarball=dict(type='str', required=True),
126 forgeapi=dict(type='str', default=FORGEAPI),
127 )
128
129 result = dict(
130 changed=False,
131 )
132
133 module = AnsibleModule(
134 argument_spec=module_args,
135 supports_check_mode=True
136 )
137
138 tarball = module.params['tarball']
139 if os.path.exists(tarball) is False:
140 module.fail_json(msg='Tarball %s does not exist' % tarball, **result)
141
142 resp = _forge_auth(module.params['forgeapi'],
143 module.params['username'],
144 module.params['password'])
145
146 if resp.status_code != 200:
147 msg = 'Forge API auth failed with code: %d' % resp.status_code
148 module.fail_json(msg=msg, **result)
149
150 if module.check_mode:
151 return result
152
153 auth = resp.json()
154 token = auth['access_token']
155
156 resp = _forge_upload(module.params['forgeapi'], token, tarball)
157
158 if resp.status_code == 409:
159 msg = 'Module %s already exists on Forge' % tarball
160 module.exit_json(msg=msg, **result)
161
162 if resp.status_code != 201:
163 try:
164 data = resp.json()
165 errors = ','.join(data['errors'])
166 except Exception:
167 errors = 'unknown'
168 msg = 'Forge API failed to upload tarball with code: %d errors: %s' % (
169 resp.status_code, errors)
170 module.fail_json(msg=msg, **result)
171
172 result['changed'] = True
173 module.exit_json(msg='Successfully uploaded tarball %s' % tarball,
174 **result)
175
176
177def main():
178 run_module()
179
180
181if __name__ == '__main__':
182 main()
diff --git a/roles/upload-forge/library/test_forge_upload.py b/roles/upload-forge/library/test_forge_upload.py
new file mode 100644
index 0000000..d3b0d5d
--- /dev/null
+++ b/roles/upload-forge/library/test_forge_upload.py
@@ -0,0 +1,28 @@
1#!/usr/bin/env python
2
3# Copyright (c) 2019 Binero
4# Author: Tobias Urdin <tobias.urdin@binero.se>
5#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
18
19import testtools
20from .forge_upload import _get_url
21
22
23class TestForgeUpload(testtools.TestCase):
24 def test_get_url(self):
25 base_url = 'https://forgeapi.puppet.com'
26 expected = 'https://forgeapi.puppet.com/test'
27 self.assertEqual(_get_url(base_url, '/test'), expected)
28 self.assertEqual(_get_url(base_url, 'test'), expected)
diff --git a/roles/upload-forge/tasks/main.yaml b/roles/upload-forge/tasks/main.yaml
new file mode 100644
index 0000000..4bae16a
--- /dev/null
+++ b/roles/upload-forge/tasks/main.yaml
@@ -0,0 +1,13 @@
1- name: Check required variables
2 assert:
3 that:
4 - "forge_username is defined"
5 - "forge_password is defined"
6 - "forge_tarball is defined"
7
8- name: Upload module to Forge
9 forge_upload:
10 username: "{{ forge_username }}"
11 password: "{{ forge_password }}"
12 tarball: "{{ forge_tarball }}"
13 forgeapi: "{{ forge_url }}"
diff --git a/roles/upload-logs/README.rst b/roles/upload-logs/README.rst
index e9e51f4..6669f3d 100644
--- a/roles/upload-logs/README.rst
+++ b/roles/upload-logs/README.rst
@@ -1,6 +1,22 @@
1Upload logs to a static webserver 1Upload logs to a static webserver
2 2
3This uploads logs to a static webserver using SSH. 3This uploads logs to a static server using SSH. The server must have
4been previously added to the inventory; this can be done with the
5:zuul:role:`add-fileserver` role; see that role's documentation for a
6description of the site_logs secret in this example post-run playbook:
7
8.. code-block:: yaml
9
10 - hosts: localhost
11 roles:
12 - role: add-fileserver
13 fileserver: "{{ site_logs }}"
14
15 - hosts: "{{ site_logs.fqdn }}"
16 gather_facts: False
17 roles:
18 - role: upload-logs
19 zuul_log_url: "http://logs.example.org"
4 20
5**Role Variables** 21**Role Variables**
6 22
diff --git a/roles/upload-pypi/README.rst b/roles/upload-pypi/README.rst
index cbcc0da..aba4b56 100644
--- a/roles/upload-pypi/README.rst
+++ b/roles/upload-pypi/README.rst
@@ -35,3 +35,9 @@ Upload python packages to PyPI
35 :default: twine 35 :default: twine
36 36
37 Path to twine executable. 37 Path to twine executable.
38
39.. zuul:rolevar:: pypi_register_first
40 :default: false
41
42 Whether the role should register the package before uploading it. This may
43 be required when uploading for the first time to a devPI instance.
diff --git a/roles/upload-pypi/defaults/main.yaml b/roles/upload-pypi/defaults/main.yaml
index c0cb93b..1d4b750 100644
--- a/roles/upload-pypi/defaults/main.yaml
+++ b/roles/upload-pypi/defaults/main.yaml
@@ -3,3 +3,4 @@ pypi_path: "src/{{ zuul.project.canonical_name }}/dist"
3pypi_repository: "{{ pypi_info.repository|default('pypi') }}" 3pypi_repository: "{{ pypi_info.repository|default('pypi') }}"
4pypi_repository_url: "{{ pypi_info.repository_url|default(None) }}" 4pypi_repository_url: "{{ pypi_info.repository_url|default(None) }}"
5pypi_twine_executable: twine 5pypi_twine_executable: twine
6pypi_register_first: false
diff --git a/roles/upload-pypi/tasks/main.yaml b/roles/upload-pypi/tasks/main.yaml
index 864fe4d..fb5c10c 100644
--- a/roles/upload-pypi/tasks/main.yaml
+++ b/roles/upload-pypi/tasks/main.yaml
@@ -20,6 +20,11 @@
20 msg: "Found no wheels to upload: {{found_wheels.msg}}" 20 msg: "Found no wheels to upload: {{found_wheels.msg}}"
21 when: found_wheels.files == [] 21 when: found_wheels.files == []
22 22
23- name: Register packages on the PyPI server (via wheels)
24 command: "{{ pypi_twine_executable }} register --config-file {{ _pypirc_tmp.path }} --repository {{ pypi_repository }} {{ item.path }}"
25 with_items: "{{ found_wheels.files }}"
26 when: pypi_register_first
27
23- name: Upload wheel with twine before tarballs 28- name: Upload wheel with twine before tarballs
24 command: "{{ pypi_twine_executable }} upload --config-file {{ _pypirc_tmp.path }} -r {{ pypi_repository }} {{ item.path }}" 29 command: "{{ pypi_twine_executable }} upload --config-file {{ _pypirc_tmp.path }} -r {{ pypi_repository }} {{ item.path }}"
25 with_items: "{{ found_wheels.files }}" 30 with_items: "{{ found_wheels.files }}"
@@ -35,6 +40,13 @@
35 msg: "Found no tarballs to upload: {{found_tarballs.msg}}" 40 msg: "Found no tarballs to upload: {{found_tarballs.msg}}"
36 when: found_tarballs.files == [] 41 when: found_tarballs.files == []
37 42
43- name: Register packages on the PyPI server (via tarballs)
44 command: "{{ pypi_twine_executable }} register --config-file {{ _pypirc_tmp.path }} --repository {{ pypi_repository }} {{ item.path }}"
45 with_items: "{{ found_tarballs.files }}"
46 when:
47 - pypi_register_first
48 - found_wheels.files == []
49
38- name: Upload tarballs with twine 50- name: Upload tarballs with twine
39 command: "{{ pypi_twine_executable }} upload --config-file {{ _pypirc_tmp.path }} -r {{ pypi_repository }} {{ item.path }}" 51 command: "{{ pypi_twine_executable }} upload --config-file {{ _pypirc_tmp.path }} -r {{ pypi_repository }} {{ item.path }}"
40 with_items: "{{ found_tarballs.files }}" 52 with_items: "{{ found_tarballs.files }}"
diff --git a/roles/use-buildset-registry/README.rst b/roles/use-buildset-registry/README.rst
new file mode 100644
index 0000000..8c93942
--- /dev/null
+++ b/roles/use-buildset-registry/README.rst
@@ -0,0 +1,41 @@
1Adds a buildset registry to the docker configuration.
2
3Use this role on any host which should use the buildset registry.
4
5**Role Variables**
6
7.. zuul:rolevar:: buildset_registry
8
9 Information about the registry, as returned by
10 :zuul:role:`run-buildset-registry`.
11
12 .. zuul:rolevar:: host
13
14 The host (IP address) of the registry.
15
16 .. zuul:rolevar:: port
17
18 The port on which the registry is listening.
19
20 .. zuul:rolevar:: proxy_port
21
22 The port on which the registry proxy is listening.
23
24 .. zuul:rolevar:: username
25
26 The username used to access the registry via HTTP basic auth.
27
28 .. zuul:rolevar:: password
29
30 The password used to access the registry via HTTP basic auth.
31
32 .. zuul:rolevar:: cert
33
34 The (self-signed) certificate used by the registry.
35
36.. zuul:rolevar:: buildset_registry_docker_user
37 :default: {{ ansible_user }}
38
39 The system user to configure to use the docker registry. The
40 docker configuration file for this user will be updated. By
41 default, the user Ansible is running as.
diff --git a/roles/use-buildset-registry/tasks/main.yaml b/roles/use-buildset-registry/tasks/main.yaml
new file mode 100644
index 0000000..9977ffd
--- /dev/null
+++ b/roles/use-buildset-registry/tasks/main.yaml
@@ -0,0 +1,77 @@
1- name: Ensure docker directory exists
2 become: yes
3 file:
4 state: directory
5 path: /etc/docker
6- name: Ensure buildset registry cert directory exists
7 become: true
8 file:
9 path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/"
10 state: directory
11- name: Ensure proxy registry cert directory exists
12 become: true
13 file:
14 path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}/"
15 state: directory
16- name: Write buildset registry TLS certificate
17 become: true
18 copy:
19 content: "{{ buildset_registry.cert }}"
20 dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"
21- name: Write proxy registry TLS certificate
22 become: true
23 copy:
24 content: "{{ buildset_registry.cert }}"
25 dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}/ca.crt"
26
27# Update daemon config
28- name: Check if docker daemon configuration exists
29 stat:
30 path: /etc/docker/daemon.json
31 register: docker_config_stat
32- name: Load docker daemon configuration
33 when: docker_config_stat.stat.exists
34 slurp:
35 path: /etc/docker/daemon.json
36 register: docker_config
37- name: Parse docker daemon configuration
38 when: docker_config_stat.stat.exists
39 set_fact:
40 docker_config: "{{ docker_config.content | b64decode | from_json }}"
41- name: Set default docker daemon configuration
42 when: not docker_config_stat.stat.exists
43 set_fact:
44 docker_config:
45 registry-mirrors: []
46- name: Add registry to docker daemon configuration
47 vars:
48 new_config:
49 registry-mirrors: "['https://{{ buildset_registry.host }}:{{ buildset_registry.port}}/', 'https://{{ buildset_registry.host }}:{{ buildset_registry.proxy_port}}/']"
50 set_fact:
51 docker_config: "{{ docker_config | combine(new_config) }}"
52- name: Save docker daemon configuration
53 copy:
54 content: "{{ docker_config | to_nice_json }}"
55 dest: /etc/docker/daemon.json
56 become: true
57
58- name: Restart docker daemon
59 service:
60 name: docker
61 state: restarted
62 become: true
63 register: docker_restart
64 failed_when: docker_restart is failed and not 'Could not find the requested service' in docker_restart.msg
65
66# We use 'block' here to cause the become to apply to all the tasks
67# (which does not automatically happen with include_tasks).
68- name: Update docker user config to use buildset registry
69 become: true
70 become_user: "{{ buildset_registry_docker_user }}"
71 when: buildset_registry_docker_user is defined
72 block:
73 - include_tasks: user-config.yaml
74- name: Update docker user config to use buildset registry
75 when: buildset_registry_docker_user is not defined
76 block:
77 - include_tasks: user-config.yaml
diff --git a/roles/use-buildset-registry/tasks/user-config.yaml b/roles/use-buildset-registry/tasks/user-config.yaml
new file mode 100644
index 0000000..35fc8fe
--- /dev/null
+++ b/roles/use-buildset-registry/tasks/user-config.yaml
@@ -0,0 +1,43 @@
1# Update user config
2- name: Ensure docker user directory exists
3 file:
4 state: directory
5 path: "~/.docker"
6 mode: 0700
7- name: Check if docker user configuration exists
8 stat:
9 path: "~/.docker/config.json"
10 register: docker_config_stat
11- name: Load docker user configuration
12 when: docker_config_stat.stat.exists
13 slurp:
14 path: "~/.docker/config.json"
15 register: docker_config
16- name: Parse docker user configuration
17 when: docker_config_stat.stat.exists
18 set_fact:
19 docker_config: "{{ docker_config.content | b64decode | from_json }}"
20- name: Set default docker user configuration
21 when: not docker_config_stat.stat.exists
22 set_fact:
23 docker_config:
24 auths: {}
25- name: Add registry to docker user configuration
26 vars:
27 new_config:
28 auths: |
29 {
30 "https://index.docker.io/v1/":
31 {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
32 "{{ buildset_registry.host }}:{{ buildset_registry.port }}":
33 {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
34 "{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}":
35 {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}
36 }
37 set_fact:
38 docker_config: "{{ docker_config | combine(new_config, recursive=True) }}"
39- name: Save docker user configuration
40 copy:
41 content: "{{ docker_config | to_nice_json }}"
42 dest: "~/.docker/config.json"
43 mode: 0600
diff --git a/roles/use-docker-mirror/README.rst b/roles/use-docker-mirror/README.rst
new file mode 100644
index 0000000..aedcb49
--- /dev/null
+++ b/roles/use-docker-mirror/README.rst
@@ -0,0 +1,20 @@
1Configure docker to use mirrors if available.
2
3**Role Variables**
4
5.. zuul:rolevar:: mirror_fqdn
6 :default: {{ zuul_site_mirror_fqdn }}
7
8 The base host for mirror servers.
9
10.. zuul:rolevar:: docker_mirror
11
12 URL to override the generated docker hub mirror url based on
13 :zuul:rolevar:`install-docker.mirror_fqdn`.
14
15.. zuul:rolevar:: docker_insecure_registries
16 :default: undefined
17
18 Declare this with a list of insecure registries to define the
19 registries which are allowed to communicate with HTTP only or
20 HTTPS with no valid certificate.
diff --git a/roles/use-docker-mirror/tasks/main.yaml b/roles/use-docker-mirror/tasks/main.yaml
new file mode 100644
index 0000000..dd65f95
--- /dev/null
+++ b/roles/use-docker-mirror/tasks/main.yaml
@@ -0,0 +1,11 @@
1- name: Set mirror_fqdn fact
2 when:
3 - mirror_fqdn is not defined
4 - zuul_site_mirror_fqdn is defined
5 set_fact:
6 mirror_fqdn: "{{ zuul_site_mirror_fqdn }}"
7
8- name: Set up docker mirrors
9 include: mirror.yaml
10 when: mirror_fqdn is defined
11 static: no
diff --git a/roles/install-docker/tasks/mirror.yaml b/roles/use-docker-mirror/tasks/mirror.yaml
index 17a9686..17a9686 100644
--- a/roles/install-docker/tasks/mirror.yaml
+++ b/roles/use-docker-mirror/tasks/mirror.yaml
diff --git a/roles/use-docker-mirror/templates/daemon.json.j2 b/roles/use-docker-mirror/templates/daemon.json.j2
new file mode 100644
index 0000000..2520818
--- /dev/null
+++ b/roles/use-docker-mirror/templates/daemon.json.j2
@@ -0,0 +1,4 @@
1{
2 {% if docker_insecure_registries is defined -%}"insecure-registries": {{ docker_insecure_registries | to_json }},{% endif %}
3 "registry-mirrors": ["{{ docker_mirror }}"]
4}
diff --git a/roles/validate-dco-license/README.rst b/roles/validate-dco-license/README.rst
new file mode 100644
index 0000000..2cf9b0b
--- /dev/null
+++ b/roles/validate-dco-license/README.rst
@@ -0,0 +1,12 @@
1Validate all commits have Signed-off-by header
2
3**Role Variables**
4
5.. zuul:rolevar:: dco_license_failure
6
7 Message to display when Signed-off-by header is missing.
8
9.. zuul:rolevar:: zuul_work_dir
10 :default: {{ zuul.project.src_dir }}
11
12 Directory to DCO license check in.
diff --git a/roles/validate-dco-license/defaults/main.yaml b/roles/validate-dco-license/defaults/main.yaml
new file mode 100644
index 0000000..2a6712c
--- /dev/null
+++ b/roles/validate-dco-license/defaults/main.yaml
@@ -0,0 +1,9 @@
1---
2dco_license_failure: |
3 One or more commits have not been signed properly using --signoff.
4
5 The meaning of a signoff depends on the project, but it typically certifies
6 that committer has the rights to submit this work under the same license and
7 agrees to a Developer Certificate of Origin
8 (see http://developercertificate.org/ for more information).
9zuul_work_dir: "{{ zuul.project.src_dir }}"
diff --git a/roles/validate-dco-license/tasks/main.yaml b/roles/validate-dco-license/tasks/main.yaml
new file mode 100644
index 0000000..47228af
--- /dev/null
+++ b/roles/validate-dco-license/tasks/main.yaml
@@ -0,0 +1,25 @@
1- name: Developer Certificate of Origin (DCO) license check
2 shell:
3 cmd: |
4 set -e
5 result=0
6 for commit in $(git cherry -v origin/{{ zuul.branch }} HEAD | cut -d " " -f2)
7 do
8 if ! git show -s $commit | grep -q "Signed-off-by:"; then
9 echo "---"
10 git show -s $commit
11 echo "---"
12 echo "does not have a Signed-off-by header"
13 result=1
14 fi
15 done
16 exit $result
17 chdir: "{{ zuul_work_dir }}"
18 executable: /bin/bash
19 register: _dco
20 failed_when: _dco.rc > 1
21
22- name: License check failed
23 fail:
24 msg: "{{ dco_license_failure }}"
25 when: _dco.rc != 0
diff --git a/roles/yarn/tasks/main.yaml b/roles/yarn/tasks/main.yaml
index f751cfd..e51187c 100644
--- a/roles/yarn/tasks/main.yaml
+++ b/roles/yarn/tasks/main.yaml
@@ -5,7 +5,7 @@
5 5
6- name: Run yarn lifecycle command 6- name: Run yarn lifecycle command
7 when: yarn_command in yarn_lifecycle_phases 7 when: yarn_command in yarn_lifecycle_phases
8 command: "yarn {{ yarn_command }} --verbose" 8 command: "yarn {{ yarn_command }}"
9 # Need to set DISPLAY to the value that will be set when the virtual 9 # Need to set DISPLAY to the value that will be set when the virtual
10 # framebuffer is set up for doing browser tests. 10 # framebuffer is set up for doing browser tests.
11 environment: 11 environment:
@@ -15,7 +15,7 @@
15 15
16- name: Run yarn custom command 16- name: Run yarn custom command
17 when: yarn_command not in yarn_lifecycle_phases 17 when: yarn_command not in yarn_lifecycle_phases
18 command: "yarn run {{ yarn_command }} --verbose" 18 command: "yarn run {{ yarn_command }}"
19 # Need to set DISPLAY to the value that will be set when the virtual 19 # Need to set DISPLAY to the value that will be set when the virtual
20 # framebuffer is set up for doing browser tests. 20 # framebuffer is set up for doing browser tests.
21 environment: 21 environment:
diff --git a/test-requirements.txt b/test-requirements.txt
index 5cce771..b7e0546 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -22,3 +22,5 @@ openstacksdk>=0.17.1
22requests 22requests
23requestsexceptions 23requestsexceptions
24bs4 24bs4
25sphinxcontrib-blockdiag>=1.1.0
26sphinxcontrib-seqdiag
diff --git a/tox.ini b/tox.ini
index 4ac6fc5..b53e98b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -32,7 +32,8 @@ passenv =
32 # see openstack-zuul-jobs-linters job for more information. 32 # see openstack-zuul-jobs-linters job for more information.
33 ANSIBLE_ROLES_PATH 33 ANSIBLE_ROLES_PATH
34setenv = 34setenv =
35 ANSIBLE_LIBRARY= {envsitepackagesdir}/zuul/ansible/library 35 ANSIBLE_LIBRARY= {envsitepackagesdir}/zuul/ansible/base/library
36 ANSIBLE_ACTION_PLUGINS = {envsitepackagesdir}/zuul/ansible/base/actiongeneral
36whitelist_externals = bash 37whitelist_externals = bash
37commands = 38commands =
38 flake8 {posargs} 39 flake8 {posargs}
diff --git a/zuul.yaml b/zuul.yaml
index c9aeb37..6de795c 100644
--- a/zuul.yaml
+++ b/zuul.yaml
@@ -2,6 +2,14 @@
2# Assumes a 'base' job defined elsewhere 2# Assumes a 'base' job defined elsewhere
3 3
4- job: 4- job:
5 name: dco-license
6 description: |
7 A job to validate all new commits have been signed using --signoff.
8 run: playbooks/dco-license/run.yaml
9 nodeset:
10 nodes: []
11
12- job:
5 name: unittests 13 name: unittests
6 abstract: true 14 abstract: true
7 description: | 15 description: |
@@ -13,6 +21,36 @@
13 post-run: playbooks/unittests/post.yaml 21 post-run: playbooks/unittests/post.yaml
14 22
15- job: 23- job:
24 name: build-docker-image
25 description: |
26 Build a docker image.
27
28 .. include:: ../../playbooks/docker-image/README.rst
29 pre-run: playbooks/docker-image/pre.yaml
30 run: playbooks/docker-image/run.yaml
31
32- job:
33 name: upload-docker-image
34 parent: build-docker-image
35 description: |
36 Build and upload a docker image.
37
38 .. include:: ../../playbooks/docker-image/README.rst
39 .. include:: ../../playbooks/docker-image/credentials.rst
40 post-run: playbooks/docker-image/upload.yaml
41
42- job:
43 name: promote-docker-image
44 description: |
45 Retag a previously-uploaded docker image.
46
47 .. include:: ../../playbooks/docker-image/README.rst
48 .. include:: ../../playbooks/docker-image/credentials.rst
49 run: playbooks/docker-image/promote.yaml
50 nodeset:
51 nodes: []
52
53- job:
16 name: tox 54 name: tox
17 parent: unittests 55 parent: unittests
18 description: | 56 description: |