summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJames E. Blair <jeblair@redhat.com>2019-03-01 15:52:01 -0800
committerJames E. Blair <jeblair@redhat.com>2019-03-01 15:52:01 -0800
commit9c0d25f3498f38126c9d47b6b6a16a008e58fa40 (patch)
tree3a67fc357e20fb5688f8ca2ba4a9f840536b9661
parent2da8976da050e6088ec8c4bcee34db6c98ae7609 (diff)
Fix buildset registry
The approach of having the proxy serve the local data as well as the remote wasn't working -- it seems that the proxy would always check upstream and prefer that data even if it had been pushed locally. To correct this, separate the data stores of the two registries, and add both of them to the registry_mirror setting for the docker daemon. Now we will pull from our buildset registry first, and fall back on the proxy to talk to upstream if an image is not found locally. The proxy is still required in order to mask out the username and password which dockerd will otherwise use when talking to upstream. Change-Id: Iab11954a4b5431d3b1a4d4753f519b6b71f64094
Notes
Notes (review): Code-Review+2: Clark Boylan <cboylan@sapwetik.org> Code-Review+2: James E. Blair <corvus@inaugust.com> Workflow+1: James E. Blair <corvus@inaugust.com> Code-Review+2: Jeremy Stanley <fungi@yuggoth.org> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Sat, 02 Mar 2019 00:50:28 +0000 Reviewed-on: https://review.openstack.org/640557 Project: openstack-infra/zuul-jobs Branch: refs/heads/master
-rw-r--r--roles/build-docker-image/tasks/push.yaml4
-rw-r--r--roles/pull-from-intermediate-registry/tasks/main.yaml6
-rw-r--r--roles/run-buildset-registry/README.rst12
-rw-r--r--roles/run-buildset-registry/tasks/main.yaml19
-rw-r--r--roles/use-buildset-registry/README.rst8
-rw-r--r--roles/use-buildset-registry/tasks/main.yaml14
-rw-r--r--roles/use-buildset-registry/tasks/user-config.yaml2
7 files changed, 26 insertions, 39 deletions
diff --git a/roles/build-docker-image/tasks/push.yaml b/roles/build-docker-image/tasks/push.yaml
index 1f8e449..d49edd1 100644
--- a/roles/build-docker-image/tasks/push.yaml
+++ b/roles/build-docker-image/tasks/push.yaml
@@ -1,12 +1,12 @@
1- name: Tag image for buildset registry 1- name: Tag image for buildset registry
2 command: >- 2 command: >-
3 docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/{{ image.repository }}:{{ image_tag }} 3 docker tag {{ image.repository }}:{{ image_tag }} {{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
4 loop: "{{ image.tags | default(['latest']) }}" 4 loop: "{{ image.tags | default(['latest']) }}"
5 loop_control: 5 loop_control:
6 loop_var: image_tag 6 loop_var: image_tag
7- name: Push tag to buildset registry 7- name: Push tag to buildset registry
8 command: >- 8 command: >-
9 docker push {{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/{{ image.repository }}:{{ image_tag }} 9 docker push {{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ image.repository }}:{{ image_tag }}
10 loop: "{{ image.tags | default(['latest']) }}" 10 loop: "{{ image.tags | default(['latest']) }}"
11 loop_control: 11 loop_control:
12 loop_var: image_tag 12 loop_var: image_tag
diff --git a/roles/pull-from-intermediate-registry/tasks/main.yaml b/roles/pull-from-intermediate-registry/tasks/main.yaml
index ee4c572..f8d1205 100644
--- a/roles/pull-from-intermediate-registry/tasks/main.yaml
+++ b/roles/pull-from-intermediate-registry/tasks/main.yaml
@@ -5,19 +5,19 @@
5 buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}" 5 buildset_registry: "{{ (lookup('file', zuul.executor.work_root + '/results.json') | from_json)['buildset_registry'] }}"
6- name: Ensure registry cert directory exists 6- name: Ensure registry cert directory exists
7 file: 7 file:
8 path: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/" 8 path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/"
9 state: directory 9 state: directory
10- name: Write registry TLS certificate 10- name: Write registry TLS certificate
11 copy: 11 copy:
12 content: "{{ buildset_registry.cert }}" 12 content: "{{ buildset_registry.cert }}"
13 dest: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/ca.crt" 13 dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"
14- name: Pull artifact from intermediate registry 14- name: Pull artifact from intermediate registry
15 command: >- 15 command: >-
16 skopeo --insecure-policy copy 16 skopeo --insecure-policy copy
17 --src-creds={{ intermediate_registry.username }}:{{ intermediate_registry.password }} 17 --src-creds={{ intermediate_registry.username }}:{{ intermediate_registry.password }}
18 --dest-creds={{ buildset_registry.username }}:{{ buildset_registry.password }} 18 --dest-creds={{ buildset_registry.username }}:{{ buildset_registry.password }}
19 {{ item.url }} 19 {{ item.url }}
20 docker://{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }} 20 docker://{{ buildset_registry.host }}:{{ buildset_registry.port }}/{{ item.metadata.repository }}:{{ item.metadata.tag }}
21 when: "item.metadata.type | default('') == 'container_image'" 21 when: "item.metadata.type | default('') == 'container_image'"
22 loop: "{{ zuul.artifacts | default([]) }}" 22 loop: "{{ zuul.artifacts | default([]) }}"
23 # no_log: true TODO(corvus): replace 23 # no_log: true TODO(corvus): replace
diff --git a/roles/run-buildset-registry/README.rst b/roles/run-buildset-registry/README.rst
index 2133cda..4f93764 100644
--- a/roles/run-buildset-registry/README.rst
+++ b/roles/run-buildset-registry/README.rst
@@ -3,9 +3,7 @@ Runs a docker registry for the use of this buildset.
3This may be used for a single job running on a single node, or it may 3This may be used for a single job running on a single node, or it may
4be used at the root of a job graph so that multiple jobs running for a 4be used at the root of a job graph so that multiple jobs running for a
5single change can share the registry. Two registry endpoints are 5single change can share the registry. Two registry endpoints are
6provided -- one is a read-only endpoint which acts as a pull-through 6provided -- one is a local registry, the second is an upstream proxy.
7proxy and serves upstream images as well as those which are pushed to
8the registry. The second is intended only for pushing images.
9 7
10**Role Variables** 8**Role Variables**
11 9
@@ -28,13 +26,9 @@ the registry. The second is intended only for pushing images.
28 26
29 The port on which the registry is listening. 27 The port on which the registry is listening.
30 28
31 .. zuul:rolevar:: push_host 29 .. zuul:rolevar:: proxy_port
32 30
33 The host (IP address) to use when pushing images to the registry. 31 The port on which the proxy is listening.
34
35 .. zuul:rolevar:: push_port
36
37 The port to use when pushing images to the registry.
38 32
39 .. zuul:rolevar:: username 33 .. zuul:rolevar:: username
40 34
diff --git a/roles/run-buildset-registry/tasks/main.yaml b/roles/run-buildset-registry/tasks/main.yaml
index 0b41065..3f7c858 100644
--- a/roles/run-buildset-registry/tasks/main.yaml
+++ b/roles/run-buildset-registry/tasks/main.yaml
@@ -59,9 +59,9 @@
59- name: Decode TLS certificate 59- name: Decode TLS certificate
60 set_fact: 60 set_fact:
61 certificate: "{{ certificate.content | b64decode }}" 61 certificate: "{{ certificate.content | b64decode }}"
62- name: Start a docker proxy 62- name: Start a docker registry
63 docker_container: 63 docker_container:
64 name: buildset_proxy 64 name: buildset_registry
65 image: registry:2 65 image: registry:2
66 state: started 66 state: started
67 restart_policy: always 67 restart_policy: always
@@ -73,16 +73,12 @@
73 REGISTRY_AUTH: htpasswd 73 REGISTRY_AUTH: htpasswd
74 REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd 74 REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
75 REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm 75 REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
76 REGISTRY_PROXY_REMOTEURL: https://registry-1.docker.io
77 REGISTRY_PROXY_USERNAME: ''
78 REGISTRY_PROXY_PASSWORD: ''
79 volumes: 76 volumes:
80 - "{{ buildset_registry_root}}/data:/var/lib/registry"
81 - "{{ buildset_registry_root}}/certs:/certs" 77 - "{{ buildset_registry_root}}/certs:/certs"
82 - "{{ buildset_registry_root}}/auth:/auth" 78 - "{{ buildset_registry_root}}/auth:/auth"
83- name: Start a docker registry 79- name: Start a docker proxy
84 docker_container: 80 docker_container:
85 name: buildset_registry 81 name: buildset_proxy
86 image: registry:2 82 image: registry:2
87 state: started 83 state: started
88 restart_policy: always 84 restart_policy: always
@@ -94,8 +90,10 @@
94 REGISTRY_AUTH: htpasswd 90 REGISTRY_AUTH: htpasswd
95 REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd 91 REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
96 REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm 92 REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
93 REGISTRY_PROXY_REMOTEURL: https://registry-1.docker.io
94 REGISTRY_PROXY_USERNAME: ''
95 REGISTRY_PROXY_PASSWORD: ''
97 volumes: 96 volumes:
98 - "{{ buildset_registry_root}}/data:/var/lib/registry"
99 - "{{ buildset_registry_root}}/certs:/certs" 97 - "{{ buildset_registry_root}}/certs:/certs"
100 - "{{ buildset_registry_root}}/auth:/auth" 98 - "{{ buildset_registry_root}}/auth:/auth"
101- name: Set registry information fact 99- name: Set registry information fact
@@ -103,8 +101,7 @@
103 buildset_registry: 101 buildset_registry:
104 host: "{{ ansible_host }}" 102 host: "{{ ansible_host }}"
105 port: 5000 103 port: 5000
106 push_host: "{{ ansible_host }}" 104 proxy_port: 5001
107 push_port: 5001
108 username: zuul 105 username: zuul
109 password: "{{ registry_password }}" 106 password: "{{ registry_password }}"
110 cert: "{{ certificate }}" 107 cert: "{{ certificate }}"
diff --git a/roles/use-buildset-registry/README.rst b/roles/use-buildset-registry/README.rst
index 8219157..8c93942 100644
--- a/roles/use-buildset-registry/README.rst
+++ b/roles/use-buildset-registry/README.rst
@@ -17,13 +17,9 @@ Use this role on any host which should use the buildset registry.
17 17
18 The port on which the registry is listening. 18 The port on which the registry is listening.
19 19
20 .. zuul:rolevar:: push_host 20 .. zuul:rolevar:: proxy_port
21 21
22 The host (IP address) to use when pushing images to the registry. 22 The port on which the registry proxy is listening.
23
24 .. zuul:rolevar:: push_port
25
26 The port to use when pushing images to the registry.
27 23
28 .. zuul:rolevar:: username 24 .. zuul:rolevar:: username
29 25
diff --git a/roles/use-buildset-registry/tasks/main.yaml b/roles/use-buildset-registry/tasks/main.yaml
index e31a622..9977ffd 100644
--- a/roles/use-buildset-registry/tasks/main.yaml
+++ b/roles/use-buildset-registry/tasks/main.yaml
@@ -3,26 +3,26 @@
3 file: 3 file:
4 state: directory 4 state: directory
5 path: /etc/docker 5 path: /etc/docker
6- name: Ensure registry cert directory exists 6- name: Ensure buildset registry cert directory exists
7 become: true 7 become: true
8 file: 8 file:
9 path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/" 9 path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/"
10 state: directory 10 state: directory
11- name: Ensure push registry cert directory exists 11- name: Ensure proxy registry cert directory exists
12 become: true 12 become: true
13 file: 13 file:
14 path: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/" 14 path: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}/"
15 state: directory 15 state: directory
16- name: Write registry TLS certificate 16- name: Write buildset registry TLS certificate
17 become: true 17 become: true
18 copy: 18 copy:
19 content: "{{ buildset_registry.cert }}" 19 content: "{{ buildset_registry.cert }}"
20 dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt" 20 dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.port }}/ca.crt"
21- name: Write push registry TLS certificate 21- name: Write proxy registry TLS certificate
22 become: true 22 become: true
23 copy: 23 copy:
24 content: "{{ buildset_registry.cert }}" 24 content: "{{ buildset_registry.cert }}"
25 dest: "/etc/docker/certs.d/{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}/ca.crt" 25 dest: "/etc/docker/certs.d/{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}/ca.crt"
26 26
27# Update daemon config 27# Update daemon config
28- name: Check if docker daemon configuration exists 28- name: Check if docker daemon configuration exists
@@ -46,7 +46,7 @@
46- name: Add registry to docker daemon configuration 46- name: Add registry to docker daemon configuration
47 vars: 47 vars:
48 new_config: 48 new_config:
49 registry-mirrors: "['https://{{ buildset_registry.host }}:{{ buildset_registry.port}}/']" 49 registry-mirrors: "['https://{{ buildset_registry.host }}:{{ buildset_registry.port}}/', 'https://{{ buildset_registry.host }}:{{ buildset_registry.proxy_port}}/']"
50 set_fact: 50 set_fact:
51 docker_config: "{{ docker_config | combine(new_config) }}" 51 docker_config: "{{ docker_config | combine(new_config) }}"
52- name: Save docker daemon configuration 52- name: Save docker daemon configuration
diff --git a/roles/use-buildset-registry/tasks/user-config.yaml b/roles/use-buildset-registry/tasks/user-config.yaml
index 24b5e52..35fc8fe 100644
--- a/roles/use-buildset-registry/tasks/user-config.yaml
+++ b/roles/use-buildset-registry/tasks/user-config.yaml
@@ -31,7 +31,7 @@
31 {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}, 31 {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
32 "{{ buildset_registry.host }}:{{ buildset_registry.port }}": 32 "{{ buildset_registry.host }}:{{ buildset_registry.port }}":
33 {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}, 33 {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"},
34 "{{ buildset_registry.push_host }}:{{ buildset_registry.push_port }}": 34 "{{ buildset_registry.host }}:{{ buildset_registry.proxy_port }}":
35 {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"} 35 {"auth": "{{ (buildset_registry.username + ":" + buildset_registry.password) | b64encode }}"}
36 } 36 }
37 set_fact: 37 set_fact: