[CI] Run Kolla Ansible from its own venv

This avoids polluting the module path for other deployed projects,
like Tenks.

Change-Id: I17802f628e1e7db8d1f5caaa815170a8415e995c

tests/deploy-bifrost.sh

@@ -10,6 +10,8 @@ export PYTHONUNBUFFERED=1
 function deploy_bifrost {
     RAW_INVENTORY=/etc/kolla/inventory
 
+    source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
+
     # TODO(mgoddard): run prechecks.
     # Deploy the bifrost container.
     # TODO(mgoddard): add pull action when we have a local registry service in

tests/deploy.sh

@@ -10,6 +10,8 @@ export PYTHONUNBUFFERED=1
 function deploy {
     RAW_INVENTORY=/etc/kolla/inventory
 
+    source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
+
     #TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
     sudo chmod -R 777 /etc/kolla
     # generate self-signed certificates for the optional internal TLS tests

tests/reconfigure.sh

@@ -10,6 +10,8 @@ export PYTHONUNBUFFERED=1
 function reconfigure {
     RAW_INVENTORY=/etc/kolla/inventory
 
+    source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
+
     # TODO(jeffrey4l): make some configure file change and
     # trigger a real reconfigure
     kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/reconfigure-prechecks

tests/run.yml

@@ -25,8 +25,7 @@
         dashboard_enabled: "{{ openstack_core_enabled or scenario in ['monasca'] }}"
         upper_constraints_file: "{{ ansible_env.HOME }}/src/opendev.org/openstack/requirements/upper-constraints.txt"
         docker_image_tag_suffix: "{{ '-aarch64' if ansible_architecture == 'aarch64' else '' }}"
-        pip_user_path_env:
-          PATH: "{{ ansible_env.HOME + '/.local/bin:' + ansible_env.PATH }}"
+        kolla_ansible_venv_path: "{{ ansible_env.HOME }}/kolla-ansible-venv"
 
     - name: Install dig for Designate testing
       become: true
@@ -66,7 +65,6 @@
         # NOTE(yoctozepto): to avoid issues with IPv6 not enabled in the docker daemon
         # and since we don't need isolated networks here, use host networking
         network_mode: host
-  environment: "{{ pip_user_path_env }}"
   tasks:
     - name: detect whether need build images
       set_fact:
@@ -226,12 +224,22 @@
         state: directory
       become: true
 
-    # NOTE(mgoddard): We need a recent pip to install the latest cryptography
-    # library. See https://github.com/pyca/cryptography/issues/5753
-    - name: install pip 19.1.1+
-      command: >-
-        python3 -m pip install --user
-        pip>=19.1.1
+    - name: Create Kolla Ansible venv
+      command:
+        cmd: "python3 -m venv {{ kolla_ansible_venv_path }}"
+        creates: "{{ kolla_ansible_venv_path }}"
+
+    - name: Ensure the latest tested pip
+      pip:
+        name: "pip==22.*"
+        state: latest
+        virtualenv: "{{ kolla_ansible_venv_path }}"
+
+    - name: Ensure the latest tested setuptools
+      pip:
+        name: "setuptools==65.*"
+        state: latest
+        virtualenv: "{{ kolla_ansible_venv_path }}"
 
     - name: install kolla-ansible and dependencies
       vars:
@@ -240,19 +248,23 @@
         # Test latest ansible version on Ubuntu, minimum supported on others.
         ansible_version_constraint: >-
           {{ ansible_version_min if is_upgrade or base_distro != 'ubuntu' else ansible_version_max }}
-      command: >-
-        python3 -m pip install --user
-        -c {{ upper_constraints_file }}
-        {{ kolla_ansible_src_dir }}
-        ansible{{ ansible_version_constraint }}
-        ara<1.0.0
+      pip:
+        extra_args: "-c {{ upper_constraints_file }}"
+        name:
+          - "{{ kolla_ansible_src_dir }}"
+          - "ansible{{ ansible_version_constraint }}"
+          - "ara<1.0.0"
+        virtualenv: "{{ kolla_ansible_venv_path }}"
 
     - name: install Ansible collections
-      command: >-
+      shell: |
+        source {{ kolla_ansible_venv_path }}/bin/activate
         kolla-ansible install-deps
+      args:
+        executable: /bin/bash
 
     - name: get ARA callback plugin path
-      command: "python3 -m ara.setup.callback_plugins"
+      command: "{{ kolla_ansible_venv_path }}/bin/python3 -m ara.setup.callback_plugins"
       changed_when: false
       register: ara_callback_plugins
 
@@ -269,7 +281,7 @@
         remote_src: true
 
     - name: generate passwords
-      command: kolla-genpwd
+      command: "{{ kolla_ansible_venv_path }}/bin/kolla-genpwd"
 
     - name: slurp kolla passwords
       slurp:
@@ -294,7 +306,7 @@
 
     - name: Record the running state of the environment as seen by the setup module
       shell:
-        cmd: ansible all -i {{ kolla_inventory_path }} -e ansible_user={{ ansible_user }} -m setup > /tmp/logs/ansible/initial-setup
+        cmd: "{{ kolla_ansible_venv_path }}/bin/ansible all -i {{ kolla_inventory_path }} -e ansible_user={{ ansible_user }} -m setup > /tmp/logs/ansible/initial-setup"
 
     - name: Set facts for actions
       set_fact:
@@ -306,8 +318,12 @@
     # because the latter hijacks /etc/kolla permissions (due to same directory on the
     # same host being used by both)
     - name: create TLS certificates for octavia
-      command: kolla-ansible octavia-certificates
+      shell: |
+        source {{ kolla_ansible_venv_path }}/bin/activate
+        kolla-ansible octavia-certificates
       when: scenario in ['octavia']
+      args:
+        executable: /bin/bash
 
     # NOTE(mgoddard): We are using the script module here and later to ensure
     # we use the local copy of these scripts, rather than the one on the remote
@@ -324,6 +340,7 @@
         KOLLA_SRC_DIR: "{{ ansible_env.HOME }}/src/opendev.org/openstack/kolla"
         SCENARIO: "{{ scenario }}"
         UPPER_CONSTRAINTS: "{{ upper_constraints_file }}"
+        KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}"
 
     - name: Run init-swift.sh script
       script:
@@ -390,6 +407,8 @@
             chdir: "{{ kolla_ansible_src_dir }}"
           environment:
             TLS_ENABLED: "{{ tls_enabled }}"
+            KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}"
+
         # NOTE(yoctozepto): this is nice as the first step after the deployment
         # because it waits for the services to stabilize well enough so that
         # the dashboard is able to show the login prompt
@@ -476,6 +495,8 @@
             executable: /bin/bash
             chdir: "{{ kolla_ansible_src_dir }}"
           when: scenario == "magnum"
+          environment:
+            KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}"
 
         - name: Run test-octavia.sh script
           script:
@@ -511,6 +532,8 @@
             executable: /bin/bash
             chdir: "{{ kolla_ansible_src_dir }}"
           when: scenario == "mariadb"
+          environment:
+            KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}"
 
         - name: Run test-prometheus-efk.sh script
           script:
@@ -559,7 +582,6 @@
 
 - hosts: primary
   any_errors_fatal: true
-  environment: "{{ pip_user_path_env }}"
   tasks:
     # Upgrade: update config.
     - block:
@@ -628,14 +650,18 @@
               collections: "{{ (old_requirements.collections | rejectattr('name', 'search', 'ansible-collection-kolla') | list) + [new_requirement] }}"
 
         - name: upgrade kolla-ansible
-          command: >-
-            python3 -m pip install --user
-            -c {{ upper_constraints_file }}
-            {{ kolla_ansible_src_dir }}
+          pip:
+            extra_args: "-c {{ upper_constraints_file }}"
+            name:
+              - "{{ kolla_ansible_src_dir }}"
+            virtualenv: "{{ kolla_ansible_venv_path }}"
 
         - name: install Ansible collections
-          command: >-
+          shell: |
+            source {{ kolla_ansible_venv_path }}/bin/activate
             kolla-ansible install-deps
+          args:
+            executable: /bin/bash
 
         # Update passwords.yml to include any new passwords added in this
         # release.
@@ -649,11 +675,11 @@
             remote_src: true
 
         - name: generate new passwords
-          command: kolla-genpwd
+          command: "{{ kolla_ansible_venv_path }}/bin/kolla-genpwd"
 
         - name: merge old and new passwords
           command: >-
-            kolla-mergepwd
+            {{ kolla_ansible_venv_path }}/bin/kolla-mergepwd
             --old /etc/kolla/passwords.yml.old
             --new /etc/kolla/passwords.yml
             --final /etc/kolla/passwords.yml
@@ -664,6 +690,8 @@
             cmd: tests/upgrade.sh
             executable: /bin/bash
             chdir: "{{ kolla_ansible_src_dir }}"
+          environment:
+            KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}"
 
         # NOTE(yoctozepto): this is nice as the first step after the upgrade
         # because it waits for the services to stabilize well enough so that
@@ -708,6 +736,8 @@
             cmd: tests/deploy-bifrost.sh
             executable: /bin/bash
             chdir: "{{ kolla_ansible_src_dir }}"
+          environment:
+            KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}"
 
         - name: Run test-bifrost.sh script
           shell:
@@ -720,6 +750,8 @@
             cmd: tests/upgrade-bifrost.sh
             executable: /bin/bash
             chdir: "{{ kolla_ansible_src_dir }}"
+          environment:
+            KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}"
       when: scenario == "bifrost"
 
 # NOTE(yoctozepto): each host checks itself
@@ -748,13 +780,14 @@
 
 - hosts: primary
   any_errors_fatal: true
-  environment: "{{ pip_user_path_env }}"
   tasks:
     - name: Run reconfigure.sh script
       script:
         cmd: reconfigure.sh
         executable: /bin/bash
         chdir: "{{ kolla_ansible_src_dir }}"
+      environment:
+        KOLLA_ANSIBLE_VENV_PATH: "{{ kolla_ansible_venv_path }}"
       when:
         - not is_upgrade
         - scenario != "bifrost"

tests/setup_gate.sh

@@ -141,6 +141,9 @@ EOF
 setup_openstack_clients
 
 RAW_INVENTORY=/etc/kolla/inventory
+
+source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
 kolla-ansible -i ${RAW_INVENTORY} -e ansible_user=$USER -vvv bootstrap-servers &> /tmp/logs/ansible/bootstrap-servers
+deactivate
 
 prepare_images

tests/test-magnum.sh

@@ -38,7 +38,11 @@ zone_id = ${ZONE_ID}
 EOF
 
     RAW_INVENTORY=/etc/kolla/inventory
+    deactivate
+    source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
     kolla-ansible -i ${RAW_INVENTORY} --tags designate -vvv reconfigure &> /tmp/logs/ansible/reconfigure-designate
+    deactivate
+    source ~/openstackclient-venv/bin/activate
 
     # Create an instance, and check that its name resolves.
     openstack server create --wait --image cirros --flavor m1.tiny --key-name mykey --network demo-net dns-test --wait

tests/test-mariadb.sh

@@ -32,6 +32,7 @@ function test_recovery {
 
 function test_mariadb_logged {
     RAW_INVENTORY=/etc/kolla/inventory
+    source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
     test_recovery
 }
 

tests/upgrade-bifrost.sh

@@ -10,6 +10,8 @@ export PYTHONUNBUFFERED=1
 function upgrade_bifrost {
     RAW_INVENTORY=/etc/kolla/inventory
 
+    source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
+
     # TODO(mgoddard): run prechecks.
     # TODO(mgoddard): add pull action when we have a local registry service in
     # CI.

tests/upgrade.sh

@@ -10,6 +10,8 @@ export PYTHONUNBUFFERED=1
 function upgrade {
     RAW_INVENTORY=/etc/kolla/inventory
 
+    source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
+
     kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/upgrade-prechecks
     kolla-ansible -i ${RAW_INVENTORY} -vvv pull &> /tmp/logs/ansible/pull-upgrade
     kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade &> /tmp/logs/ansible/upgrade