Use multiple Ceph secrets in ci_dcn_site role

Without this patch the ci_dcn_site role creates
one secret with all Ceph keys and distributes
all of these keys to all DCN sites. This patch
makes the role create one secret for each DCN
site and only that secret is used at that site.
The secret for each DCN site only has the keys
that are required. This will make the testing
consistent with the documentation.

Additional secrets for Ceph configuration are
created and `extraMounts` for both the control
plane and each dataplane is extended to include
the extra secrets so that not all of the Ceph
keys are distributed.

Jira: https://issues.redhat.com/browse/OSPRH-6603

Signed-off-by: John Fulton <fulton@redhat.com>

roles/ci_dcn_site/tasks/az.yml

@@ -44,6 +44,8 @@
       openstack aggregate create {{ _az }} --zone {{ _az }}
 
 - name: Add only the missing edpm hosts to AZ
+  ignore_errors: true
+  register: ignore_errors_register
   loop: "{{ _edpm_hosts | dict2items }}"
   when:
     - item.key not in az_hosts_list

roles/ci_dcn_site/tasks/post-ceph.yml

@@ -54,13 +54,44 @@
   ansible.builtin.set_fact:
     ci_dcn_site_glance_map: "{{ ci_dcn_site_glance_map | combine( { item: ['az0', item ] } ) }}"
 
+- name: Render ceph secret for this _az
+  when: _az != "az0"
+  vars:
+    _head: "/tmp/"
+    _tail_key: ".client.openstack.keyring"
+    _tail_conf: ".conf"
+  ansible.builtin.template:
+    mode: "0644"
+    backup: false
+    src: "templates/ceph_secerts.yaml.j2"
+    dest: "{{ ci_dcn_site_arch_path }}/ceph_secrets_{{ _az }}.yaml"
+
+- name: Apply ceph secret for this _az
+  when: _az != "az0"
+  register: result
+  retries: 5
+  delay: 10
+  until: result is not failed
+  kubernetes.core.k8s:
+    api_key: "{{ _auth_results.openshift_auth.api_key }}"
+    state: present
+    apply: true
+    src: "{{ ci_dcn_site_arch_path }}/ceph_secrets_{{ _az }}.yaml"
+
 - name: Render the post-ceph service-values.yaml
   ansible.builtin.template:
     mode: "0644"
     backup: true
     src: "templates/service-values.yaml.j2"
     dest: "{{ ci_dcn_site_arch_path }}/service-values.yaml"
 
+- name: Render EDPM extra_mounts.yaml for this _az
+  ansible.builtin.template:
+    mode: "0644"
+    backup: true
+    src: "templates/edpm_extra_mounts.yaml.j2"
+    dest: "{{ ci_dcn_site_arch_repo_path }}/dt/dcn/edpm-post-ceph/nodeset/extra_mounts.yaml"
+
 - name: Kustomize post-ceph NodeSet
   ansible.builtin.set_fact:
     post_ceph_nodeset_cr: >-

roles/ci_dcn_site/templates/ceph_secerts.yaml.j2

@@ -0,0 +1,12 @@
+---
+kind: Secret
+metadata:
+  name: ceph-conf-files-{{ _az }}
+  namespace: openstack
+type: Opaque
+apiVersion: v1
+data:
+{% for __az in ['az0', _az] %}
+  {{ __az ~ _tail_key }}: {{ lookup('file', _head ~ __az ~ _tail_key, rstrip=False) | b64encode }}
+  {{ __az ~ _tail_conf }}: {{ lookup('file', _head ~ __az ~ _tail_conf, rstrip=False) | b64encode }}
+{% endfor %}

roles/ci_dcn_site/templates/edpm_extra_mounts.yaml.j2

@@ -0,0 +1,21 @@
+---
+apiVersion: dataplane.openstack.org/v1beta1
+kind: OpenStackDataPlaneNodeSet
+metadata:
+  name: openstack-edpm
+spec:
+  nodeTemplate:
+    extraMounts:
+      - extraVolType: Ceph
+        mounts:
+          - mountPath: /etc/ceph
+            name: ceph
+            readOnly: true
+        volumes:
+          - name: ceph
+            secret:
+{% if _az != 'az0' %}
+              secretName: ceph-conf-files-{{ _az }}
+{% else %}
+              secretName: ceph-conf-files
+{% endif %}

roles/ci_dcn_site/templates/service-values.yaml.j2

@@ -8,6 +8,8 @@ metadata:
     config.kubernetes.io/local-config: "true"
 data:
   preserveJobs: false
+  cinder:
+    uniquePodNames: false
   cinderAPI:
     replicas: 3
     customServiceConfig: |
@@ -115,10 +117,8 @@ data:
       region: r1
       extraVol:
         - propagation:
-            - CinderVolume
+            - az0
             - CinderBackup
-            - GlanceAPI
-            - ManilaShare
           extraVolType: Ceph
           volumes:
             - name: ceph
@@ -130,3 +130,20 @@ data:
             - name: ceph
               mountPath: /etc/ceph
               readOnly: true
+{% for _ceph in _ceph_vars_list %}
+{% if _ceph.cifmw_ceph_client_cluster != 'az0' %}
+        - propagation:
+            - {{ _ceph.cifmw_ceph_client_cluster }}
+          extraVolType: Ceph
+          volumes:
+            - name: ceph-{{ _ceph.cifmw_ceph_client_cluster }}
+              projected:
+                sources:
+                  - secret:
+                      name: ceph-conf-files-{{ _ceph.cifmw_ceph_client_cluster }}
+          mounts:
+            - name: ceph-{{ _ceph.cifmw_ceph_client_cluster }}
+              mountPath: /etc/ceph
+              readOnly: true
+{% endif %}
+{% endfor %}