PERFSCALE-1848 AWS Windows workers enablement

OCP-4.X/deploy-cluster.yml

@@ -38,6 +38,8 @@
       when: openshift_install|bool
     - role: post-install
       when: openshift_post_install|bool
+    - role: winc-install
+      when: openshift_winc_install|bool
     - role: rhacs_install
       when: rhacs_enable|bool
     - role: cerberus_install

OCP-4.X/roles/openshift-install/files/cluster-network-03-config.yml

@@ -0,0 +1,11 @@
+apiVersion: operator.openshift.io/v1
+kind: Network
+metadata:
+  name: cluster
+spec:
+  defaultNetwork:
+    ovnKubernetesConfig:
+      hybridOverlayConfig:
+        hybridClusterNetwork:
+        - cidr: 10.132.0.0/14
+          hostPrefix: 23

OCP-4.X/roles/openshift-install/tasks/main.yml

@@ -304,7 +304,23 @@
 
   when: platform == "alibaba"
 
-- name: Run openshift installer
+- name: Create installer dir
+  shell: |
+    set -o pipefail
+    cd {{ ansible_user_dir }}/{{ dynamic_deploy_path }}
+    export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE={{ openshift_install_release_image_override }}
+    export GOOGLE_CREDENTIALS={{ gcp_auth_key_file|default() }}
+    bin/openshift-install create manifests --log-level=debug
+  when: openshfit_network_ovn_hybrid is defined and openshfit_network_ovn_hybrid
+
+# There is a CIDR hard-coded in this Network object that does not conflict with openshift_cidr for the DAG. If you run into issues, check that the subnets do not overlap.
+- name: Template file dir
+  copy:
+    src: cluster-network-03-config.yml
+    dest: "{{ ansible_user_dir }}/{{ dynamic_deploy_path }}/manifests/"
+  when: openshfit_network_ovn_hybrid is defined and openshfit_network_ovn_hybrid
+
+- name: Install cluster
   shell: |
     set -o pipefail
     cd {{ ansible_user_dir }}/{{ dynamic_deploy_path }}

OCP-4.X/roles/winc-install/defaults/main.yml

@@ -0,0 +1,5 @@
+openshift_toggle_windows_node: false
+openshift_wmco_image: "quay.io/winc/wmco-index:6.0.0"
+openshift_wmco_channel: "stable"
+openshift_windows_node_instance_type: "{{ openshift_worker_instance_type }}"
+primary_windows_image: "Windows_Server-2019-English-Full-ContainersLatest"

OCP-4.X/roles/winc-install/tasks/main.yml

@@ -0,0 +1,110 @@
+---
+#
+# Applies Windows Worker nodes configuration to a cluster post installation
+#
+# Performs:
+#  * Creates infra node machineset
+#  * Creates workload node machineset
+#  * Moves infra node pods/workload to infra nodes
+#  These registers be set from post-install tasks, but if you skip those tasks, you can enable them here to run this role one-off.
+# - name: Get cluster name
+#   shell: |
+#     {%raw%}oc get machineset -n openshift-machine-api -o=go-template='{{(index (index .items 0).metadata.labels {%endraw%} "{{ machineset_metadata_label_prefix }}/cluster-api-cluster" {%raw%})}}'{%endraw%}
+#   register: cluster_name
+#   environment:
+#     KUBECONFIG: "{{ kubeconfig_path }}"
+
+# - name: (AWS) set cluster region
+#   shell: |
+#     {%raw%}oc get machineset -n openshift-machine-api -o=go-template='{{(index .items 0).spec.template.spec.providerSpec.value.placement.region}}'{%endraw%}
+#   register: aws_region
+#   environment:
+#     KUBECONFIG: "{{ kubeconfig_path }}"
+
+- name: winC AWS - Get Windows AMI ID
+  shell: |
+    aws ec2 describe-images --filters Name=name,Values={{ primary_windows_image }}* --region {{ aws_region.stdout }} --query {%raw%}'sort_by(Images, &CreationDate)[-1].[ImageId]'{%endraw%} --output text
+  register: windows_ami_id
+  when: platform == "aws"
+
+- name: winC - Create WMCO catalog source
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig_path }}"
+    name: openshift-windows-machine-config-operator
+    template: catalogsource.yaml.j2
+  retries: 3
+
+- name: winC - Wait for catalog source
+  shell: oc get catalogsource wmco -o=jsonpath={.status.connectionState.lastObservedState} -n openshift-marketplace
+  environment:
+    KUBECONFIG: "{{ kubeconfig_path }}"
+  register: result
+  until: result.stdout == "READY"
+  retries: 3
+  delay: 10
+
+- name: winC - Create WMCO namespace
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig_path }}"
+    state: present
+    api_version: v1
+    kind: Namespace
+    name: openshift-windows-machine-config-operator
+
+- name: winC - Add monitoring label to wmco namespace
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig_path }}"
+    state: patched
+    kind: Namespace
+    name: openshift-windows-machine-config-operator
+    definition:
+      metadata:
+        labels:
+          openshift.io/cluster-monitoring: "true"
+
+- name: winC - Create WMCO operatorgroup
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig_path }}"
+    state: present
+    template: operatorgroup.yaml
+
+- name: winC - Create SSH key secret for WMCO to reach nodes
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig_path }}"
+    api_version: v1
+    kind: Secret
+    name: cloud-private-key
+    namespace: openshift-windows-machine-config-operator
+    definition:
+      type: Opaque
+      data:
+        private-key.pem: "{{ lookup('file', lookup( 'env', 'PRIVATE_KEY' ) ) | b64encode }}"
+
+- name: winC - Create WMCO subscription
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig_path }}"
+    state: present
+    template: subscription.yaml
+
+- name: winC - Wait for WMCO Deployment to be ready
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ kubeconfig_path }}"
+    api_version: apps/v1
+    kind: Deployment
+    name: windows-machine-config-operator
+    namespace: openshift-windows-machine-config-operator
+    wait: yes
+    wait_condition:
+      type: Available
+      status: True
+      reason: MinimumReplicasAvailable
+  retries: 3
+  delay: 10
+
+- name: winC AWS - Template out machineset yamls
+  kubernetes.core.k8s:
+    kubeconfig: "{{ kubeconfig_path }}"
+    state: present
+    template: aws-windows-node-machineset.yml.j2
+  when: platform == "aws"
+

OCP-4.X/roles/winc-install/templates/aws-windows-node-machineset.yml.j2

@@ -0,0 +1,220 @@
+apiVersion: v1
+items:
+- apiVersion: machine.openshift.io/v1beta1
+  kind: MachineSet
+  metadata:
+    creationTimestamp: null
+    labels:
+      {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}}
+      {{machineset_metadata_label_prefix}}/cluster-api-machine-role: worker
+      {{machineset_metadata_label_prefix}}/cluster-api-machine-type: worker
+    name: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}a
+    namespace: openshift-machine-api
+  spec:
+    replicas: 1
+    selector:
+      matchLabels:
+        {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}}
+        {{machineset_metadata_label_prefix}}/cluster-api-machineset: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}a
+    template:
+      metadata:
+        creationTimestamp: null
+        labels:
+          machine.openshift.io/os-id: Windows
+          {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}}
+          {{machineset_metadata_label_prefix}}/cluster-api-machine-role: worker
+          {{machineset_metadata_label_prefix}}/cluster-api-machine-type: worker
+          {{machineset_metadata_label_prefix}}/cluster-api-machineset: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}a
+      spec:
+        metadata:
+          creationTimestamp: null
+          labels:
+            node-role.kubernetes.io/worker: ""
+        providerSpec:
+          value:
+            ami:
+              id: {{windows_ami_id.stdout}}
+            apiVersion: awsproviderconfig.openshift.io/v1beta1
+            blockDevices:
+            - ebs:
+                iops: {{openshift_worker_root_volume_iops}}
+                volumeSize: {{openshift_worker_root_volume_size}}
+                volumeType: {{openshift_worker_root_volume_type}}
+            credentialsSecret:
+              name: aws-cloud-credentials
+            deviceIndex: 0
+            iamInstanceProfile:
+              id: {{cluster_name.stdout}}-worker-profile
+            instanceType: {{openshift_windows_node_instance_type}}
+            kind: AWSMachineProviderConfig
+            metadata:
+              creationTimestamp: null
+            placement:
+              availabilityZone: {{aws_region.stdout}}a
+              region: {{aws_region.stdout}}
+            publicIp: false
+            securityGroups:
+            - filters:
+              - name: tag:Name
+                values:
+                - {{cluster_name.stdout}}-worker-sg
+            subnet:
+              filters:
+              - name: tag:Name
+                values:
+                - {{cluster_name.stdout}}-private-{{aws_region.stdout}}a
+            tags:
+            - name: kubernetes.io/cluster/{{cluster_name.stdout}}
+              value: owned
+            userDataSecret:
+              name: windows-user-data
+        versions:
+          kubelet: ""
+  status:
+    replicas: 0
+- apiVersion: machine.openshift.io/v1beta1
+  kind: MachineSet
+  metadata:
+    creationTimestamp: null
+    labels:
+      {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}}
+      {{machineset_metadata_label_prefix}}/cluster-api-machine-role: worker
+      {{machineset_metadata_label_prefix}}/cluster-api-machine-type: worker
+    name: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}b
+    namespace: openshift-machine-api
+  spec:
+    replicas: 1
+    selector:
+      matchLabels:
+        {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}}
+        {{machineset_metadata_label_prefix}}/cluster-api-machineset: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}b
+    template:
+      metadata:
+        creationTimestamp: null
+        labels:
+          machine.openshift.io/os-id: Windows
+          {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}}
+          {{machineset_metadata_label_prefix}}/cluster-api-machine-role: worker
+          {{machineset_metadata_label_prefix}}/cluster-api-machine-type: worker
+          {{machineset_metadata_label_prefix}}/cluster-api-machineset: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}b
+      spec:
+        metadata:
+          creationTimestamp: null
+          labels:
+            node-role.kubernetes.io/worker: ""
+        providerSpec:
+          value:
+            ami:
+              id: {{windows_ami_id.stdout}}
+            apiVersion: awsproviderconfig.openshift.io/v1beta1
+            blockDevices:
+            - ebs:
+                iops: {{openshift_worker_root_volume_iops}}
+                volumeSize: {{openshift_worker_root_volume_size}}
+                volumeType: {{openshift_worker_root_volume_type}}
+            credentialsSecret:
+              name: aws-cloud-credentials
+            deviceIndex: 0
+            iamInstanceProfile:
+              id: {{cluster_name.stdout}}-worker-profile
+            instanceType: {{openshift_windows_node_instance_type}}
+            kind: AWSMachineProviderConfig
+            metadata:
+              creationTimestamp: null
+            placement:
+              availabilityZone: {{aws_region.stdout}}b
+              region: {{aws_region.stdout}}
+            publicIp: false
+            securityGroups:
+            - filters:
+              - name: tag:Name
+                values:
+                - {{cluster_name.stdout}}-worker-sg
+            subnet:
+              filters:
+              - name: tag:Name
+                values:
+                - {{cluster_name.stdout}}-private-{{aws_region.stdout}}b
+            tags:
+            - name: kubernetes.io/cluster/{{cluster_name.stdout}}
+              value: owned
+            userDataSecret:
+              name: windows-user-data
+        versions:
+          kubelet: ""
+  status:
+    replicas: 0
+- apiVersion: machine.openshift.io/v1beta1
+  kind: MachineSet
+  metadata:
+    creationTimestamp: null
+    labels:
+      {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}}
+      {{machineset_metadata_label_prefix}}/cluster-api-machine-role: worker
+      {{machineset_metadata_label_prefix}}/cluster-api-machine-type: worker
+    name: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}c
+    namespace: openshift-machine-api
+  spec:
+    replicas: 1
+    selector:
+      matchLabels:
+        {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}}
+        {{machineset_metadata_label_prefix}}/cluster-api-machineset: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}c
+    template:
+      metadata:
+        creationTimestamp: null
+        labels:
+          machine.openshift.io/os-id: Windows
+          {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}}
+          {{machineset_metadata_label_prefix}}/cluster-api-machine-role: worker
+          {{machineset_metadata_label_prefix}}/cluster-api-machine-type: worker
+          {{machineset_metadata_label_prefix}}/cluster-api-machineset: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}c
+      spec:
+        metadata:
+          creationTimestamp: null
+          labels:
+            node-role.kubernetes.io/worker: ""
+        providerSpec:
+          value:
+            ami:
+              id: {{windows_ami_id.stdout}}
+            apiVersion: awsproviderconfig.openshift.io/v1beta1
+            blockDevices:
+            - ebs:
+                iops: {{openshift_worker_root_volume_iops}}
+                volumeSize: {{openshift_worker_root_volume_size}}
+                volumeType: {{openshift_worker_root_volume_type}}
+            credentialsSecret:
+              name: aws-cloud-credentials
+            deviceIndex: 0
+            iamInstanceProfile:
+              id: {{cluster_name.stdout}}-worker-profile
+            instanceType: {{openshift_windows_node_instance_type}}
+            kind: AWSMachineProviderConfig
+            metadata:
+              creationTimestamp: null
+            placement:
+              availabilityZone: {{aws_region.stdout}}c
+              region: {{aws_region.stdout}}
+            publicIp: false
+            securityGroups:
+            - filters:
+              - name: tag:Name
+                values:
+                - {{cluster_name.stdout}}-worker-sg
+            subnet:
+              filters:
+              - name: tag:Name
+                values:
+                - {{cluster_name.stdout}}-private-{{aws_region.stdout}}c
+            tags:
+            - name: kubernetes.io/cluster/{{cluster_name.stdout}}
+              value: owned
+            userDataSecret:
+              name: windows-user-data
+        versions:
+          kubelet: ""
+  status:
+    replicas: 0
+kind: List
+metadata: {}