diff --git a/OCP-4.X/deploy-cluster.yml b/OCP-4.X/deploy-cluster.yml index 126c4e83..2da86a8b 100644 --- a/OCP-4.X/deploy-cluster.yml +++ b/OCP-4.X/deploy-cluster.yml @@ -32,100 +32,102 @@ set_fact: kubeconfig_path: "{{ ansible_user_dir }}/{{ dynamic_deploy_path }}/auth/kubeconfig" roles: - - role: openshift-cleanup - when: openshift_cleanup|bool - - role: openshift-install - when: openshift_install|bool - - role: post-install - when: openshift_post_install|bool - - role: rhacs_install - when: rhacs_enable|bool - - role: cerberus_install - when: cerberus_enable|bool +# - role: openshift-cleanup +# when: openshift_cleanup|bool +# - role: openshift-install +# when: openshift_install|bool +# - role: post-install +# when: openshift_post_install|bool + - role: winc-install + when: openshift_winc_install|bool +# - role: rhacs_install +# when: rhacs_enable|bool +# - role: cerberus_install +# when: cerberus_enable|bool - role: post-config-on-aws when: openshift_post_config|bool and (platform == "aws" or platform == "aws-arm") - - role: post-config-on-azure - when: openshift_post_config|bool and platform == "azure" - - role: post-config-on-gcp - when: openshift_post_config|bool and platform == "gcp" - - role: post-config-on-alibaba - when: openshift_post_config|bool and platform == "alibaba" - - role: cerberus - when: cerberus_enable|bool - - role: data_server_install - when: data_server_enable|bool +# - role: post-config-on-azure +# when: openshift_post_config|bool and platform == "azure" +# - role: post-config-on-gcp +# when: openshift_post_config|bool and platform == "gcp" +# - role: post-config-on-alibaba +# when: openshift_post_config|bool and platform == "alibaba" +# - role: cerberus +# when: cerberus_enable|bool +# - role: data_server_install +# when: data_server_enable|bool -- name: (Optional) Debugging configuration of workload node - hosts: workload - gather_facts: true - become: true - remote_user: core - vars_files: - - vars/install-common-vars.yml - pre_tasks: - - name: Include platform variables - include_vars: - file: "vars/install-on-{{ {{ (platform == 'aws-arm') | ternary('aws', platform) }} }}.yml" - roles: - - role: node-debug-config - when: openshift_debug_config|bool - post_tasks: - - name: Add other nodes to in-memory inventory - when: openshift_debug_config|bool - block: - - name: Get master nodes - shell: | - oc get nodes -l 'node-role.kubernetes.io/master=' | awk 'NR>1 {print $1}' - register: ocp_master_nodes - - - name: Get infra nodes - shell: | - oc get nodes -l 'node-role.kubernetes.io/infra=' | awk 'NR>1 {print $1}' - register: ocp_infra_nodes - - - name: Get worker nodes - shell: | - oc get nodes -l 'node-role.kubernetes.io/worker=' | awk 'NR>1 {print $1}' - register: ocp_worker_nodes - - - name: Add master nodes to in-memory inventory - add_host: - name: "{{ item }}" - groups: master - ansible_ssh_common_args: |- - -o ProxyCommand='ssh -i {{ansible_private_key_file}} -W %h:%p core@{{inventory_hostname}}' - with_items: - - "{{ ocp_master_nodes.stdout_lines }}" - - - name: Add infra nodes to in-memory inventory - add_host: - name: "{{ item }}" - groups: infra - ansible_ssh_common_args: |- - -o ProxyCommand='ssh -i {{ansible_private_key_file}} -W %h:%p core@{{inventory_hostname}}' - with_items: - - "{{ ocp_infra_nodes.stdout_lines }}" - - - name: Add worker nodes to in-memory inventory - add_host: - name: "{{ item }}" - groups: worker - ansible_ssh_common_args: |- - -o ProxyCommand='ssh -i {{ansible_private_key_file}} -W %h:%p core@{{inventory_hostname}}' - with_items: - - "{{ ocp_worker_nodes.stdout_lines }}" - -- name: (Optional) Debugging configuration of initially deployed nodes - hosts: master, infra, worker - gather_facts: true - become: true - remote_user: core - vars_files: - - vars/install-common-vars.yml - pre_tasks: - - name: Include platform variables - include_vars: - file: "vars/install-on-{{ {{ (platform == 'aws-arm') | ternary('aws', platform) }} }}.yml" - roles: - - role: node-debug-config - when: openshift_debug_config|bool +#- name: (Optional) Debugging configuration of workload node +# hosts: workload +# gather_facts: true +# become: true +# remote_user: core +# vars_files: +# - vars/install-common-vars.yml +# pre_tasks: +# - name: Include platform variables +# include_vars: +# file: "vars/install-on-{{ {{ (platform == 'aws-arm') | ternary('aws', platform) }} }}.yml" +# roles: +# - role: node-debug-config +# when: openshift_debug_config|bool +# post_tasks: +# - name: Add other nodes to in-memory inventory +# when: openshift_debug_config|bool +# block: +# - name: Get master nodes +# shell: | +# oc get nodes -l 'node-role.kubernetes.io/master=' | awk 'NR>1 {print $1}' +# register: ocp_master_nodes +# +# - name: Get infra nodes +# shell: | +# oc get nodes -l 'node-role.kubernetes.io/infra=' | awk 'NR>1 {print $1}' +# register: ocp_infra_nodes +# +# - name: Get worker nodes +# shell: | +# oc get nodes -l 'node-role.kubernetes.io/worker=' | awk 'NR>1 {print $1}' +# register: ocp_worker_nodes +# +# - name: Add master nodes to in-memory inventory +# add_host: +# name: "{{ item }}" +# groups: master +# ansible_ssh_common_args: |- +# -o ProxyCommand='ssh -i {{ansible_private_key_file}} -W %h:%p core@{{inventory_hostname}}' +# with_items: +# - "{{ ocp_master_nodes.stdout_lines }}" +# +# - name: Add infra nodes to in-memory inventory +# add_host: +# name: "{{ item }}" +# groups: infra +# ansible_ssh_common_args: |- +# -o ProxyCommand='ssh -i {{ansible_private_key_file}} -W %h:%p core@{{inventory_hostname}}' +# with_items: +# - "{{ ocp_infra_nodes.stdout_lines }}" +# +# - name: Add worker nodes to in-memory inventory +# add_host: +# name: "{{ item }}" +# groups: worker +# ansible_ssh_common_args: |- +# -o ProxyCommand='ssh -i {{ansible_private_key_file}} -W %h:%p core@{{inventory_hostname}}' +# with_items: +# - "{{ ocp_worker_nodes.stdout_lines }}" +# +#- name: (Optional) Debugging configuration of initially deployed nodes +# hosts: master, infra, worker +# gather_facts: true +# become: true +# remote_user: core +# vars_files: +# - vars/install-common-vars.yml +# pre_tasks: +# - name: Include platform variables +# include_vars: +# file: "vars/install-on-{{ {{ (platform == 'aws-arm') | ternary('aws', platform) }} }}.yml" +# roles: +# - role: node-debug-config +# when: openshift_debug_config|bool diff --git a/OCP-4.X/roles/openshift-install/files/cluster-network-03-config.yml b/OCP-4.X/roles/openshift-install/files/cluster-network-03-config.yml new file mode 100644 index 00000000..cda2e94f --- /dev/null +++ b/OCP-4.X/roles/openshift-install/files/cluster-network-03-config.yml @@ -0,0 +1,11 @@ +apiVersion: operator.openshift.io/v1 +kind: Network +metadata: + name: cluster +spec: + defaultNetwork: + ovnKubernetesConfig: + hybridOverlayConfig: + hybridClusterNetwork: + - cidr: 10.132.0.0/14 + hostPrefix: 23 \ No newline at end of file diff --git a/OCP-4.X/roles/openshift-install/tasks/main.yml b/OCP-4.X/roles/openshift-install/tasks/main.yml index b0d595eb..667dd965 100644 --- a/OCP-4.X/roles/openshift-install/tasks/main.yml +++ b/OCP-4.X/roles/openshift-install/tasks/main.yml @@ -304,7 +304,23 @@ when: platform == "alibaba" -- name: Run openshift installer +- name: Create installer dir + shell: | + set -o pipefail + cd {{ ansible_user_dir }}/{{ dynamic_deploy_path }} + export OPENSHIFT_INSTALL_RELEASE_IMAGE_OVERRIDE={{ openshift_install_release_image_override }} + export GOOGLE_CREDENTIALS={{ gcp_auth_key_file|default() }} + bin/openshift-install create manifests --log-level=debug + when: openshfit_network_ovn_hybrid is defined and openshfit_network_ovn_hybrid + +# There is a CIDR hard-coded in this Network object that does not conflict with openshift_cidr for the DAG. If you run into issues, check that the subnets do not overlap. +- name: Template file dir + copy: + src: cluster-network-03-config.yml + dest: "{{ ansible_user_dir }}/{{ dynamic_deploy_path }}/manifests/" + when: openshfit_network_ovn_hybrid is defined and openshfit_network_ovn_hybrid + +- name: Install cluster shell: | set -o pipefail cd {{ ansible_user_dir }}/{{ dynamic_deploy_path }} diff --git a/OCP-4.X/roles/winc-install/defaults/main.yml b/OCP-4.X/roles/winc-install/defaults/main.yml new file mode 100644 index 00000000..e7234758 --- /dev/null +++ b/OCP-4.X/roles/winc-install/defaults/main.yml @@ -0,0 +1,5 @@ +openshift_toggle_windows_node: false +openshift_wmco_image: "quay.io/winc/wmco-index:6.0.0" +openshift_wmco_channel: "stable" +openshift_windows_node_instance_type: "{{ openshift_worker_instance_type }}" +primary_windows_image: "Windows_Server-2019-English-Full-ContainersLatest" \ No newline at end of file diff --git a/OCP-4.X/roles/winc-install/tasks/main.yml b/OCP-4.X/roles/winc-install/tasks/main.yml new file mode 100644 index 00000000..c79af7c9 --- /dev/null +++ b/OCP-4.X/roles/winc-install/tasks/main.yml @@ -0,0 +1,110 @@ +--- +# +# Applies Windows Worker nodes configuration to a cluster post installation +# +# Performs: +# * Creates infra node machineset +# * Creates workload node machineset +# * Moves infra node pods/workload to infra nodes +# These registers be set from post-install tasks, but if you skip those tasks, you can enable them here to run this role one-off. +- name: Get cluster name + shell: | + {%raw%}oc get machineset -n openshift-machine-api -o=go-template='{{(index (index .items 0).metadata.labels {%endraw%} "{{ machineset_metadata_label_prefix }}/cluster-api-cluster" {%raw%})}}'{%endraw%} + register: cluster_name + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + +- name: (AWS) set cluster region + shell: | + {%raw%}oc get machineset -n openshift-machine-api -o=go-template='{{(index .items 0).spec.template.spec.providerSpec.value.placement.region}}'{%endraw%} + register: aws_region + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + +- name: winC AWS - Get Windows AMI ID + shell: | + aws ec2 describe-images --filters Name=name,Values={{ primary_windows_image }}* --region {{ aws_region.stdout }} --query {%raw%}'sort_by(Images, &CreationDate)[-1].[ImageId]'{%endraw%} --output text + register: windows_ami_id + when: platform == "aws" + +- name: winC - Create WMCO catalog source + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig_path }}" + name: openshift-windows-machine-config-operator + template: catalogsource.yaml.j2 + retries: 3 + +- name: winC - Wait for catalog source + shell: oc get catalogsource wmco -o=jsonpath={.status.connectionState.lastObservedState} -n openshift-marketplace + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + register: result + until: result.stdout == "READY" + retries: 3 + delay: 10 + +- name: winC - Create WMCO namespace + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig_path }}" + state: present + api_version: v1 + kind: Namespace + name: openshift-windows-machine-config-operator + +- name: winC - Add monitoring label to wmco namespace + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig_path }}" + state: patched + kind: Namespace + name: openshift-windows-machine-config-operator + definition: + metadata: + labels: + openshift.io/cluster-monitoring: "true" + +- name: winC - Create WMCO operatorgroup + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig_path }}" + state: present + template: operatorgroup.yaml + +- name: winC - Create SSH key secret for WMCO to reach nodes + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig_path }}" + api_version: v1 + kind: Secret + name: cloud-private-key + namespace: openshift-windows-machine-config-operator + definition: + type: Opaque + data: + private-key.pem: "{{ lookup('file', lookup( 'env', 'PRIVATE_KEY' ) ) | b64encode }}" + +- name: winC - Create WMCO subscription + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig_path }}" + state: present + template: subscription.yaml + +- name: winC - Wait for WMCO Deployment to be ready + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfig_path }}" + api_version: apps/v1 + kind: Deployment + name: windows-machine-config-operator + namespace: openshift-windows-machine-config-operator + wait: yes + wait_condition: + type: Available + status: True + reason: MinimumReplicasAvailable + retries: 3 + delay: 10 + +- name: winC AWS - Template out machineset yamls + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig_path }}" + state: present + template: aws-windows-node-machineset.yml.j2 + when: platform == "aws" + diff --git a/OCP-4.X/roles/winc-install/templates/aws-windows-node-machineset.yml.j2 b/OCP-4.X/roles/winc-install/templates/aws-windows-node-machineset.yml.j2 new file mode 100644 index 00000000..a540d473 --- /dev/null +++ b/OCP-4.X/roles/winc-install/templates/aws-windows-node-machineset.yml.j2 @@ -0,0 +1,220 @@ +apiVersion: v1 +items: +- apiVersion: machine.openshift.io/v1beta1 + kind: MachineSet + metadata: + creationTimestamp: null + labels: + {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}} + {{machineset_metadata_label_prefix}}/cluster-api-machine-role: worker + {{machineset_metadata_label_prefix}}/cluster-api-machine-type: worker + name: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}a + namespace: openshift-machine-api + spec: + replicas: 1 + selector: + matchLabels: + {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}} + {{machineset_metadata_label_prefix}}/cluster-api-machineset: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}a + template: + metadata: + creationTimestamp: null + labels: + machine.openshift.io/os-id: Windows + {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}} + {{machineset_metadata_label_prefix}}/cluster-api-machine-role: worker + {{machineset_metadata_label_prefix}}/cluster-api-machine-type: worker + {{machineset_metadata_label_prefix}}/cluster-api-machineset: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}a + spec: + metadata: + creationTimestamp: null + labels: + node-role.kubernetes.io/worker: "" + providerSpec: + value: + ami: + id: {{windows_ami_id.stdout}} + apiVersion: awsproviderconfig.openshift.io/v1beta1 + blockDevices: + - ebs: + iops: {{openshift_worker_root_volume_iops}} + volumeSize: {{openshift_worker_root_volume_size}} + volumeType: {{openshift_worker_root_volume_type}} + credentialsSecret: + name: aws-cloud-credentials + deviceIndex: 0 + iamInstanceProfile: + id: {{cluster_name.stdout}}-worker-profile + instanceType: {{openshift_windows_node_instance_type}} + kind: AWSMachineProviderConfig + metadata: + creationTimestamp: null + placement: + availabilityZone: {{aws_region.stdout}}a + region: {{aws_region.stdout}} + publicIp: false + securityGroups: + - filters: + - name: tag:Name + values: + - {{cluster_name.stdout}}-worker-sg + subnet: + filters: + - name: tag:Name + values: + - {{cluster_name.stdout}}-private-{{aws_region.stdout}}a + tags: + - name: kubernetes.io/cluster/{{cluster_name.stdout}} + value: owned + userDataSecret: + name: windows-user-data + versions: + kubelet: "" + status: + replicas: 0 +- apiVersion: machine.openshift.io/v1beta1 + kind: MachineSet + metadata: + creationTimestamp: null + labels: + {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}} + {{machineset_metadata_label_prefix}}/cluster-api-machine-role: worker + {{machineset_metadata_label_prefix}}/cluster-api-machine-type: worker + name: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}b + namespace: openshift-machine-api + spec: + replicas: 1 + selector: + matchLabels: + {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}} + {{machineset_metadata_label_prefix}}/cluster-api-machineset: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}b + template: + metadata: + creationTimestamp: null + labels: + machine.openshift.io/os-id: Windows + {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}} + {{machineset_metadata_label_prefix}}/cluster-api-machine-role: worker + {{machineset_metadata_label_prefix}}/cluster-api-machine-type: worker + {{machineset_metadata_label_prefix}}/cluster-api-machineset: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}b + spec: + metadata: + creationTimestamp: null + labels: + node-role.kubernetes.io/worker: "" + providerSpec: + value: + ami: + id: {{windows_ami_id.stdout}} + apiVersion: awsproviderconfig.openshift.io/v1beta1 + blockDevices: + - ebs: + iops: {{openshift_worker_root_volume_iops}} + volumeSize: {{openshift_worker_root_volume_size}} + volumeType: {{openshift_worker_root_volume_type}} + credentialsSecret: + name: aws-cloud-credentials + deviceIndex: 0 + iamInstanceProfile: + id: {{cluster_name.stdout}}-worker-profile + instanceType: {{openshift_windows_node_instance_type}} + kind: AWSMachineProviderConfig + metadata: + creationTimestamp: null + placement: + availabilityZone: {{aws_region.stdout}}b + region: {{aws_region.stdout}} + publicIp: false + securityGroups: + - filters: + - name: tag:Name + values: + - {{cluster_name.stdout}}-worker-sg + subnet: + filters: + - name: tag:Name + values: + - {{cluster_name.stdout}}-private-{{aws_region.stdout}}b + tags: + - name: kubernetes.io/cluster/{{cluster_name.stdout}} + value: owned + userDataSecret: + name: windows-user-data + versions: + kubelet: "" + status: + replicas: 0 +- apiVersion: machine.openshift.io/v1beta1 + kind: MachineSet + metadata: + creationTimestamp: null + labels: + {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}} + {{machineset_metadata_label_prefix}}/cluster-api-machine-role: worker + {{machineset_metadata_label_prefix}}/cluster-api-machine-type: worker + name: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}c + namespace: openshift-machine-api + spec: + replicas: 1 + selector: + matchLabels: + {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}} + {{machineset_metadata_label_prefix}}/cluster-api-machineset: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}c + template: + metadata: + creationTimestamp: null + labels: + machine.openshift.io/os-id: Windows + {{machineset_metadata_label_prefix}}/cluster-api-cluster: {{cluster_name.stdout}} + {{machineset_metadata_label_prefix}}/cluster-api-machine-role: worker + {{machineset_metadata_label_prefix}}/cluster-api-machine-type: worker + {{machineset_metadata_label_prefix}}/cluster-api-machineset: {{cluster_name.stdout}}-windows-worker-{{aws_region.stdout}}c + spec: + metadata: + creationTimestamp: null + labels: + node-role.kubernetes.io/worker: "" + providerSpec: + value: + ami: + id: {{windows_ami_id.stdout}} + apiVersion: awsproviderconfig.openshift.io/v1beta1 + blockDevices: + - ebs: + iops: {{openshift_worker_root_volume_iops}} + volumeSize: {{openshift_worker_root_volume_size}} + volumeType: {{openshift_worker_root_volume_type}} + credentialsSecret: + name: aws-cloud-credentials + deviceIndex: 0 + iamInstanceProfile: + id: {{cluster_name.stdout}}-worker-profile + instanceType: {{openshift_windows_node_instance_type}} + kind: AWSMachineProviderConfig + metadata: + creationTimestamp: null + placement: + availabilityZone: {{aws_region.stdout}}c + region: {{aws_region.stdout}} + publicIp: false + securityGroups: + - filters: + - name: tag:Name + values: + - {{cluster_name.stdout}}-worker-sg + subnet: + filters: + - name: tag:Name + values: + - {{cluster_name.stdout}}-private-{{aws_region.stdout}}c + tags: + - name: kubernetes.io/cluster/{{cluster_name.stdout}} + value: owned + userDataSecret: + name: windows-user-data + versions: + kubelet: "" + status: + replicas: 0 +kind: List +metadata: {} diff --git a/OCP-4.X/roles/winc-install/templates/catalogsource.yaml.j2 b/OCP-4.X/roles/winc-install/templates/catalogsource.yaml.j2 new file mode 100644 index 00000000..fc061195 --- /dev/null +++ b/OCP-4.X/roles/winc-install/templates/catalogsource.yaml.j2 @@ -0,0 +1,14 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + name: wmco + # Deploying the openshift-marketplace namespace as it is the global catalog namespace + # a subscription in any namespace can refer to catalogsources in this namespace without error. + namespace: openshift-marketplace +spec: + displayName: Windows Machine Config operators + sourceType: grpc + image: "{{ openshift_wmco_image }}" + updateStrategy: + registryPoll: + interval: 5m diff --git a/OCP-4.X/roles/winc-install/templates/operatorgroup.yaml b/OCP-4.X/roles/winc-install/templates/operatorgroup.yaml new file mode 100644 index 00000000..3f0ac0f2 --- /dev/null +++ b/OCP-4.X/roles/winc-install/templates/operatorgroup.yaml @@ -0,0 +1,8 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: windows-machine-config-operator + namespace: openshift-windows-machine-config-operator +spec: + targetNamespaces: + - openshift-windows-machine-config-operator diff --git a/OCP-4.X/roles/winc-install/templates/subscription.yaml b/OCP-4.X/roles/winc-install/templates/subscription.yaml new file mode 100644 index 00000000..a1b80c95 --- /dev/null +++ b/OCP-4.X/roles/winc-install/templates/subscription.yaml @@ -0,0 +1,11 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: windows-machine-config-operator + namespace: openshift-windows-machine-config-operator +spec: + channel: "{{ openshift_wmco_channel }}" + installPlanApproval: Automatic + name: windows-machine-config-operator + source: wmco + sourceNamespace: openshift-marketplace \ No newline at end of file