diff --git a/ansible/roles/analytics-spark-provision/tasks/main.yml b/ansible/roles/analytics-spark-provision/tasks/main.yml index 65731cecb9..b690ba5480 100644 --- a/ansible/roles/analytics-spark-provision/tasks/main.yml +++ b/ansible/roles/analytics-spark-provision/tasks/main.yml @@ -146,14 +146,14 @@ - name: Download Kafka-2.11 become: yes become_user: "{{ analytics_user }}" - get_url: url=http://downloads.mesosphere.com/kafka/assets/kafka_2.11-0.10.1.0.tgz dest={{ analytics.home }}/kafka_2.11-0.10.1.0.tgz force=no owner={{ analytics_user }} group={{ analytics_group }} + get_url: url=https://archive.apache.org/dist/kafka/2.8.0/kafka_2.12-2.8.0.tgz dest={{ analytics.home }}/kafka_2.12-2.8.0.tgz force=no owner={{ analytics_user }} group={{ analytics_group }} tags: - kafka-provision - name: Unarchive Kafka become: yes become_user: "{{ analytics_user }}" - unarchive: src={{ analytics.soft_path }}/kafka_2.11-0.10.1.0.tgz dest={{ analytics.home }}/ copy=no owner={{ analytics_user }} group={{ analytics_group }} creates={{ analytics.home }}/kafka_2.11-0.10.1.0 + unarchive: src={{ analytics.soft_path }}/kafka_2.12-2.8.0.tgz dest={{ analytics.home }}/ copy=no owner={{ analytics_user }} group={{ analytics_group }} creates={{ analytics.home }}/kafka_2.12-2.8.0 tags: - kafka-provision diff --git a/ansible/roles/data-products-deploy/defaults/main.yml b/ansible/roles/data-products-deploy/defaults/main.yml index 06cdf1b0f2..a564f3c66a 100755 --- a/ansible/roles/data-products-deploy/defaults/main.yml +++ b/ansible/roles/data-products-deploy/defaults/main.yml @@ -5,7 +5,8 @@ oci_install_loc: /home/{{analytics_user}}/bin/ bucket: "telemetry-data-store" secor_bucket: "telemetry-data-store" -dp_object_store_type: "azure" +# dp_object_store_type: "azure" +cloud_service_provider: "azure" dp_raw_telemetry_backup_location: "unique/raw/" dp_storage_key_config: "azure_storage_key" dp_storage_secret_config: "azure_storage_secret" diff --git a/ansible/roles/data-products-deploy/tasks/main.yml b/ansible/roles/data-products-deploy/tasks/main.yml index 1c15e14023..6947508627 100644 --- a/ansible/roles/data-products-deploy/tasks/main.yml +++ b/ansible/roles/data-products-deploy/tasks/main.yml @@ -1,20 +1,20 @@ ## Data products deployment ## - name: Ensure azure blob storage container exists command: az storage container create --name {{ bucket }} - when: dp_object_store_type == "azure" + when: cloud_service_provider == "azure" tags: - always - name: Ensure oci oss bucket exists command: "{{oci_install_loc}}/oci os bucket get --name {{ bucket }}" register: check_bucket - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - always - name: Create oci oss bucket command: "{{oci_install_loc}}/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}" - when: dp_object_store_type == "oci" and check_bucket.rc !=0 + when: cloud_service_provider == "oci" and check_bucket.rc !=0 tags: - always @@ -27,7 +27,7 @@ command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} async: 3600 poll: 10 - when: dp_object_store_type == "azure" + when: cloud_service_provider == "azure" tags: - dataproducts-spark-cluster @@ -35,7 +35,7 @@ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force" async: 3600 poll: 10 - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - dataproducts-spark-cluster @@ -49,7 +49,7 @@ command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar async: 3600 poll: 10 - when: dp_object_store_type == "azure" + when: cloud_service_provider == "azure" tags: - ed-dataproducts-spark-cluster @@ -57,7 +57,7 @@ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force" async: 3600 poll: 10 - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - ed-dataproducts-spark-cluster @@ -70,7 +70,7 @@ command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} async: 3600 poll: 10 - when: dp_object_store_type == "azure" + when: cloud_service_provider == "azure" tags: - framework-spark-cluster @@ -78,7 +78,7 @@ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force" async: 3600 poll: 10 - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - framework-spark-cluster @@ -91,7 +91,7 @@ command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} async: 3600 poll: 10 - when: dp_object_store_type == "azure" + when: cloud_service_provider == "azure" tags: - framework-spark-cluster @@ -99,7 +99,7 @@ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} --force" async: 3600 poll: 10 - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - framework-spark-cluster @@ -136,7 +136,7 @@ line: 'spark_output_temp_dir="/var/log/sparkapp/tmp/"' tags: - framework-spark-cluster - when: dp_object_store_type != "oci" + when: cloud_service_provider != "oci" - name: Update spark temp dir value for cluster lineinfile: @@ -145,7 +145,7 @@ line: 'spark_output_temp_dir="/var/log/spark/"' tags: - framework-spark-cluster - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" - name: Update logger kafka config for cluster @@ -160,7 +160,7 @@ command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf async: 3600 poll: 10 - when: dp_object_store_type == "azure" + when: cloud_service_provider == "azure" tags: - framework-spark-cluster @@ -168,7 +168,7 @@ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --file {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf --force" async: 3600 poll: 10 - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - framework-spark-cluster @@ -416,7 +416,7 @@ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/jets3t.properties --file {{ analytics_cluster.home }}/jets3t.properties --content-type auto --force" async: 3600 poll: 10 - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - replay-job - run-job