#OB-553 fix: Deployment issue fixes for OCI

Sunbird-Obsrv · Nov 20, 2023 · 1394e66 · 1394e66
1 parent c826125
commit 1394e66
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 19 deletions.
diff --git a/ansible/roles/analytics-spark-provision/tasks/main.yml b/ansible/roles/analytics-spark-provision/tasks/main.yml
@@ -146,14 +146,14 @@
 - name: Download Kafka-2.11
   become: yes
   become_user: "{{ analytics_user }}"
-  get_url: url=http://downloads.mesosphere.com/kafka/assets/kafka_2.11-0.10.1.0.tgz dest={{ analytics.home }}/kafka_2.11-0.10.1.0.tgz force=no owner={{ analytics_user }} group={{ analytics_group }}
+  get_url: url=https://archive.apache.org/dist/kafka/2.8.0/kafka_2.12-2.8.0.tgz dest={{ analytics.home }}/kafka_2.12-2.8.0.tgz force=no owner={{ analytics_user }} group={{ analytics_group }}
   tags:
     - kafka-provision
 
 - name: Unarchive Kafka  
   become: yes
   become_user: "{{ analytics_user }}"
-  unarchive: src={{ analytics.soft_path }}/kafka_2.11-0.10.1.0.tgz dest={{ analytics.home }}/ copy=no owner={{ analytics_user }} group={{ analytics_group }} creates={{ analytics.home }}/kafka_2.11-0.10.1.0
+  unarchive: src={{ analytics.soft_path }}/kafka_2.12-2.8.0.tgz dest={{ analytics.home }}/ copy=no owner={{ analytics_user }} group={{ analytics_group }} creates={{ analytics.home }}/kafka_2.12-2.8.0
   tags:
     - kafka-provision
 

diff --git a/ansible/roles/data-products-deploy/defaults/main.yml b/ansible/roles/data-products-deploy/defaults/main.yml
@@ -5,7 +5,8 @@ oci_install_loc: /home/{{analytics_user}}/bin/
 
 bucket: "telemetry-data-store"
 secor_bucket: "telemetry-data-store"
-dp_object_store_type: "azure"
+# dp_object_store_type: "azure"
+cloud_service_provider: "azure"
 dp_raw_telemetry_backup_location: "unique/raw/"
 dp_storage_key_config: "azure_storage_key"
 dp_storage_secret_config: "azure_storage_secret"

diff --git a/ansible/roles/data-products-deploy/tasks/main.yml b/ansible/roles/data-products-deploy/tasks/main.yml
@@ -1,20 +1,20 @@
 ## Data products deployment ##
 - name: Ensure azure blob storage container exists
   command: az storage container create --name {{ bucket }}
-  when: dp_object_store_type == "azure"
+  when: cloud_service_provider == "azure"
   tags:
     - always
 
 - name: Ensure oci oss bucket exists
   command: "{{oci_install_loc}}/oci os bucket get --name {{ bucket }}"
   register: check_bucket
-  when: dp_object_store_type == "oci"
+  when: cloud_service_provider == "oci"
   tags:
     - always
 
 - name: Create oci oss bucket 
   command: "{{oci_install_loc}}/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}"
-  when: dp_object_store_type == "oci" and check_bucket.rc !=0
+  when: cloud_service_provider == "oci" and check_bucket.rc !=0
   tags:
     - always
 
@@ -27,15 +27,15 @@
   command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}
   async: 3600
   poll: 10
-  when: dp_object_store_type == "azure"
+  when: cloud_service_provider == "azure"
   tags:
     - dataproducts-spark-cluster 
 
 - name: Copy Core Data Products to oci oss
   command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force"
   async: 3600
   poll: 10
-  when: dp_object_store_type == "oci"
+  when: cloud_service_provider == "oci"
   tags:
     - dataproducts-spark-cluster 
 
@@ -49,15 +49,15 @@
   command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar
   async: 3600
   poll: 10
-  when: dp_object_store_type == "azure"
+  when: cloud_service_provider == "azure"
   tags:
     - ed-dataproducts-spark-cluster   
 
 - name: Copy Ed Data Products to oci oss
   command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force"
   async: 3600
   poll: 10
-  when: dp_object_store_type == "oci"
+  when: cloud_service_provider == "oci"
   tags:
     - ed-dataproducts-spark-cluster     
 
@@ -70,15 +70,15 @@
   command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }}
   async: 3600
   poll: 10
-  when: dp_object_store_type == "azure"
+  when: cloud_service_provider == "azure"
   tags:
     - framework-spark-cluster
 
 - name: Copy Framework Library to oci oss
   command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force"
   async: 3600
   poll: 10
-  when: dp_object_store_type == "oci"
+  when: cloud_service_provider == "oci"
   tags:
     - framework-spark-cluster
 
@@ -91,15 +91,15 @@
   command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }}
   async: 3600
   poll: 10
-  when: dp_object_store_type == "azure"
+  when: cloud_service_provider == "azure"
   tags:
     - framework-spark-cluster  
 
 - name: Copy Scruid Library to oci oss
   command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} --force"
   async: 3600
   poll: 10
-  when: dp_object_store_type == "oci"
+  when: cloud_service_provider == "oci"
   tags:
     - framework-spark-cluster  
 
@@ -136,7 +136,7 @@
     line: 'spark_output_temp_dir="/var/log/sparkapp/tmp/"'
   tags:
     - framework-spark-cluster
-  when: dp_object_store_type != "oci"
+  when: cloud_service_provider != "oci"
 
 - name: Update spark temp dir value for cluster
   lineinfile:
@@ -145,7 +145,7 @@
     line: 'spark_output_temp_dir="/var/log/spark/"'
   tags:
     - framework-spark-cluster
-  when: dp_object_store_type == "oci"
+  when: cloud_service_provider == "oci"
 
 
 - name: Update logger kafka config for cluster
@@ -160,15 +160,15 @@
   command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf
   async: 3600
   poll: 10
-  when: dp_object_store_type == "azure"
+  when: cloud_service_provider == "azure"
   tags:
     - framework-spark-cluster    
 
 - name: Copy configuration file to oci oss
   command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --file {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf --force"
   async: 3600
   poll: 10
-  when: dp_object_store_type == "oci"
+  when: cloud_service_provider == "oci"
   tags:
     - framework-spark-cluster  
 
@@ -416,7 +416,7 @@
   command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/jets3t.properties --file {{ analytics_cluster.home }}/jets3t.properties --content-type auto --force"
   async: 3600
   poll: 10
-  when: dp_object_store_type == "oci"
+  when: cloud_service_provider == "oci"
   tags:
     - replay-job
     - run-job