Merge pull request #1643 from sowmya-dixit/release-5.2.0

Secor file pattern issue fixes for Azure
Sunbird-Obsrv · Jun 4, 2024 · dbb3267 · dbb3267
2 parents bf1d391 + 4222106
commit dbb3267
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 1 deletion.
diff --git a/ansible/roles/analytics-spark-provision/defaults/main.yml b/ansible/roles/analytics-spark-provision/defaults/main.yml
@@ -3,7 +3,7 @@ analytics:
    soft_path: /mount/data/analytics
    base_path: /home/analytics 
 scala_version: 2.12.10
-spark_version: 3.2.1
+spark_version: 3.1.3
 model_version: "2.0" 
 spark:
   home: "{{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7"

diff --git a/ansible/roles/content-snapshot-indexer/defaults/main.yml b/ansible/roles/content-snapshot-indexer/defaults/main.yml
@@ -48,6 +48,7 @@ cloud_storage:
   provider: "azure"
   account_name: "{{cloud_public_storage_accountname}}"
   account_key: "{{cloud_public_storage_secret}}"
+  account_endpoint: ""
 
 cassandra:
   host: "{{lp_cassandra_host}}" ## LMS-Cassandra IP Address.

diff --git a/kubernetes/helm_charts/secor/config/secor.azure.properties b/kubernetes/helm_charts/secor/config/secor.azure.properties
@@ -22,6 +22,20 @@ secor.azure.container.name={{ $.Values.azure_container_name }}
 # Microsoft Azure blob storage path where files are stored within the container.
 secor.azure.path={{- get (get $.Values.secor_jobs $.Release.Name) "base_path" }}
 
+# Output file pattern excluding prefix. Defaults to topic/partition/generation_kafkaPartition_fmOffset.gz.
+# Available placeholders are 
+# topic - The topic name the data is being fetched
+# partition - The partition name
+# generation - Generation
+# kafkaPartition - The kafka partition
+# fmOffset - First Message offset in the file.
+# randomHex - A 4 character random hex to append to the file name
+# currentTimestamp - Time of upload in epoch format
+# currentTime - Time of upload in HH-mm format
+# currentDate - Time of upload in YYYYMMDD format
+# folder - Folder to use based on message id map lookup
+secor.s3.output_file_pattern={{ get (get $.Values.secor_jobs $.Release.Name) "output_file_pattern" }}
+
 ################
 # END MUST SET #
 ################