Legg til ny datastream fra postgres instansen dataprodukt annullering…

…er til BigQuery.

Co-authored-by: Øyvind Hagen <oyvind.hagen@nav.no>
Co-authored-by: Eirik Gallefoss <eirik.gallefoss@nav.no>

prod/datastream-dataprodukt-annulleringer.tf

@@ -0,0 +1,86 @@
+resource "google_bigquery_dataset" "annulleringer_dataset" {
+  dataset_id = "annulleringer_dataset"
+  location   = var.gcp_project["region"]
+  project    = var.gcp_project["project"]
+  access {
+    role          = "OWNER"
+    special_group = "projectOwners"
+  }
+  access {
+    role          = "READER"
+    special_group = "projectReaders"
+  }
+  access {
+    role          = "WRITER"
+    special_group = "projectWriters"
+  }
+}
+
+resource "google_datastream_connection_profile" "annulleringer_postgresql_connection_profile" {
+  location              = var.gcp_project["region"]
+  display_name          = "annulleringer-postgresql-connection-profile"
+  connection_profile_id = "annulleringer-postgresql-connection-profile"
+
+  postgresql_profile {
+    hostname = google_compute_instance.tbd_datastream_cloud_sql_proxy_vm.network_interface[0].network_ip
+    port     = var.dataprodukt_annulleringer_cloud_sql_port
+    username = local.dataprodukt_annulleringer_db_credentials["username"]
+    password = local.dataprodukt_annulleringer_db_credentials["password"]
+    database = "annulleringer"
+  }
+
+  private_connectivity {
+    private_connection = google_datastream_private_connection.tbd_datastream_private_connection.id
+  }
+}
+
+resource "google_datastream_stream" "annulleringer_datastream" {
+  stream_id     = "annulleringer-datastream"
+  display_name  = "annulleringer-datastream"
+  desired_state = "RUNNING"
+  project       = var.gcp_project["project"]
+  location      = var.gcp_project["region"]
+  labels        = {}
+  backfill_all {}
+  timeouts {}
+
+  source_config {
+    source_connection_profile = google_datastream_connection_profile.annulleringer_postgresql_connection_profile.id
+
+    postgresql_source_config {
+      max_concurrent_backfill_tasks = 0
+      publication                   = "dataprodukt_annulleringer_publication"
+      replication_slot              = "dataprodukt_annulleringer_replication"
+
+      # Exlude flyway schema history table
+      exclude_objects {
+        postgresql_schemas {
+          schema = "public"
+
+          postgresql_tables {
+            table = "flyway_schema_history"
+          }
+        }
+      }
+
+      # Include all tables in the public schema, other than already excluded
+      include_objects {
+        postgresql_schemas {
+          schema = "public"
+        }
+      }
+    }
+  }
+
+  destination_config {
+    destination_connection_profile = google_datastream_connection_profile.datastream_bigquery_connection_profile.id
+
+    bigquery_destination_config {
+      data_freshness = "900s"
+
+      single_target_dataset {
+        dataset_id = "${var.gcp_project["project"]}:${google_bigquery_dataset.annulleringer_dataset.dataset_id}"
+      }
+    }
+  }
+}

prod/datastream-vpc.tf

@@ -41,7 +41,8 @@ resource "google_compute_firewall" "allow_datastream_to_cloud_sql" {
       var.dataprodukt_arbeidsgiveropplysninger_cloud_sql_port,
       var.dataprodukt_forstegangsbehandling_cloud_sql_port,
       var.spaghet_cloud_sql_port,
-      var.spre_styringsinfo_cloud_sql_port
+      var.spre_styringsinfo_cloud_sql_port,
+      var.dataprodukt_annulleringer_cloud_sql_port
     ]
   }
 
@@ -64,12 +65,17 @@ data "google_sql_database_instance" "spre_styringsinfo_db" {
   name = "spre-styringsinfo"
 }
 
+data "google_sql_database_instance" "annulleringer_db" {
+  name = "dataprodukt-annulleringer"
+}
+
 locals {
   proxy_instances = [
     "${data.google_sql_database_instance.dataprodukt_arbeidsgiveropplysninger_db.connection_name}=tcp:0.0.0.0:${var.dataprodukt_arbeidsgiveropplysninger_cloud_sql_port}",
     "${data.google_sql_database_instance.dataprodukt_forstegangsbehandling_db.connection_name}=tcp:0.0.0.0:${var.dataprodukt_forstegangsbehandling_cloud_sql_port}",
     "${data.google_sql_database_instance.spaghet_db.connection_name}=tcp:0.0.0.0:${var.spaghet_cloud_sql_port}",
-    "${data.google_sql_database_instance.spre_styringsinfo_db.connection_name}=tcp:0.0.0.0:${var.spre_styringsinfo_cloud_sql_port}"
+    "${data.google_sql_database_instance.spre_styringsinfo_db.connection_name}=tcp:0.0.0.0:${var.spre_styringsinfo_cloud_sql_port}",
+    "${data.google_sql_database_instance.annulleringer_db.connection_name}=tcp:0.0.0.0:${var.dataprodukt_annulleringer_cloud_sql_port}",
   ]
 }
 

prod/secrets.tf

@@ -14,26 +14,29 @@ data "google_secret_manager_secret_version" "spre_styringsinfo_datastream_user_s
   secret = "spre_styringsinfo_datastream_user_secret"
 }
 
+data "google_secret_manager_secret_version" "annulleringer_datastream_user_secret" {
+  secret = "annulleringer_datastream_user_secret"
+}
+
+# Locals for å decode secrets fra JSON format
 locals {
   dataprodukt_arbeidsgiveropplysninger_db_credentials = jsondecode(
     data.google_secret_manager_secret_version.arbeidsgiveropplysninger_datastream_user_secret.secret_data
   )
-}
 
-locals {
   dataprodukt_forstegangsbehandling_db_credentials = jsondecode(
     data.google_secret_manager_secret_version.forstegangsbehandling_datastream_user_secret.secret_data
   )
-}
 
-locals {
   spaghet_db_credentials = jsondecode(
     data.google_secret_manager_secret_version.spaghet_datastream_user_secret.secret_data
   )
-}
 
-locals {
   spre_styringsinfo_db_credentials = jsondecode(
     data.google_secret_manager_secret_version.spre_styringsinfo_datastream_user_secret.secret_data
   )
+
+  dataprodukt_annulleringer_db_credentials = jsondecode(
+    data.google_secret_manager_secret_version.annulleringer_datastream_user_secret.secret_data
+  )
 }

prod/variables.tf

@@ -31,3 +31,9 @@ variable "spre_styringsinfo_cloud_sql_port" {
   type        = string
   default     = "5435"
 }
+
+variable "dataprodukt_annulleringer_cloud_sql_port" {
+  description = "The port exposed by the spre-styringsinfo database Cloud SQL instance."
+  type        = string
+  default     = "5436"
+}