diff --git a/prod/datastream-dataprodukt-annulleringer.tf b/prod/datastream-dataprodukt-annulleringer.tf new file mode 100644 index 0000000..e55f8c6 --- /dev/null +++ b/prod/datastream-dataprodukt-annulleringer.tf @@ -0,0 +1,86 @@ +resource "google_bigquery_dataset" "annulleringer_dataset" { + dataset_id = "annulleringer_dataset" + location = var.gcp_project["region"] + project = var.gcp_project["project"] + access { + role = "OWNER" + special_group = "projectOwners" + } + access { + role = "READER" + special_group = "projectReaders" + } + access { + role = "WRITER" + special_group = "projectWriters" + } +} + +resource "google_datastream_connection_profile" "annulleringer_postgresql_connection_profile" { + location = var.gcp_project["region"] + display_name = "annulleringer-postgresql-connection-profile" + connection_profile_id = "annulleringer-postgresql-connection-profile" + + postgresql_profile { + hostname = google_compute_instance.tbd_datastream_cloud_sql_proxy_vm.network_interface[0].network_ip + port = var.dataprodukt_annulleringer_cloud_sql_port + username = local.dataprodukt_annulleringer_db_credentials["username"] + password = local.dataprodukt_annulleringer_db_credentials["password"] + database = "annulleringer" + } + + private_connectivity { + private_connection = google_datastream_private_connection.tbd_datastream_private_connection.id + } +} + +resource "google_datastream_stream" "annulleringer_datastream" { + stream_id = "annulleringer-datastream" + display_name = "annulleringer-datastream" + desired_state = "RUNNING" + project = var.gcp_project["project"] + location = var.gcp_project["region"] + labels = {} + backfill_all {} + timeouts {} + + source_config { + source_connection_profile = google_datastream_connection_profile.annulleringer_postgresql_connection_profile.id + + postgresql_source_config { + max_concurrent_backfill_tasks = 0 + publication = "dataprodukt_annulleringer_publication" + replication_slot = "dataprodukt_annulleringer_replication" + + # Exlude flyway schema history table + exclude_objects { + postgresql_schemas { + schema = "public" + + postgresql_tables { + table = "flyway_schema_history" + } + } + } + + # Include all tables in the public schema, other than already excluded + include_objects { + postgresql_schemas { + schema = "public" + } + } + } + } + + destination_config { + destination_connection_profile = google_datastream_connection_profile.datastream_bigquery_connection_profile.id + + bigquery_destination_config { + data_freshness = "900s" + + single_target_dataset { + dataset_id = "${var.gcp_project["project"]}:${google_bigquery_dataset.annulleringer_dataset.dataset_id}" + } + } + } +} diff --git a/prod/datastream-vpc.tf b/prod/datastream-vpc.tf index 25c48ee..c5fe288 100644 --- a/prod/datastream-vpc.tf +++ b/prod/datastream-vpc.tf @@ -41,7 +41,8 @@ resource "google_compute_firewall" "allow_datastream_to_cloud_sql" { var.dataprodukt_arbeidsgiveropplysninger_cloud_sql_port, var.dataprodukt_forstegangsbehandling_cloud_sql_port, var.spaghet_cloud_sql_port, - var.spre_styringsinfo_cloud_sql_port + var.spre_styringsinfo_cloud_sql_port, + var.dataprodukt_annulleringer_cloud_sql_port ] } @@ -64,12 +65,17 @@ data "google_sql_database_instance" "spre_styringsinfo_db" { name = "spre-styringsinfo" } +data "google_sql_database_instance" "annulleringer_db" { + name = "dataprodukt-annulleringer" +} + locals { proxy_instances = [ "${data.google_sql_database_instance.dataprodukt_arbeidsgiveropplysninger_db.connection_name}=tcp:0.0.0.0:${var.dataprodukt_arbeidsgiveropplysninger_cloud_sql_port}", "${data.google_sql_database_instance.dataprodukt_forstegangsbehandling_db.connection_name}=tcp:0.0.0.0:${var.dataprodukt_forstegangsbehandling_cloud_sql_port}", "${data.google_sql_database_instance.spaghet_db.connection_name}=tcp:0.0.0.0:${var.spaghet_cloud_sql_port}", - "${data.google_sql_database_instance.spre_styringsinfo_db.connection_name}=tcp:0.0.0.0:${var.spre_styringsinfo_cloud_sql_port}" + "${data.google_sql_database_instance.spre_styringsinfo_db.connection_name}=tcp:0.0.0.0:${var.spre_styringsinfo_cloud_sql_port}", + "${data.google_sql_database_instance.annulleringer_db.connection_name}=tcp:0.0.0.0:${var.dataprodukt_annulleringer_cloud_sql_port}", ] } diff --git a/prod/secrets.tf b/prod/secrets.tf index 9fd0032..6244887 100644 --- a/prod/secrets.tf +++ b/prod/secrets.tf @@ -14,26 +14,29 @@ data "google_secret_manager_secret_version" "spre_styringsinfo_datastream_user_s secret = "spre_styringsinfo_datastream_user_secret" } +data "google_secret_manager_secret_version" "annulleringer_datastream_user_secret" { + secret = "annulleringer_datastream_user_secret" +} + +# Locals for å decode secrets fra JSON format locals { dataprodukt_arbeidsgiveropplysninger_db_credentials = jsondecode( data.google_secret_manager_secret_version.arbeidsgiveropplysninger_datastream_user_secret.secret_data ) -} -locals { dataprodukt_forstegangsbehandling_db_credentials = jsondecode( data.google_secret_manager_secret_version.forstegangsbehandling_datastream_user_secret.secret_data ) -} -locals { spaghet_db_credentials = jsondecode( data.google_secret_manager_secret_version.spaghet_datastream_user_secret.secret_data ) -} -locals { spre_styringsinfo_db_credentials = jsondecode( data.google_secret_manager_secret_version.spre_styringsinfo_datastream_user_secret.secret_data ) + + dataprodukt_annulleringer_db_credentials = jsondecode( + data.google_secret_manager_secret_version.annulleringer_datastream_user_secret.secret_data + ) } diff --git a/prod/variables.tf b/prod/variables.tf index 54ff5d4..734e2df 100644 --- a/prod/variables.tf +++ b/prod/variables.tf @@ -31,3 +31,9 @@ variable "spre_styringsinfo_cloud_sql_port" { type = string default = "5435" } + +variable "dataprodukt_annulleringer_cloud_sql_port" { + description = "The port exposed by the spre-styringsinfo database Cloud SQL instance." + type = string + default = "5436" +}