From 8e9fd99c5bcb80ffbe35d38943e19c73029aa258 Mon Sep 17 00:00:00 2001 From: YangWu1227 Date: Sun, 1 Dec 2024 01:39:06 -0500 Subject: [PATCH] feat: terraform --- .github/workflows/ecr_deployment.yml | 1 - .gitignore | 38 +++ .../stack_parameters.json | 34 +-- src/deploy_stack.py | 197 --------------- terraform/ecs_fargate/.terraform.lock.hcl | 25 ++ terraform/ecs_fargate/backend.hcl | 4 + terraform/ecs_fargate/ecs_fargate.tf | 80 +++++++ terraform/ecs_fargate/main.tf | 35 +++ terraform/ecs_fargate/outputs.tf | 14 ++ terraform/ecs_fargate/variables.tf | 59 +++++ terraform/iam/.terraform.lock.hcl | 25 ++ terraform/iam/backend.hcl | 4 + terraform/iam/iam.tf | 226 ++++++++++++++++++ terraform/iam/main.tf | 24 ++ terraform/iam/outputs.tf | 19 ++ terraform/iam/variables.tf | 39 +++ .../lambda_eventbridge/.terraform.lock.hcl | 25 ++ terraform/lambda_eventbridge/backend.hcl | 4 + .../lambda_eventbridge/lambda_eventbridge.tf | 36 +++ terraform/lambda_eventbridge/main.tf | 34 +++ terraform/lambda_eventbridge/variables.tf | 64 +++++ terraform/s3_ecr/.terraform.lock.hcl | 24 ++ terraform/s3_ecr/backend.hcl | 4 + terraform/s3_ecr/ecr.tf | 45 ++++ terraform/s3_ecr/main.tf | 14 ++ terraform/s3_ecr/outputs.tf | 19 ++ terraform/s3_ecr/s3.tf | 8 + terraform/s3_ecr/variables.tf | 24 ++ terraform/vpc_private/.terraform.lock.hcl | 25 ++ terraform/vpc_private/backend.hcl | 4 + terraform/vpc_private/main.tf | 14 ++ terraform/vpc_private/outputs.tf | 9 + terraform/vpc_private/variables.tf | 34 +++ terraform/vpc_private/vpc.tf | 121 ++++++++++ terraform/vpc_public/.terraform.lock.hcl | 25 ++ terraform/vpc_public/backend.hcl | 4 + terraform/vpc_public/main.tf | 14 ++ terraform/vpc_public/outputs.tf | 9 + terraform/vpc_public/variables.tf | 29 +++ terraform/vpc_public/vpc.tf | 66 +++++ 40 files changed, 1264 insertions(+), 215 deletions(-) delete mode 100644 src/deploy_stack.py create mode 100644 terraform/ecs_fargate/.terraform.lock.hcl create mode 100644 terraform/ecs_fargate/backend.hcl create mode 100644 terraform/ecs_fargate/ecs_fargate.tf create mode 100644 terraform/ecs_fargate/main.tf create mode 100644 terraform/ecs_fargate/outputs.tf create mode 100644 terraform/ecs_fargate/variables.tf create mode 100644 terraform/iam/.terraform.lock.hcl create mode 100644 terraform/iam/backend.hcl create mode 100644 terraform/iam/iam.tf create mode 100644 terraform/iam/main.tf create mode 100644 terraform/iam/outputs.tf create mode 100644 terraform/iam/variables.tf create mode 100644 terraform/lambda_eventbridge/.terraform.lock.hcl create mode 100644 terraform/lambda_eventbridge/backend.hcl create mode 100644 terraform/lambda_eventbridge/lambda_eventbridge.tf create mode 100644 terraform/lambda_eventbridge/main.tf create mode 100644 terraform/lambda_eventbridge/variables.tf create mode 100644 terraform/s3_ecr/.terraform.lock.hcl create mode 100644 terraform/s3_ecr/backend.hcl create mode 100644 terraform/s3_ecr/ecr.tf create mode 100644 terraform/s3_ecr/main.tf create mode 100644 terraform/s3_ecr/outputs.tf create mode 100644 terraform/s3_ecr/s3.tf create mode 100644 terraform/s3_ecr/variables.tf create mode 100644 terraform/vpc_private/.terraform.lock.hcl create mode 100644 terraform/vpc_private/backend.hcl create mode 100644 terraform/vpc_private/main.tf create mode 100644 terraform/vpc_private/outputs.tf create mode 100644 terraform/vpc_private/variables.tf create mode 100644 terraform/vpc_private/vpc.tf create mode 100644 terraform/vpc_public/.terraform.lock.hcl create mode 100644 terraform/vpc_public/backend.hcl create mode 100644 terraform/vpc_public/main.tf create mode 100644 terraform/vpc_public/outputs.tf create mode 100644 terraform/vpc_public/variables.tf create mode 100644 terraform/vpc_public/vpc.tf diff --git a/.github/workflows/ecr_deployment.yml b/.github/workflows/ecr_deployment.yml index 30053c8..6a7f3c4 100644 --- a/.github/workflows/ecr_deployment.yml +++ b/.github/workflows/ecr_deployment.yml @@ -7,7 +7,6 @@ on: paths: - Dockerfile - src/** - - '!src/deploy_stack.py' # Exclude the deploy_stack.py file from triggering the workflow - main.py - pyproject.toml - poetry.lock diff --git a/.gitignore b/.gitignore index d441ae2..5c7d37c 100644 --- a/.gitignore +++ b/.gitignore @@ -168,3 +168,41 @@ cython_debug/ yfinance.cache cloudformation_templates/stack_parameters_prod.json + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Ignore transient lock info files created by terraform apply +.terraform.tfstate.lock.info + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc diff --git a/cloudformation_templates/stack_parameters.json b/cloudformation_templates/stack_parameters.json index c050a4b..e84a6fd 100644 --- a/cloudformation_templates/stack_parameters.json +++ b/cloudformation_templates/stack_parameters.json @@ -1,35 +1,35 @@ { "vpc-private.yaml": { - "StackName": "etf-kpis-scraper-vpc" + "StackName": "etf_kpis_scraper_vpc" }, "s3-ecr.yaml": { - "StackName": "etf-kpis-scraper-s3-ecr", - "S3BucketName": "etf-kpis-scraper", - "ECRRepoName": "etf-kpis-scraper" + "StackName": "etf_kpis_scraper_s3_ecr", + "S3BucketName": "etf_kpis_scraper", + "ECRRepoName": "etf_kpis_scraper" }, "iam.yaml": { - "StackName": "etf-kpis-scraper-iam", - "S3BucketName": "etf-kpis-scraper", - "ECRRepoName": "etf-kpis-scraper", - "ECRRepoArn": "arn:aws:ecr:region:account_id:repository/etf-kpis-scraper", + "StackName": "etf_kpis_scraper_iam", + "S3BucketName": "etf_kpis_scraper", + "ECRRepoName": "etf_kpis_scraper", + "ECRRepoArn": "arn:aws:ecr:region:account_id:repository/etf_kpis_scraper", "GithubUsername": "github_username", "GithubRepoName": "github_repo_name" }, "lambda-eventbridge.yaml": { - "StackName": "etf-kpis-scraper-lambda-eventbridge", - "S3BucketName": "etf-kpis-scraper", - "EventBridgeScheduleExpression": "cron(00 22 ? * MON-FRI *)", - "LambdaExecutionRoleArn": "arn:aws:iam::region:account_id:role/etf-kpis-scraper-iam-lambda-execution-role", + "StackName": "etf_kpis_scraper_lambda_eventbridge", + "S3BucketName": "etf_kpis_scraper", + "EventBridgeScheduleExpression": "cron(00 22 ? * MON_FRI *)", + "LambdaExecutionRoleArn": "arn:aws:iam::region:account_id:role/etf_kpis_scraper_iam_lambda_execution_role", "Architectures": "x86_64", "Runtime": "python3.11", "Timeout": 30 }, "ecs-fargate.yaml": { - "StackName": "etf-kpis-scraper-ecs-fargate", - "ECRRepoName": "etf-kpis-scraper", - "ECSExecutionRoleArn": "arn:aws:iam::account_id:role/etf-kpis-scraper-iam-ecs-execution-role", - "ECSTaskRoleArn": "arn:aws:iam::account_id:role/etf-kpis-scraper-iam-ecs-task-role", - "EnvironmentFileS3Arn": "arn:aws:s3:::etf-kpis-scraper/vars.env", + "StackName": "etf_kpis_scraper_ecs_fargate", + "ECRRepoName": "etf_kpis_scraper", + "ECSExecutionRoleArn": "arn:aws:iam::account_id:role/etf_kpis_scraper_iam_ecs_execution_role", + "ECSTaskRoleArn": "arn:aws:iam::account_id:role/etf_kpis_scraper_iam_ecs_task_role", + "EnvironmentFileS3Arn": "arn:aws:s3:::etf_kpis_scraper/vars.env", "CpuArchitecture": "x86_64", "OperatingSystemFamily": "LINUX", "Cpu": 1024, diff --git a/src/deploy_stack.py b/src/deploy_stack.py deleted file mode 100644 index 51cfacf..0000000 --- a/src/deploy_stack.py +++ /dev/null @@ -1,197 +0,0 @@ -import argparse -import json -import os -from copy import deepcopy -from typing import Dict, Union - -import boto3 -from botocore.exceptions import ClientError, ParamValidationError -from mypy_boto3_cloudformation import CloudFormationClient - -from src.utils import setup_logger - - -def load_parameters( - template_file: str, parameters_file: str -) -> Dict[str, Union[str, int]]: - """ - Load the parameters for a CloudFormation stack from a JSON file. - - Parameters - ---------- - template_file : str - The file path of the CloudFormation template YAML - parameters_file : str - The file path of the stack parameters JSON file - - Returns - ------- - Dict[str, Union[str, int]] - The parameters for the CloudFormation stack as a dictionary - - Raises - ------ - ValueError - If the template key is not found in the parameters file - """ - with open(parameters_file, "r") as file: - parameters = json.load(file) - # This assumes that the template file 'xxx.yaml' is in the parameters json file - template_key = os.path.basename(template_file) - if template_key not in parameters: - raise ValueError(f"No parameters found for template: {template_key}") - return parameters[template_key] - - -def create_stack( - client: CloudFormationClient, - template_file: str, - parameters: Dict[str, Union[str, int]], -) -> Dict[str, str]: - """ - Create a CloudFormation stack based on a template and parameters. - - Parameters - ---------- - client : CloudFormationClient - The CloudFormation client - template_file : str - The file path of the CloudFormation template YAML - parameters : Dict[str, Union[str, int]] - The parameters for the CloudFormation stack - - Returns - ------- - Dict[str, str] - The response from the `create_stack` API call, which is just `{'StackId': 'string'}` - - Raises - ------ - ClientError - If the create_stack API call fails - ValueError - If the create_stack API call returns an invalid parameter error - """ - with open(template_file, "r") as file: - template_body = file.read() - - # Create a copy of the parameters dictionary and pop the StackName key - parameters = deepcopy(parameters) - stack_name = parameters.pop("StackName") - stack_parameters = [ - {"ParameterKey": param_key, "ParameterValue": str(param_value)} - for param_key, param_value in parameters.items() - ] - - try: - response = client.create_stack( - StackName=stack_name, - TemplateBody=template_body, - Parameters=stack_parameters, - # These are needed only for the IAM template, but they don't hurt for other templates - Capabilities=["CAPABILITY_IAM", "CAPABILITY_NAMED_IAM"], - ) - return response - except ClientError as client_error: - raise client_error - except ParamValidationError as param_error: - raise ValueError(f"Invalid parameter to the create_stack API: {param_error}") - - -def get_stack_outputs(client: CloudFormationClient, stack_name: str) -> Dict[str, str]: - """ - Get the key-value exported outputs from a CloudFormation stack just created. - - Parameters - ---------- - client : CloudFormationClient - The CloudFormation client - stack_name : str - The name of the CloudFormation stack - - Returns - ------- - Dict[str, str] - The outputs of the CloudFormation stack as a dictionary - - Raises - ------ - ClientError - If the describe_stacks API call fails - ValueError - If the describe_stacks API call returns an invalid parameter error - """ - try: - response = client.describe_stacks(StackName=stack_name) - # The response structure: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudformation/client/describe_stacks.html - stack = response["Stacks"][0] - outputs = { - output["OutputKey"]: output["OutputValue"] - for output in stack.get("Outputs", []) - } - return outputs - except ClientError as client_error: - raise client_error - except ParamValidationError as param_error: - raise ValueError(f"Invalid parameter to the describe_stacks API: {param_error}") - - -def main() -> int: - logger = setup_logger(name="Deploy Stack") - parser = argparse.ArgumentParser( - description="Create a CloudFormation stack and optionally save outputs to a JSON file" - ) - parser.add_argument( - "--template_file", help="The file path of the CloudFormation template YAML" - ) - parser.add_argument( - "--parameters_file", help="The file path of the stack parameters JSON file" - ) - parser.add_argument( - "--save_outputs", - action="store_true", - help="Whether to save the outputs as a JSON file", - ) - args, _ = parser.parse_known_args() - - logger.info( - f"Deploying stack with template file: {args.template_file} and parameters file: {args.parameters_file}" - ) - parameters = load_parameters(args.template_file, args.parameters_file) - - logger.info(f"Creating stack with parameters: {parameters}") - client: CloudFormationClient = boto3.client("cloudformation") - create_stack(client, args.template_file, parameters) - stack_name = str(parameters["StackName"]) - print(f"Stack creation initiated: {stack_name}") - # Wait for stack creation to complete - waiter = client.get_waiter("stack_create_complete") - try: - waiter.wait(StackName=stack_name) - except ClientError as client_error: - logger.error(f"Error while waiting for stack creation: {client_error}") - return 1 - - logger.info(f"Getting outputs for stack: {stack_name}") - outputs = get_stack_outputs(client, stack_name) - if outputs is None: - logger.info("No outputs found for the stack") - return 0 - - if args.save_outputs: - # Create a template output directory if it doesn't exist in the current working directory - output_dir = os.path.join(os.getcwd(), "stack_outputs") - os.makedirs(output_dir, exist_ok=True) - output_file = os.path.join(output_dir, f"{stack_name}_outputs.json") - with open(output_file, "w") as file: - json.dump(outputs, file, indent=4) - logger.info(f"Outputs saved to file: {output_file}") - else: - logger.info("Outputs:") - logger.info(json.dumps(outputs, indent=4)) - - return 0 - - -if __name__ == "__main__": - main() diff --git a/terraform/ecs_fargate/.terraform.lock.hcl b/terraform/ecs_fargate/.terraform.lock.hcl new file mode 100644 index 0000000..3c6e612 --- /dev/null +++ b/terraform/ecs_fargate/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.78.0" + constraints = "~> 5.0" + hashes = [ + "h1:o7jz+dFixEcwjfdubken5ldmDJm1tkvM2adPtNDei3g=", + "zh:0ae7d41b96441d0cf7ce2e1337657bdb2e1e5c9f1c2227b0642e1dcec2f9dfba", + "zh:21f8f1edf477681ea3b095c02cad6b8e85262e45015de58e84e0c7b2bfe9a1f6", + "zh:2bdc335e341bf98445255549ae93d66cfb9bca706e62b949da98fe467c182cad", + "zh:2fe4096e260367a225a9faf4a424d62b87e5498f12cb43bdb6f4e713d11b82c3", + "zh:3c63bb7a7925d65118d17461f4691a22dbb55ea39a7404e4d71f6ccca8765f8b", + "zh:6609a28a1c638a1901d8007b5386868ccfd313b4df2e98b35d9fdef436974e3b", + "zh:7ae3aef43bc4b365824cca4659cf92459d766800656e354bdbf83feabab835e8", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:c314efe454adc6ca483261c6906e64315aeb9db0c0332818714e9b81e07df0f0", + "zh:cd3e30396b554bbc1d260252db8a0f344065d619038fe60ea870689cd32c6aa9", + "zh:d1ba48fd9d8a1cb1daa927fb9e8bb708b857f2792d796e110460c6fdcd896a47", + "zh:d31c8abe75cb9cdc1c59ad9d356a1c3ae1ba8cd29ac15eb7e01b6cd01221ab04", + "zh:dc27c5c2116b4d9b404753f73bccaa635bce21f3bfb4bb7bc8e63225c36c98fe", + "zh:de491f0d05408378413187475c815d8cb2ac6bfa63d0b42a30ad5ee492e51c07", + "zh:eb44b45a40f80a309dd5b0eb7d7fcb2cbfe588fe2f18b173ef5851346898a662", + ] +} diff --git a/terraform/ecs_fargate/backend.hcl b/terraform/ecs_fargate/backend.hcl new file mode 100644 index 0000000..3f6f59a --- /dev/null +++ b/terraform/ecs_fargate/backend.hcl @@ -0,0 +1,4 @@ +bucket = "yang-templates" +key = "terraform-states/etf-kpis-scraper/ecs-fargate/terraform.tfstate" +region = "us-east-1" +profile = "admin" diff --git a/terraform/ecs_fargate/ecs_fargate.tf b/terraform/ecs_fargate/ecs_fargate.tf new file mode 100644 index 0000000..b743ce1 --- /dev/null +++ b/terraform/ecs_fargate/ecs_fargate.tf @@ -0,0 +1,80 @@ +# ECS Cluster +resource "aws_ecs_cluster" "ecs_cluster" { + name = "${var.stack_name}_ecs_cluster" + + # Enable container insights for the ECS cluster + setting { + name = "containerInsights" + value = "enabled" + } + + tags = { + project = var.stack_name + } +} + +# ECS Cluster Capacity Providers +resource "aws_ecs_cluster_capacity_providers" "ecs_cluster_capacity_providers" { + cluster_name = aws_ecs_cluster.ecs_cluster.name + capacity_providers = ["FARGATE", "FARGATE_SPOT"] + + # Split evenly between FARGATE and FARGATE_SPOT + default_capacity_provider_strategy { + weight = 1 + capacity_provider = "FARGATE" + } + + default_capacity_provider_strategy { + weight = 1 + capacity_provider = "FARGATE_SPOT" + } +} + +# ECS Task Definition +resource "aws_ecs_task_definition" "ecs_task_definition" { + family = "${var.stack_name}_task_definition" + execution_role_arn = data.terraform_remote_state.iam.outputs.ecs_execution_role_arn + task_role_arn = data.terraform_remote_state.iam.outputs.ecs_task_role_arn + network_mode = "awsvpc" + cpu = var.cpu + memory = var.memory + requires_compatibilities = ["FARGATE"] + + # Runtime platform + runtime_platform { + cpu_architecture = var.cpu_architecture + operating_system_family = var.operating_system_family + } + + # Ephemeral storage + ephemeral_storage { + size_in_gib = var.size_in_gib + } + + # Container definitions + container_definitions = jsonencode([ + { + name = "${var.stack_name}_container" + image = "${data.terraform_remote_state.s3_ecr.outputs.ecr_repo_name}:latest" + essential = true + environmentFiles = [ + { + type = "s3" + value = var.environment_file_s3_arn + } + ] + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = "/aws/ecs/${var.stack_name}" + awslogs-region = var.region + awslogs-stream-prefix = "ecs" + } + } + } + ]) + + tags = { + project = var.stack_name + } +} diff --git a/terraform/ecs_fargate/main.tf b/terraform/ecs_fargate/main.tf new file mode 100644 index 0000000..d1131e6 --- /dev/null +++ b/terraform/ecs_fargate/main.tf @@ -0,0 +1,35 @@ +terraform { + backend "s3" { + } + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.region + profile = var.profile +} + +data "terraform_remote_state" "iam" { + backend = "s3" + config = { + bucket = var.terraform_state_bucket + key = var.iam_terraform_state_key + region = var.region + profile = var.profile + } +} + +data "terraform_remote_state" "s3_ecr" { + backend = "s3" + config = { + bucket = var.terraform_state_bucket + key = var.s3_ecr_terraform_state_key + region = var.region + profile = var.profile + } +} diff --git a/terraform/ecs_fargate/outputs.tf b/terraform/ecs_fargate/outputs.tf new file mode 100644 index 0000000..2e64def --- /dev/null +++ b/terraform/ecs_fargate/outputs.tf @@ -0,0 +1,14 @@ +output "ecs_fargate_cluster_name" { + description = "The name of the ECS Fargate cluster" + value = aws_ecs_cluster.ecs_cluster.name +} + +output "ecs_fargate_task_definition_family" { + description = "The name of the ECS Fargate task definition family" + value = aws_ecs_task_definition.ecs_task_definition.family +} + +output "ecs_fargate_container_name" { + description = "The name of the ECS Fargate container" + value = "${var.stack_name}_container" +} diff --git a/terraform/ecs_fargate/variables.tf b/terraform/ecs_fargate/variables.tf new file mode 100644 index 0000000..e87cd82 --- /dev/null +++ b/terraform/ecs_fargate/variables.tf @@ -0,0 +1,59 @@ +variable "region" { + description = "The AWS region to deploy resources in" + type = string +} + +variable "profile" { + description = "The AWS credentials profile to use for deployment" + type = string +} + +variable "stack_name" { + description = "The name of the stack, used for tagging and resource identification" + type = string +} + +variable "cpu_architecture" { + description = "The CPU architecture of the task (e.g., X86_64)" + type = string +} + +variable "operating_system_family" { + description = "The operating system family of the task (e.g., LINUX)" + type = string +} + +variable "cpu" { + description = "The number of CPU units to reserve for the container" + type = number +} + +variable "memory" { + description = "The amount of memory (in MiB) to reserve for the container" + type = number +} + +variable "size_in_gib" { + description = "The amount of ephemeral storage (in GiB) to reserve for the container" + type = number +} + +variable "environment_file_s3_arn" { + description = "The S3 ARN of the environment file for the container" + type = string +} + +variable "terraform_state_bucket" { + description = "S3 bucket for IAM Terraform state files" + type = string +} + +variable "iam_terraform_state_key" { + description = "S3 key for IAM Terraform state" + type = string +} + +variable "s3_ecr_terraform_state_key" { + description = "S3 key for S3 and ECR Terraform state" + type = string +} diff --git a/terraform/iam/.terraform.lock.hcl b/terraform/iam/.terraform.lock.hcl new file mode 100644 index 0000000..3c6e612 --- /dev/null +++ b/terraform/iam/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.78.0" + constraints = "~> 5.0" + hashes = [ + "h1:o7jz+dFixEcwjfdubken5ldmDJm1tkvM2adPtNDei3g=", + "zh:0ae7d41b96441d0cf7ce2e1337657bdb2e1e5c9f1c2227b0642e1dcec2f9dfba", + "zh:21f8f1edf477681ea3b095c02cad6b8e85262e45015de58e84e0c7b2bfe9a1f6", + "zh:2bdc335e341bf98445255549ae93d66cfb9bca706e62b949da98fe467c182cad", + "zh:2fe4096e260367a225a9faf4a424d62b87e5498f12cb43bdb6f4e713d11b82c3", + "zh:3c63bb7a7925d65118d17461f4691a22dbb55ea39a7404e4d71f6ccca8765f8b", + "zh:6609a28a1c638a1901d8007b5386868ccfd313b4df2e98b35d9fdef436974e3b", + "zh:7ae3aef43bc4b365824cca4659cf92459d766800656e354bdbf83feabab835e8", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:c314efe454adc6ca483261c6906e64315aeb9db0c0332818714e9b81e07df0f0", + "zh:cd3e30396b554bbc1d260252db8a0f344065d619038fe60ea870689cd32c6aa9", + "zh:d1ba48fd9d8a1cb1daa927fb9e8bb708b857f2792d796e110460c6fdcd896a47", + "zh:d31c8abe75cb9cdc1c59ad9d356a1c3ae1ba8cd29ac15eb7e01b6cd01221ab04", + "zh:dc27c5c2116b4d9b404753f73bccaa635bce21f3bfb4bb7bc8e63225c36c98fe", + "zh:de491f0d05408378413187475c815d8cb2ac6bfa63d0b42a30ad5ee492e51c07", + "zh:eb44b45a40f80a309dd5b0eb7d7fcb2cbfe588fe2f18b173ef5851346898a662", + ] +} diff --git a/terraform/iam/backend.hcl b/terraform/iam/backend.hcl new file mode 100644 index 0000000..f23347b --- /dev/null +++ b/terraform/iam/backend.hcl @@ -0,0 +1,4 @@ +bucket = "yang-templates" +key = "terraform-states/etf-kpis-scraper/iam/terraform.tfstate" +region = "us-east-1" +profile = "admin" diff --git a/terraform/iam/iam.tf b/terraform/iam/iam.tf new file mode 100644 index 0000000..856ca8a --- /dev/null +++ b/terraform/iam/iam.tf @@ -0,0 +1,226 @@ +# Lambda Execution Role +resource "aws_iam_role" "lambda_execution_role" { + name = "${var.stack_name}_lambda_execution_role" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { Service = "lambda.amazonaws.com" } + Action = "sts:AssumeRole" + } + ] + }) + + tags = { + project = var.stack_name + } +} + +resource "aws_iam_policy" "lambda_policy" { + name = "${var.stack_name}_lambda_policy" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = "arn:aws:logs:${var.region}:${var.account_id}:log-group:/aws/lambda/*" + }, + { + Effect = "Allow" + Action = [ + "ecs:RunTask", + "ecs:DescribeTaskDefinition" + ] + Resource = "*" + }, + { + Effect = "Allow" + Action = ["iam:PassRole"] + Resource = [ + "arn:aws:iam::${var.account_id}:role/${var.stack_name}_ecs_execution_role", + "arn:aws:iam::${var.account_id}:role/${var.stack_name}_ecs_task_role" + ] + } + ] + }) +} + +resource "aws_iam_policy_attachment" "lambda_policy_attachment" { + name = "${var.stack_name}_lambda_policy_attachment" + roles = [aws_iam_role.lambda_execution_role.name] + policy_arn = aws_iam_policy.lambda_policy.arn +} + +# ECS Execution Role +resource "aws_iam_role" "ecs_execution_role" { + name = "${var.stack_name}_ecs_execution_role" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { Service = "ecs-tasks.amazonaws.com" } + Action = "sts:AssumeRole" + } + ] + }) + + tags = { + project = var.stack_name + } +} + +resource "aws_iam_policy" "ecs_execution_policy" { + name = "${var.stack_name}_ecs_execution_policy" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Resource = [ + data.terraform_remote_state.s3_ecr.outputs.ecr_repo_arn, + "arn:aws:logs:${var.region}:${var.account_id}:log-group:/aws/ecs/*" + ] + }, + { + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = "${data.terraform_remote_state.s3_ecr.outputs.s3_bucket_arn}/*" + }, + { + Effect = "Allow" + Action = ["ecr:GetAuthorizationToken"] + Resource = "*" + } + ] + }) +} + +resource "aws_iam_policy_attachment" "ecs_execution_policy_attachment" { + name = "${var.stack_name}_ecs_execution_policy_attachment" + roles = [aws_iam_role.ecs_execution_role.name] + policy_arn = aws_iam_policy.ecs_execution_policy.arn +} + +# ECS Task Role +resource "aws_iam_role" "ecs_task_role" { + name = "${var.stack_name}_ecs_task_role" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { Service = "ecs-tasks.amazonaws.com" } + Action = "sts:AssumeRole" + } + ] + }) + + tags = { + project = var.stack_name + } +} + +resource "aws_iam_policy" "ecs_task_policy" { + name = "${var.stack_name}_ecs_task_policy" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["s3:PutObject", "s3:GetObject"] + Resource = "${data.terraform_remote_state.s3_ecr.outputs.s3_bucket_arn}/*" + } + ] + }) +} + +resource "aws_iam_policy_attachment" "ecs_task_policy_attachment" { + name = "${var.stack_name}_ecs_task_policy_attachment" + roles = [aws_iam_role.ecs_task_role.name] + policy_arn = aws_iam_policy.ecs_task_policy.arn +} + +# GitHub Actions Role +resource "aws_iam_role" "github_actions_role" { + name = "${var.stack_name}_github_actions_role" + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { Federated = "arn:aws:iam::${var.account_id}:oidc-provider/token.actions.githubusercontent.com" } + Action = "sts:AssumeRoleWithWebIdentity" + Condition = { + StringEquals = { + "token.actions.githubusercontent.com:aud" = "sts.amazonaws.com" + "token.actions.githubusercontent.com:sub" = "repo:${var.github_username}/${var.github_repo_name}:ref:refs/heads/main" + } + } + } + ] + }) + + tags = { + project = var.stack_name + } +} + +resource "aws_iam_policy" "github_actions_policy" { + name = "${var.stack_name}_github_actions_policy" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "lambda:UpdateFunctionCode", + "lambda:GetFunction" + ] + Resource = "arn:aws:lambda:${var.region}:${var.account_id}:function:*" + }, + { + Effect = "Allow" + Action = [ + "ecr:BatchCheckLayerAvailability", + "ecr:CompleteLayerUpload", + "ecr:GetDownloadUrlForLayer", + "ecr:InitiateLayerUpload", + "ecr:PutImage", + "ecr:UploadLayerPart" + ] + Resource = data.terraform_remote_state.s3_ecr.outputs.ecr_repo_arn + }, + { + Effect = "Allow" + Action = ["ecr:GetAuthorizationToken"] + Resource = "*" + }, + { + Effect = "Allow" + Action = ["s3:PutObject", "s3:GetObject"] + Resource = "${data.terraform_remote_state.s3_ecr.outputs.s3_bucket_arn}/*" + } + ] + }) +} + +resource "aws_iam_policy_attachment" "github_actions_policy_attachment" { + name = "${var.stack_name}_github_actions_policy_attachment" + roles = [aws_iam_role.github_actions_role.name] + policy_arn = aws_iam_policy.github_actions_policy.arn +} diff --git a/terraform/iam/main.tf b/terraform/iam/main.tf new file mode 100644 index 0000000..cc44f18 --- /dev/null +++ b/terraform/iam/main.tf @@ -0,0 +1,24 @@ +terraform { + backend "s3" {} + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.region + profile = var.profile +} + +data "terraform_remote_state" "s3_ecr" { + backend = "s3" + config = { + bucket = var.s3_ecr_terraform_state_bucket + key = var.s3_ecr_terraform_state_key + region = var.region + profile = var.profile + } +} diff --git a/terraform/iam/outputs.tf b/terraform/iam/outputs.tf new file mode 100644 index 0000000..2d43336 --- /dev/null +++ b/terraform/iam/outputs.tf @@ -0,0 +1,19 @@ +output "lambda_execution_role_arn" { + description = "ARN of the Lambda execution role" + value = aws_iam_role.lambda_execution_role.arn +} + +output "ecs_execution_role_arn" { + description = "ARN of the ECS execution role" + value = aws_iam_role.ecs_execution_role.arn +} + +output "ecs_task_role_arn" { + description = "ARN of the ECS task role" + value = aws_iam_role.ecs_task_role.arn +} + +output "github_actions_role_arn" { + description = "ARN of the GitHub Actions role" + value = aws_iam_role.github_actions_role.arn +} diff --git a/terraform/iam/variables.tf b/terraform/iam/variables.tf new file mode 100644 index 0000000..a969586 --- /dev/null +++ b/terraform/iam/variables.tf @@ -0,0 +1,39 @@ +variable "region" { + description = "The AWS region to deploy resources in" + type = string +} + +variable "account_id" { + description = "The AWS account ID" + type = string +} + +variable "profile" { + description = "The AWS credentials profile to use for deployment" + type = string +} + +variable "stack_name" { + description = "The name of the stack, used for tagging and resource identification" + type = string +} + +variable "s3_ecr_terraform_state_bucket" { + description = "The S3 key of the Terraform state file containing outputs from the ecr and S3 deployments" + type = string +} + +variable "s3_ecr_terraform_state_key" { + description = "The S3 key of the Terraform state file containing outputs from the ecr and S3 deployments" + type = string +} + +variable "github_username" { + description = "GitHub username" + type = string +} + +variable "github_repo_name" { + description = "GitHub repository name" + type = string +} diff --git a/terraform/lambda_eventbridge/.terraform.lock.hcl b/terraform/lambda_eventbridge/.terraform.lock.hcl new file mode 100644 index 0000000..3c6e612 --- /dev/null +++ b/terraform/lambda_eventbridge/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.78.0" + constraints = "~> 5.0" + hashes = [ + "h1:o7jz+dFixEcwjfdubken5ldmDJm1tkvM2adPtNDei3g=", + "zh:0ae7d41b96441d0cf7ce2e1337657bdb2e1e5c9f1c2227b0642e1dcec2f9dfba", + "zh:21f8f1edf477681ea3b095c02cad6b8e85262e45015de58e84e0c7b2bfe9a1f6", + "zh:2bdc335e341bf98445255549ae93d66cfb9bca706e62b949da98fe467c182cad", + "zh:2fe4096e260367a225a9faf4a424d62b87e5498f12cb43bdb6f4e713d11b82c3", + "zh:3c63bb7a7925d65118d17461f4691a22dbb55ea39a7404e4d71f6ccca8765f8b", + "zh:6609a28a1c638a1901d8007b5386868ccfd313b4df2e98b35d9fdef436974e3b", + "zh:7ae3aef43bc4b365824cca4659cf92459d766800656e354bdbf83feabab835e8", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:c314efe454adc6ca483261c6906e64315aeb9db0c0332818714e9b81e07df0f0", + "zh:cd3e30396b554bbc1d260252db8a0f344065d619038fe60ea870689cd32c6aa9", + "zh:d1ba48fd9d8a1cb1daa927fb9e8bb708b857f2792d796e110460c6fdcd896a47", + "zh:d31c8abe75cb9cdc1c59ad9d356a1c3ae1ba8cd29ac15eb7e01b6cd01221ab04", + "zh:dc27c5c2116b4d9b404753f73bccaa635bce21f3bfb4bb7bc8e63225c36c98fe", + "zh:de491f0d05408378413187475c815d8cb2ac6bfa63d0b42a30ad5ee492e51c07", + "zh:eb44b45a40f80a309dd5b0eb7d7fcb2cbfe588fe2f18b173ef5851346898a662", + ] +} diff --git a/terraform/lambda_eventbridge/backend.hcl b/terraform/lambda_eventbridge/backend.hcl new file mode 100644 index 0000000..ad766c5 --- /dev/null +++ b/terraform/lambda_eventbridge/backend.hcl @@ -0,0 +1,4 @@ +bucket = "yang-templates" +key = "terraform-states/etf-kpis-scraper/lambda-eventbridge/terraform.tfstate" +region = "us-east-1" +profile = "admin" diff --git a/terraform/lambda_eventbridge/lambda_eventbridge.tf b/terraform/lambda_eventbridge/lambda_eventbridge.tf new file mode 100644 index 0000000..337c148 --- /dev/null +++ b/terraform/lambda_eventbridge/lambda_eventbridge.tf @@ -0,0 +1,36 @@ +resource "aws_lambda_function" "lambda_function" { + function_name = "${var.stack_name}_lambda_function" + description = "Lambda function triggered by EventBridge to run ECS Fargate task" + runtime = var.lambda_runtime + role = data.terraform_remote_state.iam.outputs.lambda_execution_role_arn + handler = var.lambda_handler + architectures = [var.lambda_architecture] + timeout = var.lambda_timeout + + s3_bucket = var.lambda_code_s3_bucket + s3_key = var.lambda_code_s3_key + + tags = { + project = var.stack_name + } +} + +resource "aws_cloudwatch_event_rule" "eventbridge_rule" { + name = "${var.stack_name}_eventbridge_rule" + description = "EventBridge rule triggering the Lambda function on a schedule" + schedule_expression = var.schedule_expression +} + +resource "aws_cloudwatch_event_target" "lambda_target" { + rule = aws_cloudwatch_event_rule.eventbridge_rule.name + target_id = "${var.stack_name}_lambda_function_target" + arn = aws_lambda_function.lambda_function.arn +} + +resource "aws_lambda_permission" "eventbridge_permission" { + statement_id = "AllowEventBridgeInvoke" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.lambda_function.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.eventbridge_rule.arn +} diff --git a/terraform/lambda_eventbridge/main.tf b/terraform/lambda_eventbridge/main.tf new file mode 100644 index 0000000..606a7b7 --- /dev/null +++ b/terraform/lambda_eventbridge/main.tf @@ -0,0 +1,34 @@ +terraform { + backend "s3" {} + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.region + profile = var.profile +} + +data "terraform_remote_state" "iam" { + backend = "s3" + config = { + bucket = var.terraform_state_bucket + key = var.iam_terraform_state_key + region = var.region + profile = var.profile + } +} + +data "terraform_remote_state" "s3_ecr" { + backend = "s3" + config = { + bucket = var.terraform_state_bucket + key = var.s3_ecr_terraform_state_key + region = var.region + profile = var.profile + } +} diff --git a/terraform/lambda_eventbridge/variables.tf b/terraform/lambda_eventbridge/variables.tf new file mode 100644 index 0000000..3587e31 --- /dev/null +++ b/terraform/lambda_eventbridge/variables.tf @@ -0,0 +1,64 @@ +variable "region" { + description = "The AWS region to deploy resources in" + type = string +} + +variable "profile" { + description = "The AWS credentials profile to use for deployment" + type = string +} + +variable "stack_name" { + description = "The name of the stack, used for tagging and resource identification" + type = string +} + +variable "lambda_runtime" { + description = "Runtime for the Lambda function (e.g., python3.11)" + type = string +} + +variable "lambda_handler" { + description = "The entry point for the Lambda function" + type = string +} + +variable "lambda_architecture" { + description = "Lambda architecture (e.g., x86_64, arm64)" + type = string +} + +variable "lambda_timeout" { + description = "Timeout for the Lambda function in seconds" + type = number +} + +variable "lambda_code_s3_bucket" { + description = "Name of the S3 bucket for where the zippped Lambda function code is stored" + type = string +} + +variable "lambda_code_s3_key" { + description = "S3 key for the zippped Lambda function code" + type = string +} + +variable "schedule_expression" { + description = "Schedule expression for the EventBridge rule" + type = string +} + +variable "terraform_state_bucket" { + description = "S3 bucket for IAM Terraform state files" + type = string +} + +variable "iam_terraform_state_key" { + description = "S3 key for IAM Terraform state" + type = string +} + +variable "s3_ecr_terraform_state_key" { + description = "S3 key for S3 and ECR Terraform state" + type = string +} diff --git a/terraform/s3_ecr/.terraform.lock.hcl b/terraform/s3_ecr/.terraform.lock.hcl new file mode 100644 index 0000000..6eafa90 --- /dev/null +++ b/terraform/s3_ecr/.terraform.lock.hcl @@ -0,0 +1,24 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.78.0" + hashes = [ + "h1:o7jz+dFixEcwjfdubken5ldmDJm1tkvM2adPtNDei3g=", + "zh:0ae7d41b96441d0cf7ce2e1337657bdb2e1e5c9f1c2227b0642e1dcec2f9dfba", + "zh:21f8f1edf477681ea3b095c02cad6b8e85262e45015de58e84e0c7b2bfe9a1f6", + "zh:2bdc335e341bf98445255549ae93d66cfb9bca706e62b949da98fe467c182cad", + "zh:2fe4096e260367a225a9faf4a424d62b87e5498f12cb43bdb6f4e713d11b82c3", + "zh:3c63bb7a7925d65118d17461f4691a22dbb55ea39a7404e4d71f6ccca8765f8b", + "zh:6609a28a1c638a1901d8007b5386868ccfd313b4df2e98b35d9fdef436974e3b", + "zh:7ae3aef43bc4b365824cca4659cf92459d766800656e354bdbf83feabab835e8", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:c314efe454adc6ca483261c6906e64315aeb9db0c0332818714e9b81e07df0f0", + "zh:cd3e30396b554bbc1d260252db8a0f344065d619038fe60ea870689cd32c6aa9", + "zh:d1ba48fd9d8a1cb1daa927fb9e8bb708b857f2792d796e110460c6fdcd896a47", + "zh:d31c8abe75cb9cdc1c59ad9d356a1c3ae1ba8cd29ac15eb7e01b6cd01221ab04", + "zh:dc27c5c2116b4d9b404753f73bccaa635bce21f3bfb4bb7bc8e63225c36c98fe", + "zh:de491f0d05408378413187475c815d8cb2ac6bfa63d0b42a30ad5ee492e51c07", + "zh:eb44b45a40f80a309dd5b0eb7d7fcb2cbfe588fe2f18b173ef5851346898a662", + ] +} diff --git a/terraform/s3_ecr/backend.hcl b/terraform/s3_ecr/backend.hcl new file mode 100644 index 0000000..e8f8c01 --- /dev/null +++ b/terraform/s3_ecr/backend.hcl @@ -0,0 +1,4 @@ +bucket = "yang-templates" +key = "terraform-states/etf-kpis-scraper/s3-ecr/terraform.tfstate" +region = "us-east-1" +profile = "admin" diff --git a/terraform/s3_ecr/ecr.tf b/terraform/s3_ecr/ecr.tf new file mode 100644 index 0000000..bd85879 --- /dev/null +++ b/terraform/s3_ecr/ecr.tf @@ -0,0 +1,45 @@ +resource "aws_ecr_repository" "ecr_repo" { + name = replace(var.stack_name, "_", "-") + image_tag_mutability = "MUTABLE" + image_scanning_configuration { + scan_on_push = true + } + + tags = { + project = "${var.stack_name}" + } +} + +resource "aws_ecr_lifecycle_policy" "repository_cleanup_policy" { + repository = aws_ecr_repository.ecr_repo.name + policy = jsonencode({ + rules = [ + { + rulePriority = 1 + description = "Retain only the latest 'n' images with specific tags pattern" + selection = { + tagStatus = "tagged" + tagPrefixList = ["latest"] + countType = "imageCountMoreThan" + countNumber = var.retained_image_count + } + action = { + type = "expire" + } + }, + { + rulePriority = 2 + description = "Expire untagged images older than a specified number of days" + selection = { + tagStatus = "untagged" + countType = "sinceImagePushed" + countUnit = "days" + countNumber = var.untagged_image_expiry_days + } + action = { + type = "expire" + } + } + ] + }) +} diff --git a/terraform/s3_ecr/main.tf b/terraform/s3_ecr/main.tf new file mode 100644 index 0000000..9ea19e8 --- /dev/null +++ b/terraform/s3_ecr/main.tf @@ -0,0 +1,14 @@ +terraform { + backend "s3" {} + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.region + profile = var.profile +} diff --git a/terraform/s3_ecr/outputs.tf b/terraform/s3_ecr/outputs.tf new file mode 100644 index 0000000..b6c8347 --- /dev/null +++ b/terraform/s3_ecr/outputs.tf @@ -0,0 +1,19 @@ +output "s3_bucket_name" { + description = "Name of the S3 bucket" + value = aws_s3_bucket.s3_bucket.id +} + +output "s3_bucket_arn" { + description = "ARN of the S3 bucket" + value = aws_s3_bucket.s3_bucket.arn +} + +output "ecr_repo_name" { + description = "Name of the ECR repository" + value = aws_ecr_repository.ecr_repo.name +} + +output "ecr_repo_arn" { + description = "ARN of the ECR repository" + value = aws_ecr_repository.ecr_repo.arn +} diff --git a/terraform/s3_ecr/s3.tf b/terraform/s3_ecr/s3.tf new file mode 100644 index 0000000..cffe7cb --- /dev/null +++ b/terraform/s3_ecr/s3.tf @@ -0,0 +1,8 @@ +resource "aws_s3_bucket" "s3_bucket" { + bucket = replace(var.stack_name, "_", "-") + force_destroy = true + + tags = { + project = "${var.stack_name}" + } +} diff --git a/terraform/s3_ecr/variables.tf b/terraform/s3_ecr/variables.tf new file mode 100644 index 0000000..1dc9cc5 --- /dev/null +++ b/terraform/s3_ecr/variables.tf @@ -0,0 +1,24 @@ +variable "region" { + description = "The AWS region to deploy resources in" + type = string +} + +variable "profile" { + description = "The AWS credentials profile to use for deployment" + type = string +} + +variable "stack_name" { + description = "The name of the stack, used for tagging and resource identification" + type = string +} + +variable "retained_image_count" { + description = "Number of images to retain with a specific tag" + type = number +} + +variable "untagged_image_expiry_days" { + description = "Number of days after which untagged images will expire" + type = number +} diff --git a/terraform/vpc_private/.terraform.lock.hcl b/terraform/vpc_private/.terraform.lock.hcl new file mode 100644 index 0000000..3c6e612 --- /dev/null +++ b/terraform/vpc_private/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.78.0" + constraints = "~> 5.0" + hashes = [ + "h1:o7jz+dFixEcwjfdubken5ldmDJm1tkvM2adPtNDei3g=", + "zh:0ae7d41b96441d0cf7ce2e1337657bdb2e1e5c9f1c2227b0642e1dcec2f9dfba", + "zh:21f8f1edf477681ea3b095c02cad6b8e85262e45015de58e84e0c7b2bfe9a1f6", + "zh:2bdc335e341bf98445255549ae93d66cfb9bca706e62b949da98fe467c182cad", + "zh:2fe4096e260367a225a9faf4a424d62b87e5498f12cb43bdb6f4e713d11b82c3", + "zh:3c63bb7a7925d65118d17461f4691a22dbb55ea39a7404e4d71f6ccca8765f8b", + "zh:6609a28a1c638a1901d8007b5386868ccfd313b4df2e98b35d9fdef436974e3b", + "zh:7ae3aef43bc4b365824cca4659cf92459d766800656e354bdbf83feabab835e8", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:c314efe454adc6ca483261c6906e64315aeb9db0c0332818714e9b81e07df0f0", + "zh:cd3e30396b554bbc1d260252db8a0f344065d619038fe60ea870689cd32c6aa9", + "zh:d1ba48fd9d8a1cb1daa927fb9e8bb708b857f2792d796e110460c6fdcd896a47", + "zh:d31c8abe75cb9cdc1c59ad9d356a1c3ae1ba8cd29ac15eb7e01b6cd01221ab04", + "zh:dc27c5c2116b4d9b404753f73bccaa635bce21f3bfb4bb7bc8e63225c36c98fe", + "zh:de491f0d05408378413187475c815d8cb2ac6bfa63d0b42a30ad5ee492e51c07", + "zh:eb44b45a40f80a309dd5b0eb7d7fcb2cbfe588fe2f18b173ef5851346898a662", + ] +} diff --git a/terraform/vpc_private/backend.hcl b/terraform/vpc_private/backend.hcl new file mode 100644 index 0000000..96ff2fd --- /dev/null +++ b/terraform/vpc_private/backend.hcl @@ -0,0 +1,4 @@ +bucket = "yang-templates" +key = "terraform-states/etf-kpis-scraper/vpc-private/terraform.tfstate" +region = "us-east-1" +profile = "admin" diff --git a/terraform/vpc_private/main.tf b/terraform/vpc_private/main.tf new file mode 100644 index 0000000..9ea19e8 --- /dev/null +++ b/terraform/vpc_private/main.tf @@ -0,0 +1,14 @@ +terraform { + backend "s3" {} + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.region + profile = var.profile +} diff --git a/terraform/vpc_private/outputs.tf b/terraform/vpc_private/outputs.tf new file mode 100644 index 0000000..02806b3 --- /dev/null +++ b/terraform/vpc_private/outputs.tf @@ -0,0 +1,9 @@ +output "private_subnet_ids" { + description = "The IDs of the private subnets" + value = [for subnet in aws_subnet.private : subnet.id] +} + +output "security_group_id" { + description = "The ID of the security group" + value = aws_security_group.private.id +} diff --git a/terraform/vpc_private/variables.tf b/terraform/vpc_private/variables.tf new file mode 100644 index 0000000..cdd5f77 --- /dev/null +++ b/terraform/vpc_private/variables.tf @@ -0,0 +1,34 @@ +variable "region" { + description = "The AWS region to deploy resources in" + type = string +} + +variable "profile" { + description = "The AWS credentials profile to use for deployment" + type = string +} + +variable "stack_name" { + description = "The name of the stack, used for tagging and resource identification" + type = string +} + +variable "vpc_cidr" { + description = "The CIDR block for the VPC" + type = string +} + +variable "public_subnet_cidrs" { + description = "The CIDR blocks for public subnets" + type = list(string) +} + +variable "private_subnet_cidrs" { + description = "The CIDR blocks for private subnets" + type = list(string) +} + +variable "availability_zones" { + description = "List of availability zones to use" + type = list(string) +} diff --git a/terraform/vpc_private/vpc.tf b/terraform/vpc_private/vpc.tf new file mode 100644 index 0000000..c699484 --- /dev/null +++ b/terraform/vpc_private/vpc.tf @@ -0,0 +1,121 @@ +resource "aws_vpc" "private" { + cidr_block = var.vpc_cidr + enable_dns_support = true + enable_dns_hostnames = true + tags = { + Name = "${var.stack_name}_vpc" + } +} + +resource "aws_internet_gateway" "igw" { + vpc_id = aws_vpc.private.id + tags = { + Name = "${var.stack_name}_igw" + } +} + +resource "aws_subnet" "public" { + for_each = tomap({ + for index, cidr in var.public_subnet_cidrs : + index => { cidr = cidr, az = var.availability_zones[index] } + }) + + vpc_id = aws_vpc.private.id + cidr_block = each.value.cidr + map_public_ip_on_launch = true + availability_zone = each.value.az + + tags = { + Name = "${var.stack_name}_public_subnet_${each.key}" + } +} + +resource "aws_subnet" "private" { + for_each = tomap({ + for index, cidr in var.private_subnet_cidrs : + index => { cidr = cidr, az = var.availability_zones[index] } + }) + + vpc_id = aws_vpc.private.id + cidr_block = each.value.cidr + map_public_ip_on_launch = false + availability_zone = each.value.az + + tags = { + Name = "${var.stack_name}_private_subnet_${each.key}" + } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.private.id + tags = { + Name = "${var.stack_name}_public_rtb" + } +} + +resource "aws_route" "public_internet" { + route_table_id = aws_route_table.public.id + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.igw.id +} + +resource "aws_route_table_association" "public" { + for_each = aws_subnet.public + + subnet_id = each.value.id + route_table_id = aws_route_table.public.id +} + +resource "aws_eip" "nat" { + for_each = aws_subnet.public + domain = "vpc" +} + +resource "aws_nat_gateway" "nat" { + for_each = aws_subnet.public + + subnet_id = each.value.id + allocation_id = aws_eip.nat[each.key].id + tags = { + Name = "${var.stack_name}_nat_${each.key}" + } +} + +resource "aws_route_table" "private" { + for_each = aws_subnet.private + + vpc_id = aws_vpc.private.id + tags = { + Name = "${var.stack_name}_private_rtb_${each.key}" + } +} + +resource "aws_route" "private_nat" { + for_each = aws_route_table.private + + route_table_id = each.value.id + destination_cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.nat[each.key].id +} + +resource "aws_route_table_association" "private" { + for_each = aws_subnet.private + + subnet_id = each.value.id + route_table_id = aws_route_table.private[each.key].id +} + +resource "aws_security_group" "private" { + vpc_id = aws_vpc.private.id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "${var.stack_name}_sg" + } +} diff --git a/terraform/vpc_public/.terraform.lock.hcl b/terraform/vpc_public/.terraform.lock.hcl new file mode 100644 index 0000000..3c6e612 --- /dev/null +++ b/terraform/vpc_public/.terraform.lock.hcl @@ -0,0 +1,25 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.78.0" + constraints = "~> 5.0" + hashes = [ + "h1:o7jz+dFixEcwjfdubken5ldmDJm1tkvM2adPtNDei3g=", + "zh:0ae7d41b96441d0cf7ce2e1337657bdb2e1e5c9f1c2227b0642e1dcec2f9dfba", + "zh:21f8f1edf477681ea3b095c02cad6b8e85262e45015de58e84e0c7b2bfe9a1f6", + "zh:2bdc335e341bf98445255549ae93d66cfb9bca706e62b949da98fe467c182cad", + "zh:2fe4096e260367a225a9faf4a424d62b87e5498f12cb43bdb6f4e713d11b82c3", + "zh:3c63bb7a7925d65118d17461f4691a22dbb55ea39a7404e4d71f6ccca8765f8b", + "zh:6609a28a1c638a1901d8007b5386868ccfd313b4df2e98b35d9fdef436974e3b", + "zh:7ae3aef43bc4b365824cca4659cf92459d766800656e354bdbf83feabab835e8", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:c314efe454adc6ca483261c6906e64315aeb9db0c0332818714e9b81e07df0f0", + "zh:cd3e30396b554bbc1d260252db8a0f344065d619038fe60ea870689cd32c6aa9", + "zh:d1ba48fd9d8a1cb1daa927fb9e8bb708b857f2792d796e110460c6fdcd896a47", + "zh:d31c8abe75cb9cdc1c59ad9d356a1c3ae1ba8cd29ac15eb7e01b6cd01221ab04", + "zh:dc27c5c2116b4d9b404753f73bccaa635bce21f3bfb4bb7bc8e63225c36c98fe", + "zh:de491f0d05408378413187475c815d8cb2ac6bfa63d0b42a30ad5ee492e51c07", + "zh:eb44b45a40f80a309dd5b0eb7d7fcb2cbfe588fe2f18b173ef5851346898a662", + ] +} diff --git a/terraform/vpc_public/backend.hcl b/terraform/vpc_public/backend.hcl new file mode 100644 index 0000000..b71a551 --- /dev/null +++ b/terraform/vpc_public/backend.hcl @@ -0,0 +1,4 @@ +bucket = "yang-templates" +key = "terraform-states/etf-kpis-scraper/vpc-public/terraform.tfstate" +region = "us-east-1" +profile = "admin" diff --git a/terraform/vpc_public/main.tf b/terraform/vpc_public/main.tf new file mode 100644 index 0000000..9ea19e8 --- /dev/null +++ b/terraform/vpc_public/main.tf @@ -0,0 +1,14 @@ +terraform { + backend "s3" {} + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} + +provider "aws" { + region = var.region + profile = var.profile +} diff --git a/terraform/vpc_public/outputs.tf b/terraform/vpc_public/outputs.tf new file mode 100644 index 0000000..a38d5f6 --- /dev/null +++ b/terraform/vpc_public/outputs.tf @@ -0,0 +1,9 @@ +output "public_subnet_ids" { + description = "The IDs of the public subnets" + value = [for subnet in aws_subnet.public : subnet.id] +} + +output "security_group_id" { + description = "The ID of the security group" + value = aws_security_group.public.id +} diff --git a/terraform/vpc_public/variables.tf b/terraform/vpc_public/variables.tf new file mode 100644 index 0000000..67869f9 --- /dev/null +++ b/terraform/vpc_public/variables.tf @@ -0,0 +1,29 @@ +variable "region" { + description = "The AWS region to deploy resources in" + type = string +} + +variable "profile" { + description = "The AWS credentials profile to use for deployment" + type = string +} + +variable "stack_name" { + description = "The name of the stack, used for tagging and resource identification" + type = string +} + +variable "vpc_cidr" { + description = "The CIDR block for the VPC" + type = string +} + +variable "public_subnet_cidrs" { + description = "The CIDR blocks for public subnets" + type = list(string) +} + +variable "availability_zones" { + description = "List of availability zones to use" + type = list(string) +} diff --git a/terraform/vpc_public/vpc.tf b/terraform/vpc_public/vpc.tf new file mode 100644 index 0000000..e484a38 --- /dev/null +++ b/terraform/vpc_public/vpc.tf @@ -0,0 +1,66 @@ +resource "aws_vpc" "public" { + cidr_block = var.vpc_cidr + enable_dns_support = true + enable_dns_hostnames = true + tags = { + Name = "${var.stack_name}_vpc" + } +} + +resource "aws_internet_gateway" "igw" { + vpc_id = aws_vpc.public.id + tags = { + Name = "${var.stack_name}_igw" + } +} + +resource "aws_subnet" "public" { + for_each = tomap({ + for index, cidr in var.public_subnet_cidrs : + index => { cidr = cidr, az = var.availability_zones[index] } + }) + + vpc_id = aws_vpc.public.id + cidr_block = each.value.cidr + map_public_ip_on_launch = true + availability_zone = each.value.az + + tags = { + Name = "${var.stack_name}_public_subnet_${each.key}" + } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.public.id + tags = { + Name = "${var.stack_name}_rtb" + } +} + +resource "aws_route" "public_internet" { + route_table_id = aws_route_table.public.id + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.igw.id +} + +resource "aws_route_table_association" "public" { + for_each = aws_subnet.public + + subnet_id = each.value.id + route_table_id = aws_route_table.public.id +} + +resource "aws_security_group" "public" { + vpc_id = aws_vpc.public.id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "${var.stack_name}_sg" + } +}