version 19-01-2024

CHANGELOG.md

@@ -0,0 +1,49 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## version v0.0.6
+
+### Added
+
+- Add Amazon Step Function workflow to massively parallelize jobs
+- Add Amazon FSx for Lustre cluster to optimize the upload/download of large media assets
+- Add Amazon System Manager Automation to preload large media assets from Amazon S3 to FSx for Lustre cluster
+- Add API resources for Step Functions
+- Document HTTP REST API
+
+### Changed
+
+- Refactor all cdk stacks
+- **Breaking:** Refactor HTTP REST API
+- Upgrade Nvidia Container to CUVID 12.3.1
+
+## version v0.0.5
+
+- Upgrade FFmpeg to 6.0 (snapshots)
+- Upgrade all FFmpeg libraries including decoders and encoders
+
+## version v0.0.4
+
+- Optimize code linting
+- Fix security issues
+- Refactor list of AWS EC2 Instance families per Region without boto3 (doc/architecture/0003-rollback-automatic-list-of-instance-types-per-aws-region.md)
+- Upgrade AWS CDK libraries including AWS Batch to 2.96 and CDK Nag
+- Add new compute instance family: VT1 with the support of AMD-Xilinx Video SDK 3.0 (<https://aws.amazon.com/about-aws/whats-new/2023/08/amazon-ec2-vt1-improved-control-stream-quality-latency-bandwidth/>)
+- Upgrade Python Lambda Runtime and add runtime management to AUTO
+- Upgrade Python Container Runtime
+
+## version v0.0.3
+
+- Update FFMPEG 5.1
+- Update Nvidia Cuda
+- Fix issue on AWS Athena View
+
+## version v0.0.2
+
+- Add FFmpeg Quality Metrics to AWS Glue Crawler
+- Create AWS Athena Views for PSNR, SSIM, VMAF quality metrics
+- Optimize code linting
+- Dynamically look after AWS EC2 Instance types per AWS Region
+- Add AWS Service Catalog Registry
+- Document architecture decisions

Taskfile.yml

@@ -1,72 +1,93 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
-
-version: "3"
-
-dotenv: [".env", "{{.ENV}}/.env.", "{{.HOME}}/.env"]
-
-vars:
-  IMAGE_REPO_NAME: batch-ffmpeg
-
+dotenv:
+  - .env
+  - "{{.ENV}}/.env."
+  - "{{.HOME}}/.env"
 includes:
   app:
-    taskfile: ./application
     dir: ./application
-  test:
-    taskfile: ./tests
-    dir: ./
-    optional: true
+    taskfile: ./application
   cdk:
-    taskfile: ./cdk
     dir: ./
+    taskfile: ./cdk
   dist:
+    dir: ./
+    optional: true
     taskfile: ./deployment
+  doc:
+    dir: ./doc
+    optional: true
+    taskfile: ./doc
+  test:
     dir: ./
     optional: true
-
+    taskfile: ./tests
 tasks:
+  env:
+    cmds:
+      - rm -f .env || true
+      - echo 'S3_BUCKET={{.S3_BUCKET}}' >> .env
+      - echo 'IMAGE_REPO_NAME={{.IMAGE_REPO_NAME}}' >> .env
+      - echo 'AWS_DEFAULT_REGION={{.AWS_DEFAULT_REGION}}' >> .env
+      - echo 'AWS_ACCOUNT_ID={{.AWS_ACCOUNT_ID}}' >> .env
+      - echo 'API_URL={{.API_URL}}' >> .env
+      - echo 'API_ID={{.API_ID}}' >> .env
+      - echo 'LAMBDA_METRICS_ARN={{.LAMBDA_METRICS_ARN}}' >> .env
+    desc: env. variables for scripts
+    status:
+      - test -f .env
+    vars:
+      API_ID:
+        sh:
+          aws cloudformation describe-stacks --stack-name batch-ffmpeg-api-stack
+          --query 'Stacks[0].Outputs[?ExportName==`ffmpeg-batch-api-id`].OutputValue'
+          --output text || true
+      LAMBDA_METRICS_ARN:
+        sh:
+          aws cloudformation describe-stacks --stack-name batch-ffmpeg-metrics-stack
+          --query 'Stacks[0].Outputs[?ExportName==`batch-ffmpeg-lambda-metrics-arn`].OutputValue'
+          --output text || true
+      API_URL:
+        sh:
+          aws cloudformation describe-stacks --stack-name batch-ffmpeg-api-stack
+          --query 'Stacks[0].Outputs[?ExportName==`ffmpeg-batch-api`].OutputValue'
+          --output text || true
+      AWS_ACCOUNT_ID:
+        sh: aws sts get-caller-identity --query "Account" --output text || true
+      AWS_DEFAULT_REGION:
+        sh: aws configure get region || true
+      IMAGE_REPO_NAME:
+        sh:
+          aws cloudformation describe-stacks --stack-name batch-ffmpeg-storage-stack
+          --query 'Stacks[0].Outputs[?OutputKey==`EcrRegistry`].OutputValue' --output
+          text || true
+      S3_BUCKET:
+        sh:
+          aws cloudformation describe-stacks --stack-name batch-ffmpeg-storage-stack
+          --query 'Stacks[0].Outputs[?OutputKey==`S3bucket`].OutputValue' --output
+          text || true
   venv:
-    desc: Create local python virtual env
     cmds:
       - rm -rf .venv/ || true
       - python3 -m venv .venv
       - .venv/bin/python3 -m pip install --upgrade --quiet pip
       - .venv/bin/pip install --quiet -r requirements.txt
       - .venv/bin/pip install --quiet -r application/requirements.txt
       - .venv/bin/pip install --quiet -r tests/requirements.txt
-    sources:
-      - requirements.txt
+    desc: Create local python virtual env
     generates:
       - .venv/bin/activate
-
+    sources:
+      - requirements.txt
   venv:upgrade:
-    desc: upgrade python packages in python virtual env
     cmds:
       - .venv/bin/python3 -m pip install --upgrade --quiet pip
       - .venv/bin/pip install --upgrade --quiet -r requirements.txt
       - .venv/bin/pip install --upgrade --quiet -r application/requirements.txt
       - .venv/bin/pip install --upgrade --quiet -r tests/requirements.txt
       - .venv/bin/pip list
-
-  env:
-    desc: env. variables for scripts
-    cmds:
-      - rm -f .env || true
-      - echo 'S3_BUCKET={{.S3_BUCKET}}' >> .env
-      - echo 'IMAGE_REPO_NAME={{.IMAGE_REPO_NAME}}' >> .env
-      - echo 'AWS_DEFAULT_REGION={{.AWS_DEFAULT_REGION}}' >> .env
-      - echo 'AWS_ACCOUNT_ID={{.AWS_ACCOUNT_ID}}' >> .env
-      - echo 'API_URL={{.API_URL}}' >> .env
-    vars:
-      S3_BUCKET:
-        sh: aws cloudformation describe-stacks --stack-name batch-ffmpeg-stack --query 'Stacks[0].Outputs[?OutputKey==`S3bucket`].OutputValue' --output text || true
-      IMAGE_REPO_NAME:
-        sh: aws cloudformation describe-stacks --stack-name batch-ffmpeg-registry-stack --query 'Stacks[0].Outputs[?OutputKey==`EcrRegistry`].OutputValue' --output text || true
-      API_URL:
-        sh: aws cloudformation describe-stacks --stack-name batch-ffmpeg-api-stack --query 'Stacks[0].Outputs[?OutputKey==`ffmpegbatchapi`].OutputValue' --output text || true
-      AWS_ACCOUNT_ID:
-        sh: aws sts get-caller-identity --query "Account" --output text || true
-      AWS_DEFAULT_REGION:
-        sh: aws configure get region || true
-    status:
-      - test -f .env
+    desc: upgrade python packages in python virtual env
+vars:
+  IMAGE_REPO_NAME: batch-ffmpeg
+version: "3"

app.py

@@ -9,9 +9,11 @@
 from aws_cdk import Aspects
 
 from cdk.api_stack import ApiStack
-from cdk.batch_job_ffmpeg_stack import BatchJobFfmpegStack
+from cdk.batch_stack import BatchStack
+from cdk.landing_zone_stack import LandingZoneStack
 from cdk.metrics_stack import MetricsStack
-from cdk.registry_stack import RegistryStack
+from cdk.sfn_stack import SfnStack
+from cdk.storage_stack import StorageStack
 
 account = os.environ.get("CDK_DEPLOY_ACCOUNT", os.environ["CDK_DEFAULT_ACCOUNT"])
 region = os.environ.get("CDK_DEPLOY_REGION", os.environ["CDK_DEFAULT_REGION"])
@@ -24,25 +26,71 @@
 app = cdk.App()
 cdk.Tags.of(app).add("application", "batch-ffmpeg")
 
-registry_stack = RegistryStack(app, "batch-ffmpeg-registry-stack", env=env)
-batch_stack = BatchJobFfmpegStack(
+landing_zone_stack = LandingZoneStack(
+    app, "batch-ffmpeg-landing-stack", env=env, description="AWS Batch with FFmpeg"
+)
+storage_stack = StorageStack(
+    app,
+    "batch-ffmpeg-storage-stack",
+    env=env,
+    description="AWS Batch with FFmpeg",
+    vpc=landing_zone_stack.vpc,
+)
+
+batch_stack = BatchStack(
     app,
     "batch-ffmpeg-stack",
-    ecr_registry=registry_stack.ecr_registry,
+    ecr_registry=storage_stack.ecr_registry,
     env=env,
-    description="Main stack with AWS Batch (uksb-1tg6b0m8t)",
+    vpc=landing_zone_stack.vpc,
+    s3_bucket=storage_stack.s3_bucket,
+    lustre_fs=storage_stack.lustre_fs,
+    description="AWS Batch with FFmpeg: Main stack (uksb-1tg6b0m8t)",
 )
 metrics_stack = MetricsStack(
-    app, "batch-ffmpeg-metrics-stack", s3_bucket=batch_stack.s3_bucket, env=env
+    app,
+    "batch-ffmpeg-metrics-stack",
+    s3_bucket=storage_stack.s3_bucket,
+    env=env,
+    description="AWS Batch with FFmpeg : Metrics stack",
+)
+
+sfn_stack = SfnStack(
+    app,
+    "batch-ffmpeg-sfn-stack",
+    s3_bucket=storage_stack.s3_bucket,
+    env=env,
+    description="AWS Batch with FFmpeg : AWS Step Functions",
 )
+
 api_stack = ApiStack(
     app,
     "batch-ffmpeg-api-stack",
     video_batch_jobs=batch_stack.video_batch_jobs,
-    metrics_handler=metrics_stack.handler,
+    lustre_fs=storage_stack.lustre_fs,
+    sfn_state_machine=sfn_stack.sfn_state_machine,
+    ssm_document=storage_stack.ssm_document,
     env=env,
+    description="AWS Batch with FFmpeg : API Gateway",
 )
-
+application = appreg.ApplicationAssociator(
+    app,
+    "batch-ffmepg-app",
+    applications=[
+        appreg.TargetApplication.create_application_stack(
+            application_name="batch-ffmpeg",
+            description="AWS Batch with FFMPEG : Application",
+            stack_name="batch-ffmpeg-application",
+            env=env,
+        )
+    ],
+)
+application.node.add_dependency(landing_zone_stack)
+application.node.add_dependency(storage_stack)
+application.node.add_dependency(metrics_stack)
+application.node.add_dependency(api_stack)
+application.node.add_dependency(batch_stack)
+app.synth()
 # cdk nag
 cdk_nag.NagSuppressions.add_resource_suppressions(
     app,
@@ -92,21 +140,3 @@
 Aspects.of(app).add(
     cdk_nag.AwsSolutionsChecks(log_ignores=True, verbose=True, reports=True)
 )
-
-application = appreg.ApplicationAssociator(
-    app,
-    "batch-ffmepg-app",
-    applications=[
-        appreg.TargetApplication.create_application_stack(
-            application_name="batch-ffmpeg",
-            description="AWS Solution : AWS Batch with FFMPEG",
-            stack_name="batch-ffmpeg-application",
-            env=env,
-        )
-    ],
-)
-application.node.add_dependency(registry_stack)
-application.node.add_dependency(metrics_stack)
-application.node.add_dependency(api_stack)
-application.node.add_dependency(batch_stack)
-app.synth()

application/aws/aws_helper.py

@@ -1,19 +1,29 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+import logging
 import os
 
 import boto3
 from botocore.exceptions import ClientError
 from ec2_metadata import ec2_metadata
 
+LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper()
+logging.basicConfig(level=LOGLEVEL)
+logging.getLogger("aws_xray_sdk").setLevel(LOGLEVEL)
+
 
 def s3_key_exist(client, bucket, key):
-    """Return True if exist, else None."""
+    """Return True if exist, else False."""
     try:
         client.head_object(Bucket=bucket, Key=key)
-    except ClientError as exc:
-        if exc.response["Error"]["Code"] != "404":
-            return False
+    except ClientError as e:
+        logging.info(
+            "Object (%s %s) does not exist on S3 - error code: %s",
+            bucket,
+            key,
+            e.response["Error"]["Code"],
+        )
+        return False
     return True
 
 

application/docker-images/6.0/nvidia2004-amd64/Dockerfile

@@ -9,7 +9,8 @@
 # Guillaume Marchand - AWS - gmarchan@amazon.fr
 #
 
-FROM        nvidia/cuda:12.2.0-devel-ubuntu20.04 AS base
+# Development
+FROM        nvidia/cuda:12.3.1-devel-ubuntu20.04 AS devel-base
 
 ENV	    NVIDIA_DRIVER_CAPABILITIES compute,utility,video
 ENV	    DEBIAN_FRONTEND=noninteractive
@@ -33,7 +34,33 @@ RUN add-apt-repository ppa:deadsnakes/ppa -y \
     && update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 \
     && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1
 
-FROM base as build
+# Runtime
+FROM        nvidia/cuda:12.3.1-runtime-ubuntu20.04 AS runtime-base
+
+ENV	    NVIDIA_DRIVER_CAPABILITIES compute,utility,video
+ENV	    DEBIAN_FRONTEND=noninteractive
+WORKDIR     /tmp/workdir
+
+RUN     apt-get -yqq update && \
+        apt-get -yqq upgrade && \
+        apt-get install -yqq --no-install-recommends ca-certificates expat libgomp1 software-properties-common libxcb-shape0-dev && \
+        apt-get autoremove -y && \
+        apt-get clean -y && \
+        rm -rf /var/lib/apt/lists/*
+
+# Install Python 3
+ARG PYTHON_VERSION=3.11
+RUN add-apt-repository ppa:deadsnakes/ppa -y \
+    && apt-get install -yqq --fix-missing --no-install-recommends curl python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-distutils python${PYTHON_VERSION}-venv python-is-python3 \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/* \
+    && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \
+    && python${PYTHON_VERSION} -m pip install --quiet --no-cache-dir --upgrade pip \
+    && update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 \
+    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1
+
+# Build
+FROM devel-base as build
 
 ENV     FFMPEG_VERSION=snapshot \
         NVIDIA_HEADERS_VERSION=12.1.14.0 \
@@ -442,7 +469,7 @@ RUN \
         sed "s:${PREFIX}:/usr/local:g; s:/lib64:/lib:g" <"$pc" >/usr/local/lib/pkgconfig/"${pc##*/}"; \
         done
 
-FROM        base AS release
+FROM        runtime-base AS release
 
 ENV         LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64