From 13ab92c1a8d477b4eb663be5c51a5091defd2352 Mon Sep 17 00:00:00 2001 From: Snir Sheriber Date: Sun, 22 Dec 2024 15:16:13 +0200 Subject: [PATCH] podvm:aws: add support to precreated artifacts updated README with the instructions --- config/peerpods/podvm/aws-podvm-image-cm.yaml | 19 ++ .../peerpods/podvm/aws-podvm-image-handler.sh | 172 ++++++++++++++-- config/peerpods/podvm/bootc/README.md | 15 +- config/peerpods/podvm/lib.sh | 57 ++++-- .../peerpods/podvm/osc-podvm-create-job.yaml | 3 + scripts/ami-helper.sh | 193 ++++++++++++++++++ 6 files changed, 429 insertions(+), 30 deletions(-) create mode 100755 scripts/ami-helper.sh diff --git a/config/peerpods/podvm/aws-podvm-image-cm.yaml b/config/peerpods/podvm/aws-podvm-image-cm.yaml index 5dfab399..136b722e 100644 --- a/config/peerpods/podvm/aws-podvm-image-cm.yaml +++ b/config/peerpods/podvm/aws-podvm-image-cm.yaml @@ -36,3 +36,22 @@ data: # Custom Agent Policy #AGENT_POLICY: "" # set to base64 encoded agent policy + + # precreated artifacts + #BUCKET_NAME: existing-bucket-name + #PODVM_IMAGE_URI: bootc::image-registry.openshift-image-registry.svc:5000/openshift-sandboxed-containers-operator/podvm-bootc + # Custom bootc build configuration: https://osbuild.org/docs/bootc/#-build-config + #BOOTC_BUILD_CONFIG: | + # [[customizations.user]] + # name = "peerpod" + # password = "peerpod" + # key = "ssh-rsa AAAA..." + # groups = ["wheel", "root"] + # + # [[customizations.filesystem]] + # mountpoint = "/" + # minsize = "5 GiB" + # + # [[customizations.filesystem]] + # mountpoint = "/var/kata-containers" + # minsize = "15 GiB" diff --git a/config/peerpods/podvm/aws-podvm-image-handler.sh b/config/peerpods/podvm/aws-podvm-image-handler.sh index e202b76a..83e15345 100755 --- a/config/peerpods/podvm/aws-podvm-image-handler.sh +++ b/config/peerpods/podvm/aws-podvm-image-handler.sh @@ -115,15 +115,6 @@ function create_ami_using_packer() { # If any error occurs, exit the script with an error message # The variables are set before calling the function - # Set the AMI version - # It should follow the Major(int).Minor(int).Patch(int) - AMI_VERSION="${AMI_VERSION_MAJ_MIN}.$(date +'%Y%m%d%S')" - export AMI_VERSION - - # Set the image name - AMI_NAME="${AMI_BASE_NAME}-${AMI_VERSION}" - export AMI_NAME - # If PODVM_DISTRO is not set to rhel then exit [[ "${PODVM_DISTRO}" != "rhel" ]] && error_exit "unsupport distro" @@ -150,6 +141,17 @@ function create_ami_using_packer() { } +function set_ami_name() { + # Set the AMI version + # It should follow the Major(int).Minor(int).Patch(int) + AMI_VERSION="${AMI_VERSION_MAJ_MIN}.$(date +'%Y%m%d%S')" + export AMI_VERSION + + # Set the image name + AMI_NAME="${AMI_BASE_NAME}-${AMI_VERSION}" + export AMI_NAME +} + # Function to get the ami id of the newly created image function get_ami_id() { @@ -189,6 +191,29 @@ function get_all_ami_ids() { } +# Function to convert bootc container image to cloud image +# Input: +# 1. container_image_repo_url: The registry URL of the source container image. +# 2. image_tag: The tag of the source container image. +# 3. auth_json_file (can be empty): Path to the registry secret file to use for downloading the image. +# 4. aws_ami_name: Name for the AMI in AWS +# 5. aws_bucket: Target S3 bucket name for intermediate storage when creating AMI (bucket must exist) +# Output: ami + +function bootc_to_ami() { + container_image_repo_url="${1}" + image_tag="${2}" + auth_json_file="${3}" + aws_ami_name="${4}" + aws_bucket="${5}" # bucket must exist + + [[ -n "$AWS_ACCESS_KEY_ID" ]] && [[ -n "$AWS_SECRET_ACCESS_KEY" ]] && [[ -n "$AWS_REGION" ]] || error_exit "bootc_to_ami failed: AWS_* keys or AWS_REGION are missing" + # TODO: check permissions + run_args="--env AWS_ACCESS_KEY_ID --env AWS_SECRET_ACCESS_KEY" + bib_args="--type ami --aws-ami-name ${aws_ami_name} --aws-bucket ${aws_bucket} --aws-region ${AWS_REGION}" + bootc_image_builder_conversion "${container_image_repo_url}" "${image_tag}" "${auth_json_file}" "${run_args}" "${bib_args}" +} + # Function to create or update podvm-images configmap with all the amis # Input AMI_ID_LIST is a list of ami ids @@ -287,10 +312,106 @@ function delete_ami_id_annotation_from_peer_pods_cm() { echo "Ami id annotation deleted from peer-pods-cm configmap successfully" } -# Function to create the ami in AWS +function create_ami_from_prebuilt_artifact() { + echo "Creating AWS AMI image from prebuilt artifact" -function create_ami() { - echo "Creating AWS AMI" + echo "Pulling the podvm image from the provided path" + image_src="/tmp/image" + extraction_destination_path="/image" + image_repo_auth_file="/tmp/regauth/auth.json" + + [[ ! ${BUCKET_NAME} ]] && error_exit "BUCKET_NAME is not defined" + + # Get the PODVM_IMAGE_TYPE, PODVM_IMAGE_TAG and PODVM_IMAGE_SRC_PATH + get_image_type_url_and_path + + case "${PODVM_IMAGE_TYPE}" in + oci) # TODO: test + echo "Extracting the raw image from the given path." + + mkdir -p "${extraction_destination_path}" || + error_exit "Failed to create the image directory" + + extract_container_image "${PODVM_IMAGE_URL}" \ + "${PODVM_IMAGE_TAG}" \ + "${image_src}" \ + "${extraction_destination_path}" \ + "${image_repo_auth_file}" + + # Form the path of the podvm vhd image. + podvm_image_path="${extraction_destination_path}/rootfs/${PODVM_IMAGE_SRC_PATH}" + + upload_image_to_s3 + + register_ami + ;; + bootc) + echo "Converting the bootc image to AMI" + + bootc_to_ami "${PODVM_IMAGE_URL}" "${PODVM_IMAGE_TAG}" "${image_repo_auth_file}" "${AMI_NAME}" "${BUCKET_NAME}" + ;; + *) + error_exit "Currently only OCI image unpacking is supported, exiting." + ;; + esac +} + +function upload_image_to_s3() { + echo "Uploading the image to S3" + + # Check if the image exists + [[ ! -f "${podvm_image_path}" ]] && error_exit "Image does not exist" + + # Upload the image to S3 + aws s3 cp "${podvm_image_path}" "s3://${BUCKET_NAME}/${AMI_NAME}" || + error_exit "Failed to upload the image to S3" + + echo "Image uploaded to S3 successfully" +} + +function register_ami() { + echo "Starting image import..." + + get_podvm_image_format podvm_image_path + local import_task_id=$(aws ec2 import-image \ + --description "podvm-image created from qcow2/raw" \ + --disk-containers "file://<(echo '{\"Description\":\"podvm-image created from qcow2/raw\",\"Format\":\"${PODVM_IMAGE_FORMAT}\",\"UserBucket\":{\"S3Bucket\":\"${BUCKET_NAME}\",\"S3Key\":\"${AMI_NAME}\"}}')" \ + --query 'ImportTaskId' \ + --output text) + + echo "Import task started. Task ID: $IMPORT_TASK_ID" + + echo "Monitoring the import task..." + while true; do + local status=$(aws ec2 describe-import-image-tasks \ + --import-task-ids "$import_task_id" \ + --query 'ImportImageTasks[0].Status' \ + --output text) + + echo "Current status: $status" + + if [[ "$status" == "completed" ]]; then + local ami_id=$(aws ec2 describe-import-image-tasks \ + --import-task-ids "$IMPORT_TASK_ID" \ + --query 'ImportImageTasks[0].ImageId' \ + --output text) + echo "Import completed. AMI ID: $ami_id" + break + elif [[ "$status" == "deleted" || "$status" == "cancelled" ]]; then + echo "Import task failed or was cancelled." + exit 1 + fi + + sleep 15 + done + + # Tag the AMI + aws ec2 create-tags --resources "$ami_id" --tags Key="name",Value="${AMI_NAME}" + echo "Tags added successfully." +} + +function create_ami_from_scratch() { + echo "Creating AWS AMI from scratch" # Create the AWS image # If any error occurs, exit the script with an error message @@ -318,6 +439,33 @@ function create_ami() { # Create AWS ami using packer create_ami_using_packer +} + +# Function to create the ami in AWS + +function create_ami() { + echo "Creating AWS AMI" + + # Create the AWS image + # If any error occurs, exit the script with an error message + + # Install packages if INSTALL_PACKAGES is set to yes + if [[ "${INSTALL_PACKAGES}" == "yes" ]]; then + # Install required rpm packages + install_rpm_packages + + # Install required binary packages + install_binary_packages + fi + + # generate & set the ami name + set_ami_name + + if [[ "${IMAGE_TYPE}" == "operator-built" ]]; then + create_ami_from_scratch + elif [[ "${IMAGE_TYPE}" == "pre-built" ]]; then + create_ami_from_prebuilt_artifact + fi # Get the ami id of the newly created image # This will set the AMI_ID environment variable diff --git a/config/peerpods/podvm/bootc/README.md b/config/peerpods/podvm/bootc/README.md index f7cd661d..f9ca0a63 100644 --- a/config/peerpods/podvm/bootc/README.md +++ b/config/peerpods/podvm/bootc/README.md @@ -52,12 +52,13 @@ podman run \ registry.redhat.io/rhel9/bootc-image-builder:latest \ --type qcow2 \ --rootfs xfs \ - --local \ + --local \ # if pulled locally "${IMG}" ``` Artifact will be located at output/qcow2/disk.qcow2 Upload it to your cloud-provider. +**AWS:** See [Bootc Image Builder Instructions for AMI artifact](https://github.com/osbuild/bootc-image-builder?tab=readme-ov-file#amazon-machine-images-amis) ### **In-Cluster** Podvm Disk & Image Creation @@ -87,3 +88,15 @@ BOOTC_BUILD_CONFIG: | # Custom bootc build configuration: https://osbuild.org/d minsize = "15 GiB" ``` +#### AWS specifics + +In order to convert image to AMI (Amazon Machine Image) in-cluster you'll need: +* An existing s3 bucket in the region of your cluster +* Your cluster's AWS credntials needs to have the following [permissions](https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html#iam-permissions-image) +* [vmimport service role](https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html#vmimport-role) set +* The created bucket name needs to be specified in the aws-podvm-image-cm as follows: +``` +BUCKET_NAME= +``` + +**NOTE:** you may use the [ami-helper.sh](../../../../hack/ami-helper.sh) script to help and set the above requirements diff --git a/config/peerpods/podvm/lib.sh b/config/peerpods/podvm/lib.sh index a2ea9a96..0c3525a9 100644 --- a/config/peerpods/podvm/lib.sh +++ b/config/peerpods/podvm/lib.sh @@ -3,6 +3,9 @@ [[ "$DEBUG" == "true" ]] && set -x +# Bootc Defaults +BIB_IMAGE=${BIB_IMAGE:-registry.redhat.io/rhel9/bootc-image-builder:9.5} + # Defaults for pause image # This pause image is multi-arch PAUSE_IMAGE_REPO_DEFAULT="quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256" @@ -458,53 +461,73 @@ function validate_podvm_image() { echo "Checksum of the PodVM image: $(sha256sum "$image_path")" } -# Function to download and extract a container image. -# Accepts six arguments: - -# Function to convert qcow2 image to vhd image +# Function to convert local bootc container image to cloud image # Input: # 1. container_image_repo_url: The registry URL of the source container image. # 2. image_tag: The tag of the source container image. -# 3. auth_json_file (optional): Path to the registry secret file to use for downloading the image. -# Output: qcow2 image at output/qcow2/disk.qcow2 -function bootc_to_qcow2() { +# 3. auth_json_file (can be empty): Path to the registry secret file to use for downloading the image. +# 4. run_args: arguments for the podman run command that runs bootc image builder +# 5. bib_args: arguments for bootc image builder itself +# Output: based on input, either, qcow2 image at output/qcow2/disk.qcow2 or ami +# see: https://github.com/osbuild/bootc-image-builder +function bootc_image_builder_conversion() { container_image_repo_url="${1}" image_tag="${2}" auth_json_file="${3}" + run_args="${4}" + bib_args="${5}" -# some VM customizations , TODO: allow custom config + # some VM customizations echo "${BOOTC_BUILD_CONFIG}" >> ./config.toml echo "config.toml:" cat ./config.toml # login for local registry pulling # TODO: can we use token instead? - if [[ "${PODVM_IMAGE_URL}" == *"image-registry.openshift-image-registry.svc"* ]]; then + if [[ "${container_image_repo_url}" == *"image-registry.openshift-image-registry.svc"* ]]; then + echo "login to local registry" mkdir /etc/containers/certs.d/image-registry.openshift-image-registry.svc:5000 ln -s /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt /etc/containers/certs.d/image-registry.openshift-image-registry.svc:5000/service-ca.crt podman login -u kubeadmin -p $(cat /var/run/secrets/kubernetes.io/serviceaccount/token) image-registry.openshift-image-registry.svc:5000 + podman pull "${container_image_repo_url}:${image_tag}" || error_exit "Failed to pull local bootc image" + else + # pull first to authenticate properly, if REGISTRY_AUTH_FILE is empty, it's ignored + REGISTRY_AUTH_FILE=${auth_json_file} podman pull "${container_image_repo_url}:${image_tag}" || error_exit "Failed to pull bootc image" fi - # pull first to authenticate properly, if REGISTRY_AUTH_FILE is empty, it's ignored - REGISTRY_AUTH_FILE=${auth_json_file} podman pull "${container_image_repo_url}:${image_tag}" || error_exit "Failed to pull bootc image" # execute bootc-image-builder # TODO: check if we can avoid this to drop the /store volumeMount - # TODO: use fixed version # REGISTRY_AUTH_FILE is needed to access bib image - mkdir output REGISTRY_AUTH_FILE=${CLUSTER_PULL_SECRET_AUTH_FILE} podman run \ -it \ --privileged \ --security-opt label=type:unconfined_t \ -v $(pwd)/config.toml:/config.toml:ro \ - -v $(pwd)/output:/output \ -v /store:/store \ -v /var/lib/containers/storage:/var/lib/containers/storage \ - registry.redhat.io/rhel9/bootc-image-builder:latest \ - --type qcow2 \ + ${run_args} \ + ${BIB_IMAGE} \ + ${bib_args} \ --rootfs xfs \ --local \ - "${container_image_repo_url}:${image_tag}" || error_exit "Failed to convert bootc image" + ${container_image_repo_url}:${image_tag} || error_exit "Failed to convert bootc image" +} + +# Function to convert bootc container image to qcow2 +# Input: +# 1. container_image_repo_url: The registry URL of the source container image. +# 2. image_tag: The tag of the source container image. +# 3. auth_json_file (can be empty): Path to the registry secret file to use for downloading the image. +# Output: qcow2 image at output/qcow2/disk.qcow2 +function bootc_to_qcow2() { + container_image_repo_url="${1}" + image_tag="${2}" + auth_json_file="${3}" + + mkdir output + run_args="-v $(pwd)/output:/output" + bib_args="--type qcow2" + bootc_image_builder_conversion "${container_image_repo_url}" "${image_tag}" "${auth_json_file}" "${run_args}" "${bib_args}" } # Function to convert qcow2 image to vhd image diff --git a/config/peerpods/podvm/osc-podvm-create-job.yaml b/config/peerpods/podvm/osc-podvm-create-job.yaml index 6415673a..2013100b 100644 --- a/config/peerpods/podvm/osc-podvm-create-job.yaml +++ b/config/peerpods/podvm/osc-podvm-create-job.yaml @@ -49,6 +49,9 @@ spec: envFrom: - secretRef: name: peer-pods-secret + - secretRef: + name: peer-pods-image-creation-secret # must come after peer-pods-secret to override the values + optional: true - configMapRef: name: peer-pods-cm optional: true diff --git a/scripts/ami-helper.sh b/scripts/ami-helper.sh new file mode 100755 index 00000000..3c85692c --- /dev/null +++ b/scripts/ami-helper.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# This script is used to create the bucket and the service role needed for the AMI creation +# it also asks for the credentials and set the secret needed for podvm image (AMI) creation +# which is executed during the sandboxed containers operator installtion process (skip with -s) + +[ "$DEBUG" == 'true' ] && set -x + +function usage() { + cat < Set the bucket name (otherwise it will be randomly generated) + -c Clean credentials and exit + -d Delete the bucket and exit + -h Print this help message + -r Set the region (otherwise it will be fetched from the cluster) + -s Skip credentials request +EOF +} + +while getopts ":b:cdhr:s" opt; do + case ${opt} in + b ) BUCKET_NAME=$OPTARG;; + c ) clean_credentials=true;; + d ) delete_bucket=true;; + h ) usage && exit 0;; + r ) REGION=$OPTARG;; + s ) skip_cr=true;; + \? ) echo "Invalid option: -$OPTARG" >&2 && usage && exit 1;; + esac +done + + +prepare() { + TMPDIR=$(mktemp -d) + TRUST_POLICY_JSON_FILE="${TMPDIR}/trust-policy.json" + ROLE_POLICY_JSON_FILE="${TMPDIR}/role-policy.json" + CREDENTIALS_REQUEST_YAML_FILE="${TMPDIR}/vmimport_credentials_request.yaml" + CREDENTIALS_REQUEST_JSON_FILE="${TMPDIR}/update_credentials_request.json" + SECRET_NAME="peer-pods-image-creation-secret" + echo "Temporary Workdir: ${TMPDIR}" +} + +init() { + command -v oc &> /dev/null || { echo "oc command was not found" 1>&2 ; exit 1; } + oc cluster-info &> /dev/null || { echo "cluster is not configured" 1>&2 ; exit 1; } + [[ -x "$(command -v aws)" ]] || { echo "aws is not installed" 1>&2 ; exit 1; } + aws sts get-caller-identity &>/dev/null || { echo "aws cli missing credentials"; exit 1; } + [[ $REGION ]] || REGION=$(oc get infrastructure -n cluster -o=jsonpath='{.items[0].status.platformStatus.aws.region}') || { echo "Region couln't be fetched, add as argument" 1>&2 ; exit 1; } + [[ ! $BUCKET_NAME ]] && uid=$(kubectl get infrastructure -n cluster -o jsonpath='{.items[*].metadata.uid}') && BUCKET_NAME=osc-${uid:0:6}-bucket && \ + echo "Bucket name not provided, using ${BUCKET_NAME}, make sure to set it in the aws-podvm-image-cm" + + echo "Bucket Name: ${BUCKET_NAME}" + echo "Region: ${REGION}" +} + +delete_bucket() { + echo "Delete s3 Bucket named ${BUCKET_NAME} at ${REGION}" + aws s3api delete-bucket --bucket ${BUCKET_NAME} --region ${REGION} +} + +create_bucket() { + echo "Create s3 Bucket named ${BUCKET_NAME} at ${REGION}" + if [[ ${REGION} == us-east-1 ]]; then + aws s3api create-bucket --bucket ${BUCKET_NAME} --region ${REGION} + else + aws s3api create-bucket --bucket ${BUCKET_NAME} --region ${REGION} --create-bucket-configuration LocationConstraint=${REGION} + fi +} + +set_service_role() { + echo "Create the service role" + cat < "${TRUST_POLICY_JSON_FILE}" +{ + "Version":"2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Principal":{ "Service":"vmie.amazonaws.com" }, + "Action": "sts:AssumeRole", + "Condition":{"StringEquals":{"sts:Externalid":"vmimport"}} + } + ] +} +EOF + + aws iam create-role --role-name vmimport --assume-role-policy-document "file://${TRUST_POLICY_JSON_FILE}" --region ${REGION} + + echo "Attach policy" + cat < "${ROLE_POLICY_JSON_FILE}" +{ + "Version":"2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Action":["s3:GetBucketLocation","s3:GetObject","s3:ListBucket"], + "Resource":["arn:aws:s3:::${BUCKET_NAME}","arn:aws:s3:::${BUCKET_NAME}/*"] + }, + { + "Effect":"Allow", + "Action":["ec2:ModifySnapshotAttribute","ec2:CopySnapshot","ec2:RegisterImage","ec2:Describe*"], + "Resource":"*" + } + ] +} +EOF + + aws iam put-role-policy --role-name vmimport --policy-name vmimport --policy-document "file://${ROLE_POLICY_JSON_FILE}" --region ${REGION} +} + +clean_credentials() { + echo "Clean credentials" + oc delete credentialsrequest aws-vmimport -n openshift-cloud-credential-operator +} + +get_credentials() { + [[ $skip_cr ]] && return + echo "Ask for addtional credentials" + oc get ns openshift-sandboxed-containers-operator >/dev/null 2>&1 || (echo "OSC namespace is missing, re-run after installting the operator" && exit 1) + + cat < "${CREDENTIALS_REQUEST_YAML_FILE}" +apiVersion: cloudcredential.openshift.io/v1 +kind: CredentialsRequest +metadata: + name: aws-vmimport + namespace: openshift-cloud-credential-operator +spec: + providerSpec: + apiVersion: cloudcredential.openshift.io/v1 + kind: AWSProviderSpec + statementEntries: + - effect: Allow + action: + - s3:GetBucketLocation + - s3:GetBucketAcl + resource: "arn:aws:s3:::${BUCKET_NAME}" + - effect: Allow + action: + - s3:GetOcbject + - s3:PutObject + - s3:DeleteObject + resource: "arn:aws:s3:::${BUCKET_NAME}/*" + - effect: Allow + action: + - ec2:CancelConversionTask + - ec2:CancelExportTask + - ec2:CreateImage + - ec2:CreateInstanceExportTask + - ec2:CreateTags + - ec2:DescribeConversionTasks + - ec2:DescribeExportTasks + - ec2:DescribeExportImageTasks + - ec2:DescribeImages + - ec2:DescribeInstanceStatus + - ec2:DescribeInstances + - ec2:DescribeSnapshots + - ec2:DescribeTags + - ec2:DescribeRegions + - ec2:ExportImage + - ec2:ImportInstance + - ec2:ImportVolume + - ec2:StartInstances + - ec2:StopInstances + - ec2:TerminateInstances + - ec2:ImportImage + - ec2:ImportSnapshot + - ec2:DescribeImportImageTasks + - ec2:DescribeImportSnapshotTasks + - ec2:CancelImportTask + - ec2:RegisterImage + - s3:ListAllMyBuckets + resource: "*" + secretRef: + name: ${SECRET_NAME} + namespace: openshift-sandboxed-containers-operator +EOF + oc apply -f ${CREDENTIALS_REQUEST_YAML_FILE} + while ! oc get secret ${SECRET_NAME} -n openshift-sandboxed-containers-operator; do echo "Waiting for secret."; sleep 1; done + # Convert key names to uppercase as expected + oc get secret ${SECRET_NAME} -n openshift-sandboxed-containers-operator -o yaml | sed -E 's/aws_([a-z]|_)*:/\U&/g' | oc replace -f - +} + +init + +[[ $clean_credentials ]] && clean_credentials; [[ $delete_bucket ]] && delete_bucket; [[ $clean_credentials || $delete_bucket ]] && exit 0 + +prepare + +create_bucket + +set_service_role + +get_credentials