feat(KONFLUX-4892): supply layer size information to pyxis

tasks/create-pyxis-image/README.md

@@ -17,6 +17,8 @@ The relative path of the pyxis.json file in the data workspace is output as a ta
 | isLatest    | If set to true, the images will have a latest tag added with their Pyxis entries                                                                                                                                                                                                                                                                                                                            | Yes      | false         |
 | rhPush      | If set to true, an additional entry will be created in ContainerImage.repositories with the registry and repository fields converted to use Red Hat's official registry. E.g. a mapped repository of "quay.io/redhat-pending/product---my-image" will be converted to use registry "registry.access.redhat.com" and repository "product/my-image". Also, this repository entry will be marked as published. | Yes      | false         |
 | snapshotPath | Path to the JSON string of the mapped Snapshot spec in the data workspace                                                                                                                                                                                                                                                                                                                                   | No       | -             |
+## Changes in 3.4.0
+* Added support for uploading uncompressed layer information to pyxis
 
 ## Changes in 3.3.4
 * Properly expand platform variables for oras args

tasks/create-pyxis-image/create-pyxis-image.yaml

@@ -4,7 +4,7 @@ kind: Task
 metadata:
   name: create-pyxis-image
   labels:
-    app.kubernetes.io/version: "3.3.4"
+    app.kubernetes.io/version: "3.4.0"
   annotations:
     tekton.dev/pipelines.minVersion: "0.12.1"
     tekton.dev/tags: release
@@ -48,7 +48,7 @@ spec:
       description: The relative path in the workspace to the stored pyxis data json
   steps:
     - name: create-pyxis-image
-      image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+      image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
       env:
         - name: pyxisCert
           valueFrom:
@@ -156,19 +156,66 @@ spec:
                   ORAS_ARGS+=(--platform "$OS/$ARCH")
                 fi
 
+                # Save the OCI manifest locally, to pass to a script to create the pyxis entry
+                MANIFEST_FILE="$(workspaces.data.path)/$(dirname "$(params.snapshotPath)")/oras-manifest-fetch.json"
                 oras manifest fetch \
                   --registry-config "$AUTH_FILE" \
                   "${ORAS_ARGS[@]}" \
                   "${PULLSPEC}" \
-                      | tee /tmp/oras-manifest-fetch.json
+                      | tee "${MANIFEST_FILE}"
+
+                # Augment that manifest with further information about the layers, decompressed
+                # This requires pulling the layers to decompress and then measure them
+                while IFS= read -r BLOB_DETAIL;
+                do
+                    BLOB_TYPE=$(jq -r '.mediaType' <<< "$BLOB_DETAIL")
+                    BLOB_DIGEST=$(jq -r '.digest' <<< "$BLOB_DETAIL")
+
+                    # Normal images will always have the layers compressed.
+                    # If they are not compressed, this will not save the
+                    # uncompressed data in Pyxis.
+                    #
+                    # It's also possible that the layers are compressed with
+                    # some scheme other than gzip.  In that case, the
+                    # uncompressed layer information will also not be saved in
+                    # Pyxis.
+                    # https://github.com/konflux-ci/build-definitions/issues/1264
+
+                    if [[ "$BLOB_TYPE" =~ ^.*\.gzip$|^.*\+gzip$ ]]; then
+                        BLOB_FILE="/tmp/oras-blob-fetch-${BLOB_DIGEST}"
+                        BLOB_PULLSPEC="${PULLSPEC%%@*}@${BLOB_DIGEST}"
+
+                        # Save the blob
+                        oras blob fetch \
+                          --registry-config "$AUTH_FILE" \
+                          --output "${BLOB_FILE}.gz" \
+                          "${BLOB_PULLSPEC}"
+
+                        # Decompress it
+                        gunzip "${BLOB_FILE}.gz"
+
+                        # Measure it
+                        EXPANDED_DIGEST=$(sha256sum "${BLOB_FILE}" | cut -d " " -f 1)
+                        EXPANDED_SIZE=$(wc --bytes "${BLOB_FILE}" | awk '{print $1}' | tr -d '\n')
+
+                        # Append this information to the parsed_data manifest
+                        jq \
+                          '.uncompressed_layers += [{"digest": "'"$EXPANDED_DIGEST"'", "size": '"$EXPANDED_SIZE"'}]' \
+                          "${MANIFEST_FILE}" > "${MANIFEST_FILE}.tmp"
+                        mv "${MANIFEST_FILE}.tmp"  "${MANIFEST_FILE}"
+
+                        # Clean up, in case we're dealing with large images
+                        rm "/tmp/oras-blob-fetch-${BLOB_DIGEST}"
+                    fi
+                done <<< "$(jq -c '.layers[]' "${MANIFEST_FILE}")"
 
                 PYXIS_CERT_PATH=/tmp/crt PYXIS_KEY_PATH=/tmp/key create_container_image \
                   --pyxis-url $PYXIS_URL \
                   --certified "$(params.certified)" \
                   --tags "$TAGS" \
                   --is-latest "$(params.isLatest)" \
                   --verbose \
-                  --oras-manifest-fetch "/tmp/oras-manifest-fetch.json" \
+                  --oras-manifest-fetch "${MANIFEST_FILE}" \
                   --name "$REPOSITORY" \
                   --media-type "$MEDIA_TYPE" \
                   --digest "$DIGEST" \

tasks/create-pyxis-image/tests/mocks.sh

@@ -9,7 +9,7 @@ function create_container_image() {
   # e.g. 0001, 0002, 0003...
   echo The image id is $(awk 'END{printf("%04i", NR)}' $(workspaces.data.path)/mock_create_container_image.txt)
 
-  if [[ "$*" != "--pyxis-url https://pyxis.preprod.api.redhat.com/ --certified false --tags "*" --is-latest false --verbose --oras-manifest-fetch /tmp/oras-manifest-fetch.json --name "*" --media-type "*" --digest "*" --architecture-digest "*" --architecture "*" --rh-push "* ]]
+  if [[ "$*" != "--pyxis-url https://pyxis.preprod.api.redhat.com/ --certified false --tags "*" --is-latest false --verbose --oras-manifest-fetch "*" --name "*" --media-type "*" --digest "*" --architecture-digest "*" --architecture "*" --rh-push "* ]]
   then
     echo Error: Unexpected call
     echo Mock create_container_image called with: $*
@@ -35,7 +35,7 @@ function skopeo() {
     echo '{"mediaType": "application/vnd.oci.image.index.v1+json"}'
   elif [[ "$*" == "inspect --raw docker://"* ]] || [[ "$*" == "inspect --no-tags --override-os linux --override-arch "*" docker://"* ]]
   then
-    echo '{"mediaType": "my_media_type"}'
+    echo '{"mediaType": "my_media_type+gzip"}'
   else
     echo Mock skopeo called with: $*
     if [[ "$*" != "inspect --no-tags docker://"* ]]
@@ -64,9 +64,18 @@ function oras() {
   if [[ "$*" == "manifest fetch --registry-config"*.dockerfile ]]
   then
     echo '{"layers": [{"annotations": {"org.opencontainers.image.title": "Dockerfile.custom"}}]}'
+  elif [[ "$*" == "blob fetch --registry-config"*"/tmp/oras-blob-fetch-beef.gz"* ]]
+  then
+    echo -n 'H4sIAAAAAAAAA0vKzEssqlRISSxJVEjPTy1WyEgtSgUAXVhZVxUAAAA=' | base64 -d > /tmp/oras-blob-fetch-beef.gz
+  elif [[ "$*" == "blob fetch --registry-config"*"/tmp/oras-blob-fetch-pork.gz"* ]]
+  then
+    echo -n 'H4sIAAAAAAAAA8vNL0pVSEksSQQA2pxWLAkAAAA=' | base64 -d > /tmp/oras-blob-fetch-pork.gz
+  elif [[ "$*" == "manifest fetch --registry-config"*image-with-gzipped-layers* ]]
+  then
+    echo '{"mediaType": "my_media_type", "layers": [{"mediaType": "blob+gzip", "digest": "beef"}, {"mediaType": "blob+gzip", "digest": "pork"}]}'
   elif [[ "$*" == "manifest fetch --registry-config"* ]]
   then
-    echo '{"mediaType": "my_media_type"}'
+    echo '{"mediaType": "my_media_type", "layers": [{"mediaType": "blob+other", "digest": "tofu"}]}'
   elif [[ "$*" == "pull --registry-config"*dockerfile-not-found:sha256-*.dockerfile* ]]
   then
     echo Mock oras called with: $*

tasks/create-pyxis-image/tests/test-create-pyxis-image-dockerfile-not-found.yaml

@@ -20,7 +20,7 @@ spec:
           - name: data
         steps:
           - name: setup-values
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env sh
               set -eux
@@ -70,7 +70,7 @@ spec:
           - name: data
         steps:
           - name: check-result
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env sh
               set -eux

tasks/create-pyxis-image/tests/test-create-pyxis-image-fail-dockerfile-not-pulled.yaml

@@ -22,7 +22,7 @@ spec:
           - name: data
         steps:
           - name: setup-values
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env sh
               set -eux

tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-multi-arch.yaml

@@ -19,7 +19,7 @@ spec:
           - name: data
         steps:
           - name: setup-values
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env bash
               set -eux
@@ -85,7 +85,7 @@ spec:
           - name: data
         steps:
           - name: check-result
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env bash
               set -eux

tasks/create-pyxis-image/tests/test-create-pyxis-image-multi-containerimages-one-arch.yaml

@@ -19,7 +19,7 @@ spec:
           - name: data
         steps:
           - name: setup-values
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env bash
               set -eux
@@ -85,7 +85,7 @@ spec:
           - name: data
         steps:
           - name: check-result
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env bash
               set -eux

tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-multi-arch.yaml

@@ -19,7 +19,7 @@ spec:
           - name: data
         steps:
           - name: setup-values
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env bash
               set -eux
@@ -69,7 +69,7 @@ spec:
           - name: data
         steps:
           - name: check-result
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env bash
               set -eux

tasks/create-pyxis-image/tests/test-create-pyxis-image-one-containerimage-one-arch.yaml

@@ -18,7 +18,7 @@ spec:
           - name: data
         steps:
           - name: setup-values
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env bash
               set -eux
@@ -68,7 +68,7 @@ spec:
           - name: data
         steps:
           - name: check-result
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env bash
               set -eux

tasks/create-pyxis-image/tests/test-create-pyxis-image-rhpush-and-commontag.yaml

@@ -19,7 +19,7 @@ spec:
           - name: data
         steps:
           - name: setup-values
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env bash
               set -eux
@@ -66,7 +66,7 @@ spec:
           - name: data
         steps:
           - name: check-result
-            image: quay.io/konflux-ci/release-service-utils:65d8db844c008e7736ec8dff307868525279484d
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
             script: |
               #!/usr/bin/env bash
               set -eux

tasks/create-pyxis-image/tests/test-create-pyxis-image-with-gzipped-layers.yaml

@@ -0,0 +1,97 @@
+---
+apiVersion: tekton.dev/v1
+kind: Pipeline
+metadata:
+  name: test-create-pyxis-image-with-gzipped-layers
+spec:
+  description: |
+    Run the create-pyxis-image task with an image that has gzipped layers, to
+    see that we reported the uncompressed digests correctly
+  workspaces:
+    - name: tests-workspace
+  tasks:
+    - name: setup
+      workspaces:
+        - name: data
+          workspace: tests-workspace
+      taskSpec:
+        workspaces:
+          - name: data
+        steps:
+          - name: setup-values
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
+            script: |
+              #!/usr/bin/env bash
+              set -eux
+
+              cat > "$(workspaces.data.path)"/mapped_snapshot.json << EOF
+              {
+                "application": "myapp",
+                "components": [
+                  {
+                    "name": "comp",
+                    "containerImage": "source@sha256:mydigest",
+                    "repository": "registry.io/image-with-gzipped-layers",
+                    "tags": [
+                      "testtag"
+                    ]
+                  }
+                ]
+              }
+              EOF
+    - name: run-task
+      taskRef:
+        name: create-pyxis-image
+      params:
+        - name: pyxisSecret
+          value: test-create-pyxis-image-cert
+        - name: server
+          value: stage
+        - name: snapshotPath
+          value: mapped_snapshot.json
+      workspaces:
+        - name: data
+          workspace: tests-workspace
+      runAfter:
+        - setup
+    - name: check-result
+      params:
+        - name: pyxisDataPath
+          value: $(tasks.run-task.results.pyxisDataPath)
+      workspaces:
+        - name: data
+          workspace: tests-workspace
+      taskSpec:
+        params:
+          - name: pyxisDataPath
+            type: string
+        workspaces:
+          - name: data
+        steps:
+          - name: check-result
+            image: quay.io/konflux-ci/release-service-utils:31c0a95a9604fd16b002b4969cfd74753d9c5e77
+            script: |
+              #!/usr/bin/env bash
+              set -eux
+
+              if [ "$(wc -l < "$(workspaces.data.path)"/mock_create_container_image.txt)" != 1 ]; then
+                echo Error: create_container_image was expected to be called 1 time. Actual calls:
+                cat "$(workspaces.data.path)/mock_create_container_image.txt"
+                exit 1
+              fi
+
+              if [ "$(wc -l < "$(workspaces.data.path)"/mock_oras.txt)" != 5 ]; then
+                echo Error: oras was expected to be called 4 times. Actual calls:
+                cat "$(workspaces.data.path)/mock_oras.txt"
+                exit 1
+              fi
+
+              [ "$(head -n 1 < "$(workspaces.data.path)"/mock_skopeo.txt)" \
+                = "inspect --raw docker://registry.io/image-with-gzipped-layers@sha256:mydigest" ]
+
+              # check that the size of the decompressed layers is as expected
+              jq -e '.uncompressed_layers[0].size == 21' "$(workspaces.data.path)/oras-manifest-fetch.json"
+              jq -e '.uncompressed_layers[1].size == 9' "$(workspaces.data.path)/oras-manifest-fetch.json"
+
+      runAfter:
+        - run-task