diff --git a/pipelines/docker-build-multi-platform-oci-ta/README.md b/pipelines/docker-build-multi-platform-oci-ta/README.md index 2ba97a681e..c278bf28ce 100644 --- a/pipelines/docker-build-multi-platform-oci-ta/README.md +++ b/pipelines/docker-build-multi-platform-oci-ta/README.md @@ -142,6 +142,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito ### prefetch-dependencies-oci-ta:0.1 task parameters |name|description|default value|already set by| |---|---|---|---| +|ACTIVATION_KEY| Name of secret which contains subscription activation key| activation-key| | |SOURCE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the application source code.| None| '$(tasks.clone-repository.results.SOURCE_ARTIFACT)'| |caTrustConfigMapKey| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| | |caTrustConfigMapName| The name of the ConfigMap to read CA bundle data from.| trusted-ca| | diff --git a/pipelines/docker-build-oci-ta/README.md b/pipelines/docker-build-oci-ta/README.md index e08c152e2d..1b078292d5 100644 --- a/pipelines/docker-build-oci-ta/README.md +++ b/pipelines/docker-build-oci-ta/README.md @@ -139,6 +139,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito ### prefetch-dependencies-oci-ta:0.1 task parameters |name|description|default value|already set by| |---|---|---|---| +|ACTIVATION_KEY| Name of secret which contains subscription activation key| activation-key| | |SOURCE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the application source code.| None| '$(tasks.clone-repository.results.SOURCE_ARTIFACT)'| |caTrustConfigMapKey| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| | |caTrustConfigMapName| The name of the ConfigMap to read CA bundle data from.| trusted-ca| | diff --git a/pipelines/docker-build/README.md b/pipelines/docker-build/README.md index b7fb2d5e6e..95afc49114 100644 --- a/pipelines/docker-build/README.md +++ b/pipelines/docker-build/README.md @@ -136,6 +136,7 @@ This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/reposito ### prefetch-dependencies:0.1 task parameters |name|description|default value|already set by| |---|---|---|---| +|ACTIVATION_KEY| Name of secret which contains subscription activation key| activation-key| | |caTrustConfigMapKey| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| | |caTrustConfigMapName| The name of the ConfigMap to read CA bundle data from.| trusted-ca| | |config-file-content| Pass configuration to cachi2. Note this needs to be passed as a YAML-formatted config dump, not as a file path! | | | diff --git a/pipelines/tekton-bundle-builder/README.md b/pipelines/tekton-bundle-builder/README.md index cd38c1a6db..7af3d568ce 100644 --- a/pipelines/tekton-bundle-builder/README.md +++ b/pipelines/tekton-bundle-builder/README.md @@ -66,6 +66,7 @@ ### prefetch-dependencies:0.1 task parameters |name|description|default value|already set by| |---|---|---|---| +|ACTIVATION_KEY| Name of secret which contains subscription activation key| activation-key| | |caTrustConfigMapKey| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| | |caTrustConfigMapName| The name of the ConfigMap to read CA bundle data from.| trusted-ca| | |config-file-content| Pass configuration to cachi2. Note this needs to be passed as a YAML-formatted config dump, not as a file path! | | | diff --git a/task/prefetch-dependencies-oci-ta/0.1/README.md b/task/prefetch-dependencies-oci-ta/0.1/README.md index 48065f95ea..73e8ebb5d5 100644 --- a/task/prefetch-dependencies-oci-ta/0.1/README.md +++ b/task/prefetch-dependencies-oci-ta/0.1/README.md @@ -26,6 +26,7 @@ params: ## Parameters |name|description|default value|required| |---|---|---|---| +|ACTIVATION_KEY|Name of secret which contains subscription activation key|activation-key|false| |SOURCE_ARTIFACT|The Trusted Artifact URI pointing to the artifact with the application source code.||true| |caTrustConfigMapKey|The name of the key in the ConfigMap that contains the CA bundle data.|ca-bundle.crt|false| |caTrustConfigMapName|The name of the ConfigMap to read CA bundle data from.|trusted-ca|false| diff --git a/task/prefetch-dependencies-oci-ta/0.1/prefetch-dependencies-oci-ta.yaml b/task/prefetch-dependencies-oci-ta/0.1/prefetch-dependencies-oci-ta.yaml index ecc4b44d93..4ba46d2e65 100644 --- a/task/prefetch-dependencies-oci-ta/0.1/prefetch-dependencies-oci-ta.yaml +++ b/task/prefetch-dependencies-oci-ta/0.1/prefetch-dependencies-oci-ta.yaml @@ -33,6 +33,10 @@ spec: [available configuration parameters]: https://github.com/containerbuildsystem/cachi2?tab=readme-ov-file#available-configuration-parameters params: + - name: ACTIVATION_KEY + description: Name of secret which contains subscription activation key + type: string + default: activation-key - name: SOURCE_ARTIFACT description: The Trusted Artifact URI pointing to the artifact with the application source code. @@ -79,8 +83,16 @@ spec: the application source code. type: string volumes: + - name: activation-key + secret: + optional: true + secretName: $(params.ACTIVATION_KEY) - name: config emptyDir: {} + - name: etc-pki-entitlement + emptyDir: {} + - name: shared + emptyDir: {} - name: trusted-ca configMap: items: @@ -110,6 +122,8 @@ spec: volumeMounts: - mountPath: /mnt/config name: config + - mountPath: /shared + name: shared - mountPath: /var/workdir name: workdir steps: @@ -143,15 +157,159 @@ spec: # https://github.com/containerbuildsystem/cachi2/issues/577 yq 'del(.goproxy_url)' <<<"${CONFIG_FILE_CONTENT}" >/mnt/config/config.yaml fi + - name: check-prefetch-input + image: quay.io/redhat-appstudio/cachi2:0.15.0@sha256:b141cb5cf4d98e6c5f668f1fe172e1d68f2a44ac1027403fbcff94ce1e68185d + env: + - name: INPUT + value: $(params.input) + script: | + if [ -z "${INPUT}" ]; then + # Confirm input was provided though it's likely the whole task would be skipped if it wasn't + echo "No prefetch will be performed because no input was provided for cachi2 fetch-deps" + echo "skip" >/shared/skip + fi + - name: register-red-hat + image: quay.io/redhat-appstudio/cachi2:0.15.0@sha256:b141cb5cf4d98e6c5f668f1fe172e1d68f2a44ac1027403fbcff94ce1e68185d + results: + - name: registered + type: string + volumeMounts: + - mountPath: /activation-key + name: activation-key + env: + - name: INPUT + value: $(params.input) + - name: ACTIVATION_KEY + value: $(params.ACTIVATION_KEY) + script: | + #!/bin/bash + if [ -f /shared/skip ]; then + echo "Skipping." + exit 0 + fi + + echo "false" >/shared/registered + ACTIVATION_KEY_PATH="/activation-key" + + mkdir -p /shared/rhsm/entitlement + mkdir -p /shared/rhsm/consumer + + if [ -e /activation-key/org ]; then + cp -r --preserve=mode "$ACTIVATION_KEY_PATH" /tmp/activation-key + + echo "Registering with Red Hat subscription manager." + subscription-manager register --org "$(cat /tmp/activation-key/org)" --activationkey "$(cat /tmp/activation-key/activationkey)" + + # copy generated certificates to /shared/rhsm + cp /etc/pki/entitlement/*.pem /shared/rhsm/entitlement/ + cp /etc/pki/consumer/*.pem /shared/rhsm/consumer/ + + file="$(find /shared/rhsm/entitlement -regextype egrep -regex '.*[0-9]+\.pem' -printf %f)" + echo "file: $file" + basename "$file" .pem >/shared/RHSM_ID + echo "./RHSM_ID:" + cat /shared/RHSM_ID + + # trust the CA used for Red Hat CDN + cp /etc/rhsm-host/ca/redhat-uep.pem /shared/rhsm/redhat-uep.pem + fi + - name: preprocess-input + image: quay.io/redhat-appstudio/cachi2:0.15.0@sha256:b141cb5cf4d98e6c5f668f1fe172e1d68f2a44ac1027403fbcff94ce1e68185d + args: + - $(params.input) + env: + - name: INPUT + value: $(params.input) + - name: ACTIVATION_KEY + value: $(params.ACTIVATION_KEY) + script: | + #!/bin/python3 + import json + import os + import sys + + + def string_to_json(input: str): + if input in ['bundler', 'generic', 'gomod', 'npm', 'pip', 'rpm', 'yarn-classic', 'yarn']: + input = '{"type": "%s"}' % input + print("json: %s" % input) + return input + + + def json_to_list(input: str): + input = json.loads(input) + if type(input) is dict: + input = [input] + return json.dumps(input) + + + def inject_certs(input: str, rhsm_id: str): + input_list: list = json.loads(input) + + cert = ("/shared/rhsm/entitlement/%s.pem" % rhsm_id) + key = ("/shared/rhsm/entitlement/%s-key.pem" % rhsm_id) + ca_bundle = os.getenv("CA_BUNDLE", None) + for pkg_man in input_list: + if pkg_man["type"] == "rpm": + + # preserve verify setting + verify = \ + pkg_man.get("options", {}).get("ssl", {}).get("ssl_verify", 1) + + # preserve other options + options: dict = pkg_man.get('options', {}) + + ssl_options = { + "client_key": key, + "client_cert": cert, + "ca_bundle": ca_bundle, + "ssl_verify": verify} + + options['ssl'] = ssl_options + pkg_man["options"] = options + return (json.dumps(input_list)) + + + def convert_input(input, rhsm_id): + input = string_to_json(input) + input = json_to_list(input) + input = inject_certs(input, rhsm_id) + return input + + + if __name__ == '__main__': + + if os.path.isfile("/shared/skip"): + sys.exit() + + rhsm_id = "" + input = "" + + try: + f = open("/shared/RHSM_ID", "r") + rhsm_id = f.read().strip("\n") + except FileNotFoundError: + print("No RHSM ID found.") + + if rhsm_id == "": + input = sys.argv[1] + else: + print("RHSM Cert ID is: %s" % rhsm_id) + print("Called with args: %s" % str(sys.argv)) + input = convert_input(sys.argv[1], rhsm_id) + + print("Preprocessing result: %s" % input) + with open('/shared/rhsm/preprocessed_input', 'w') as f: + f.write(input) - name: prefetch-dependencies image: quay.io/redhat-appstudio/cachi2:0.15.0@sha256:b141cb5cf4d98e6c5f668f1fe172e1d68f2a44ac1027403fbcff94ce1e68185d volumeMounts: - mountPath: /mnt/trusted-ca name: trusted-ca readOnly: true + - mountPath: /activation-key + name: activation-key env: - - name: INPUT - value: $(params.input) - name: DEV_PACKAGE_MANAGERS value: $(params.dev-package-managers) - name: LOG_LEVEL @@ -165,9 +323,10 @@ spec: - name: WORKSPACE_NETRC_PATH value: $(workspaces.netrc.path) script: | - if [ -z "${INPUT}" ]; then - # Confirm input was provided though it's likely the whole task would be skipped if it wasn't - echo "No prefetch will be performed because no input was provided for cachi2 fetch-deps" + #!/bin/bash + + if [ -f /shared/skip ]; then + echo "Skipping." exit 0 fi @@ -183,6 +342,16 @@ spec: dev_pacman_flag="" fi + INPUT=$(cat /shared/rhsm/preprocessed_input) + export INPUT + + # trust Red Hat CA cert used for Red Hat CDN + if [ -f /shared/rhsm/redhat-uep.pem ]; then + echo "Adding Red Hat CA certificate to trusted roots." + cp /shared/rhsm/redhat-uep.pem /etc/pki/ca-trust/source/anchors/ + update-ca-trust + fi + # Copied from https://github.com/konflux-ci/build-definitions/blob/main/task/git-clone/0.1/git-clone.yaml if [ "${WORKSPACE_GIT_AUTH_BOUND}" = "true" ]; then if [ -f "${WORKSPACE_GIT_AUTH_PATH}/.git-credentials" ] && [ -f "${WORKSPACE_GIT_AUTH_PATH}/.gitconfig" ]; then @@ -225,6 +394,18 @@ spec: cachi2 --log-level="$LOG_LEVEL" inject-files /var/workdir/cachi2/output \ --for-output-dir=/cachi2/output + - name: unregister-rhsm + image: quay.io/redhat-appstudio/cachi2:0.15.0@sha256:b141cb5cf4d98e6c5f668f1fe172e1d68f2a44ac1027403fbcff94ce1e68185d + script: | + #!/bin/bash + if [ -f /shared/skip ]; then + echo "Skipping." + exit 0 + fi + + cp /shared/rhsm/consumer/* /etc/pki/consumer/ + cp /shared/rhsm/entitlement/* /etc/pki/entitlement/ + subscription-manager unregister || true - name: create-trusted-artifact image: quay.io/redhat-appstudio/build-trusted-artifacts:latest@sha256:81c4864dae6bb11595f657be887e205262e70086a05ed16ada827fd6391926ac args: diff --git a/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml b/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml index 1b804860bd..9a859d42de 100644 --- a/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml +++ b/task/prefetch-dependencies/0.1/prefetch-dependencies.yaml @@ -53,6 +53,10 @@ spec: type: string description: The name of the key in the ConfigMap that contains the CA bundle data. default: ca-bundle.crt + - name: ACTIVATION_KEY + default: activation-key + description: Name of secret which contains subscription activation key + type: string stepTemplate: env: @@ -61,7 +65,8 @@ spec: volumeMounts: - name: config mountPath: /mnt/config - + - mountPath: /shared + name: shared steps: - name: sanitize-cachi2-config-file-with-yq image: quay.io/konflux-ci/yq:latest@sha256:343c2ca0a347ae87fe43750ee0873e1fe813f77eff56e9722c840bb75d97fef2 @@ -74,13 +79,164 @@ spec: yq 'del(.goproxy_url)' <<< "${CONFIG_FILE_CONTENT}" > /mnt/config/config.yaml fi - - image: quay.io/redhat-appstudio/cachi2:0.15.0@sha256:b141cb5cf4d98e6c5f668f1fe172e1d68f2a44ac1027403fbcff94ce1e68185d + - name: check-prefetch-input + image: quay.io/redhat-appstudio/cachi2:0.15.0@sha256:b141cb5cf4d98e6c5f668f1fe172e1d68f2a44ac1027403fbcff94ce1e68185d # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting # the cluster will set imagePullPolicy to IfNotPresent - name: prefetch-dependencies env: - name: INPUT value: $(params.input) + script: | + if [ -z "${INPUT}" ] + then + # Confirm input was provided though it's likely the whole task would be skipped if it wasn't + echo "No prefetch will be performed because no input was provided for cachi2 fetch-deps" + echo "skip" > /shared/skip + fi + + - name: register-red-hat + image: quay.io/redhat-appstudio/cachi2:0.15.0@sha256:b141cb5cf4d98e6c5f668f1fe172e1d68f2a44ac1027403fbcff94ce1e68185d + env: + - name: INPUT + value: $(params.input) + - name: ACTIVATION_KEY + value: $(params.ACTIVATION_KEY) + volumeMounts: + - mountPath: /activation-key + name: activation-key + results: + - name: registered + type: string + script: | + #!/bin/bash + if [ -f /shared/skip ]; then + echo "Skipping." + exit 0 + fi + + echo "false" > /shared/registered + ACTIVATION_KEY_PATH="/activation-key" + + mkdir -p /shared/rhsm/entitlement + mkdir -p /shared/rhsm/consumer + + if [ -e /activation-key/org ]; then + cp -r --preserve=mode "$ACTIVATION_KEY_PATH" /tmp/activation-key + + echo "Registering with Red Hat subscription manager." + subscription-manager register --org "$(cat /tmp/activation-key/org)" --activationkey "$(cat /tmp/activation-key/activationkey)" + + # copy generated certificates to /shared/rhsm + cp /etc/pki/entitlement/*.pem /shared/rhsm/entitlement/ + cp /etc/pki/consumer/*.pem /shared/rhsm/consumer/ + + file="$(find /shared/rhsm/entitlement -regextype egrep -regex '.*[0-9]+\.pem' -printf %f)" + echo "file: $file" + basename "$file" .pem > /shared/RHSM_ID + echo "./RHSM_ID:" + cat /shared/RHSM_ID + + # trust the CA used for Red Hat CDN + cp /etc/rhsm-host/ca/redhat-uep.pem /shared/rhsm/redhat-uep.pem + fi + + - name: preprocess-input + image: quay.io/redhat-appstudio/cachi2:0.15.0@sha256:b141cb5cf4d98e6c5f668f1fe172e1d68f2a44ac1027403fbcff94ce1e68185d + # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting + # the cluster will set imagePullPolicy to IfNotPresent + + env: + - name: INPUT + value: $(params.input) + - name: ACTIVATION_KEY + value: $(params.ACTIVATION_KEY) + args: ["$(params.input)"] + script: | + #!/bin/python3 + import json + import os + import sys + + + def string_to_json(input: str): + if input in ['bundler', 'generic', 'gomod', 'npm', 'pip', 'rpm', 'yarn-classic', 'yarn']: + input = '{"type": "%s"}' % input + print("json: %s" % input) + return input + + + def json_to_list(input: str): + input = json.loads(input) + if type(input) is dict: + input = [input] + return json.dumps(input) + + + def inject_certs(input: str, rhsm_id: str): + input_list: list = json.loads(input) + + cert = ("/shared/rhsm/entitlement/%s.pem" % rhsm_id) + key = ("/shared/rhsm/entitlement/%s-key.pem" % rhsm_id) + ca_bundle = os.getenv("CA_BUNDLE", None) + for pkg_man in input_list: + if pkg_man["type"] == "rpm": + + # preserve verify setting + verify = \ + pkg_man.get("options", {}).get("ssl", {}).get("ssl_verify", 1) + + # preserve other options + options: dict = pkg_man.get('options', {}) + + ssl_options = { + "client_key": key, + "client_cert": cert, + "ca_bundle": ca_bundle, + "ssl_verify": verify} + + options['ssl'] = ssl_options + pkg_man["options"] = options + return (json.dumps(input_list)) + + + def convert_input(input, rhsm_id): + input = string_to_json(input) + input = json_to_list(input) + input = inject_certs(input, rhsm_id) + return input + + + if __name__ == '__main__': + + if os.path.isfile("/shared/skip"): + sys.exit() + + rhsm_id = "" + input = "" + + try: + f = open("/shared/RHSM_ID", "r") + rhsm_id = f.read().strip("\n") + except FileNotFoundError: + print("No RHSM ID found.") + + if rhsm_id == "": + input = sys.argv[1] + else: + print("RHSM Cert ID is: %s" % rhsm_id) + print("Called with args: %s" % str(sys.argv)) + input = convert_input(sys.argv[1], rhsm_id) + + print("Preprocessing result: %s" % input) + with open('/shared/rhsm/preprocessed_input', 'w') as f: + f.write(input) + + + - name: prefetch-dependencies + image: quay.io/redhat-appstudio/cachi2:0.15.0@sha256:b141cb5cf4d98e6c5f668f1fe172e1d68f2a44ac1027403fbcff94ce1e68185d + # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting + # the cluster will set imagePullPolicy to IfNotPresent + env: - name: DEV_PACKAGE_MANAGERS value: $(params.dev-package-managers) - name: LOG_LEVEL @@ -97,11 +253,13 @@ spec: - name: trusted-ca mountPath: /mnt/trusted-ca readOnly: true + - mountPath: /activation-key + name: activation-key script: | - if [ -z "${INPUT}" ] - then - # Confirm input was provided though it's likely the whole task would be skipped if it wasn't - echo "No prefetch will be performed because no input was provided for cachi2 fetch-deps" + #!/bin/bash + + if [ -f /shared/skip ]; then + echo "Skipping." exit 0 fi @@ -117,6 +275,16 @@ spec: dev_pacman_flag="" fi + INPUT=$(cat /shared/rhsm/preprocessed_input) + export INPUT + + # trust Red Hat CA cert used for Red Hat CDN + if [ -f /shared/rhsm/redhat-uep.pem ]; then + echo "Adding Red Hat CA certificate to trusted roots." + cp /shared/rhsm/redhat-uep.pem /etc/pki/ca-trust/source/anchors/ + update-ca-trust + fi + # Copied from https://github.com/konflux-ci/build-definitions/blob/main/task/git-clone/0.1/git-clone.yaml if [ "${WORKSPACE_GIT_AUTH_BOUND}" = "true" ] ; then if [ -f "${WORKSPACE_GIT_AUTH_PATH}/.git-credentials" ] && [ -f "${WORKSPACE_GIT_AUTH_PATH}/.gitconfig" ]; then @@ -159,6 +327,22 @@ spec: cachi2 --log-level="$LOG_LEVEL" inject-files $(workspaces.source.path)/cachi2/output \ --for-output-dir=/cachi2/output + + - name: unregister-rhsm + image: quay.io/redhat-appstudio/cachi2:0.15.0@sha256:b141cb5cf4d98e6c5f668f1fe172e1d68f2a44ac1027403fbcff94ce1e68185d + # per https://kubernetes.io/docs/concepts/containers/images/#imagepullpolicy-defaulting + # the cluster will set imagePullPolicy to IfNotPresent + script: | + #!/bin/bash + if [ -f /shared/skip ]; then + echo "Skipping." + exit 0 + fi + + cp /shared/rhsm/consumer/* /etc/pki/consumer/ + cp /shared/rhsm/entitlement/* /etc/pki/entitlement/ + subscription-manager unregister || true + workspaces: - name: source description: Workspace with the source code, cachi2 artifacts will be stored on the workspace as well @@ -175,6 +359,14 @@ spec: performing http(s) requests. optional: true volumes: + - name: shared + emptyDir: {} + - name: etc-pki-entitlement + emptyDir: {} + - name: activation-key + secret: + optional: true + secretName: $(params.ACTIVATION_KEY) - name: trusted-ca configMap: name: $(params.caTrustConfigMapName)