Merge pull request #2049 from lcarva/EC-894

Add support for setting severity dynamically

acceptance/examples/dynamic_severity.rego

@@ -0,0 +1,31 @@
+package main
+
+import rego.v1
+
+deny contains result if {
+    result := {
+        "msg": "Failure to warning",
+        "severity": "warning"
+    }
+}
+
+deny contains result if {
+    result := {
+        "msg": "Failure to failure",
+        "severity": "failure"
+    }
+}
+
+warn contains result if {
+    result := {
+        "msg": "Warning to failure",
+        "severity": "failure"
+    }
+}
+
+warn contains result if {
+    result := {
+        "msg": "Warning to warning",
+        "severity": "warning"
+    }
+}

features/__snapshots__/validate_image.snap

@@ -4882,3 +4882,109 @@ Error: success criteria not met
 [fetch OCI image files:stderr - 1]
 
 ---
+
+[severity is dynamically adjusted:stdout - 1]
+{
+  "success": false,
+  "components": [
+    {
+      "name": "Unnamed",
+      "containerImage": "${REGISTRY}/acceptance/ec-happy-day@sha256:${REGISTRY_acceptance/ec-happy-day:latest_DIGEST}",
+      "source": {},
+      "violations": [
+        {
+          "msg": "Failure to failure",
+          "metadata": {
+            "severity": "failure"
+          }
+        },
+        {
+          "msg": "Warning to failure",
+          "metadata": {
+            "severity": "failure"
+          }
+        }
+      ],
+      "warnings": [
+        {
+          "msg": "Failure to warning",
+          "metadata": {
+            "severity": "warning"
+          }
+        },
+        {
+          "msg": "Warning to warning",
+          "metadata": {
+            "severity": "warning"
+          }
+        }
+      ],
+      "successes": [
+        {
+          "msg": "Pass",
+          "metadata": {
+            "code": "builtin.attestation.signature_check",
+            "description": "The attestation signature matches available signing materials.",
+            "title": "Attestation signature check passed"
+          }
+        },
+        {
+          "msg": "Pass",
+          "metadata": {
+            "code": "builtin.attestation.syntax_check",
+            "description": "The attestation has correct syntax.",
+            "title": "Attestation syntax check passed"
+          }
+        },
+        {
+          "msg": "Pass",
+          "metadata": {
+            "code": "builtin.image.signature_check",
+            "description": "The image signature matches available signing materials.",
+            "title": "Image signature check passed"
+          }
+        }
+      ],
+      "success": false,
+      "signatures": [
+        {
+          "keyid": "",
+          "sig": "${IMAGE_SIGNATURE_acceptance/ec-happy-day}"
+        }
+      ],
+      "attestations": [
+        {
+          "type": "https://in-toto.io/Statement/v0.1",
+          "predicateType": "https://slsa.dev/provenance/v0.2",
+          "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2",
+          "signatures": [
+            {
+              "keyid": "",
+              "sig": "${ATTESTATION_SIGNATURE_acceptance/ec-happy-day}"
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "key": "${known_PUBLIC_KEY_JSON}",
+  "policy": {
+    "sources": [
+      {
+        "policy": [
+          "git::https://${GITHOST}/git/dynamic-severity-policy.git?ref=${LATEST_COMMIT}"
+        ]
+      }
+    ],
+    "rekorUrl": "${REKOR}",
+    "publicKey": "${known_PUBLIC_KEY}"
+  },
+  "ec-version": "${EC_VERSION}",
+  "effective-time": "${TIMESTAMP}"
+}
+---
+
+[severity is dynamically adjusted:stderr - 1]
+Error: success criteria not met
+
+---

features/validate_image.feature

@@ -209,6 +209,19 @@ Feature: evaluate enterprise contract
     Then the exit status should be 1
     Then the output should match the snapshot
 
+  Scenario: severity is dynamically adjusted
+    Given a key pair named "known"
+    Given an image named "acceptance/ec-happy-day"
+    Given a valid image signature of "acceptance/ec-happy-day" image signed by the "known" key
+    Given a valid Rekor entry for image signature of "acceptance/ec-happy-day"
+    Given a valid attestation of "acceptance/ec-happy-day" signed by the "known" key
+    Given a valid Rekor entry for attestation of "acceptance/ec-happy-day"
+    Given a git repository named "dynamic-severity-policy" with
+      | main.rego | examples/dynamic_severity.rego |
+    When ec command is run with "validate image --image ${REGISTRY}/acceptance/ec-happy-day --policy {"sources":[{"policy":["git::https://${GITHOST}/git/dynamic-severity-policy.git"]}]} --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json --info"
+    Then the exit status should be 1
+    Then the output should match the snapshot
+
   Scenario: multiple policy sources with multiple source groups
     Given a key pair named "known"
     Given an image named "acceptance/ec-multiple-sources"

internal/evaluator/conftest_evaluator.go

@@ -171,12 +171,18 @@ const (
 	metadataCollections = "collections"
 	metadataDependsOn   = "depends_on"
 	metadataDescription = "description"
+	metadataSeverity    = "severity"
 	metadataEffectiveOn = "effective_on"
 	metadataSolution    = "solution"
 	metadataTerm        = "term"
 	metadataTitle       = "title"
 )
 
+const (
+	severityWarning = "warning"
+	severityFailure = "failure"
+)
+
 // ConfigProvider is a subset of the policy.Policy interface. Its purpose is to codify which parts
 // of Policy are actually used and to make it easier to use mock in tests.
 type ConfigProvider interface {
@@ -464,7 +470,12 @@ func (c conftestEvaluator) Evaluate(ctx context.Context, target EvaluationTarget
 				log.Debugf("Skipping result warning: %#v", warning)
 				continue
 			}
-			warnings = append(warnings, warning)
+
+			if getSeverity(warning) == severityFailure {
+				failures = append(failures, warning)
+			} else {
+				warnings = append(warnings, warning)
+			}
 		}
 
 		for i := range result.Failures {
@@ -476,8 +487,7 @@ func (c conftestEvaluator) Evaluate(ctx context.Context, target EvaluationTarget
 				continue
 			}
 
-			if !isResultEffective(failure, effectiveTime) {
-				// TODO: Instead of moving to warnings, create new attribute: "futureViolations"
+			if getSeverity(failure) == severityWarning || !isResultEffective(failure, effectiveTime) {
 				warnings = append(warnings, failure)
 			} else {
 				failures = append(failures, failure)
@@ -502,7 +512,7 @@ func (c conftestEvaluator) Evaluate(ctx context.Context, target EvaluationTarget
 		result.Skipped = skipped
 
 		// Replace the placeholder successes slice with the actual successes.
-		result.Successes = c.computeSuccesses(result, rules, effectiveTime, target.Target)
+		result.Successes = c.computeSuccesses(result, rules, target.Target)
 
 		totalRules += len(result.Warnings) + len(result.Failures) + len(result.Successes)
 
@@ -537,7 +547,7 @@ func toRules(results []output.Result) []Result {
 // computeSuccesses generates success results, these are not provided in the
 // Conftest results, so we reconstruct these from the parsed rules, any rule
 // that hasn't been touched by adding metadata must have succeeded
-func (c conftestEvaluator) computeSuccesses(result Outcome, rules policyRules, effectiveTime time.Time, target string) []Result {
+func (c conftestEvaluator) computeSuccesses(result Outcome, rules policyRules, target string) []Result {
 	// what rules, by code, have we seen in the Conftest results, use map to
 	// take advantage of hashing for quicker lookup
 	seenRules := map[string]bool{}
@@ -647,6 +657,9 @@ func addMetadataToResults(ctx context.Context, r *Result, rule rule.Info) {
 	if rule.EffectiveOn != "" {
 		r.Metadata[metadataEffectiveOn] = rule.EffectiveOn
 	}
+	if rule.Severity != "" {
+		r.Metadata[metadataSeverity] = rule.Severity
+	}
 	if rule.Description != "" {
 		r.Metadata[metadataDescription] = rule.Description
 	}
@@ -779,6 +792,26 @@ func (c *conftestEvaluator) createCapabilitiesFile(ctx context.Context) error {
 	return nil
 }
 
+func getSeverity(r Result) string {
+	raw, found := r.Metadata[metadataSeverity]
+	if !found {
+		return ""
+	}
+	severity, ok := raw.(string)
+	if !ok {
+		log.Warnf("Ignoring non-string %q value %#v", metadataSeverity, raw)
+		return ""
+	}
+
+	switch severity {
+	case severityFailure, severityWarning:
+		return severity
+	default:
+		log.Warnf("Ignoring unexpected %q value %s", metadataSeverity, severity)
+		return ""
+	}
+}
+
 // isResultEffective returns whether or not the given result's effective date is before now.
 // Failure to determine the effective date is reported as the result being effective.
 func isResultEffective(failure Result, now time.Time) bool {