chore: fix the result of witness_provenance_l1_check in case no witness provenance discovered

Signed-off-by: Nathan Nguyen <[email protected]>

Author: nathanwn

src/macaron/slsa_analyzer/checks/provenance_witness_l1_check.py

@@ -169,12 +169,18 @@ def run_check(self, ctx: AnalyzeContext, check_result: CheckResult) -> CheckResu
 
                         verified_artifact_assets.extend(artifact_assets)
 
-        check_result["justification"].append("Successfully verified the following artifacts:")
-        for asset in verified_artifact_assets:
-            check_result["justification"].append(f"* {asset.url}")
-
-        check_result["result_tables"].append(ProvenanceWitnessL1Table())
-        return CheckResultType.PASSED
+        # If Macaron cannot discover any witness provenance, we "fail" the check.
+        # Here, there status ``FAILED`` means: Macaron fails to discover any witness provenance.
+        # This is consistent with how Souffle works: facts in Souffle usually represent things that exist.
+        if len(verified_artifact_assets) > 0:
+            check_result["justification"].append("Successfully verified the following artifacts:")
+            for asset in verified_artifact_assets:
+                check_result["justification"].append(f"* {asset.url}")
+            check_result["result_tables"].append(ProvenanceWitnessL1Table())
+            return CheckResultType.PASSED
+
+        check_result["justification"].append("Failed to discover any witness provenance.")
+        return CheckResultType.FAILED
 
 
 registry.register(ProvenanceWitnessL1Check())

tests/e2e/expected_results/micronaut-core/micronaut-core.json

@@ -752,8 +752,8 @@
         "checks": {
             "summary": {
                 "DISABLED": 0,
-                "FAILED": 1,
-                "PASSED": 7,
+                "FAILED": 2,
+                "PASSED": 6,
                 "SKIPPED": 0,
                 "UNKNOWN": 1
             },
@@ -846,8 +846,10 @@
                         "Provenance content - Identifies artifacts - SLSA Level 1",
                         "Provenance content - Identifies builder - SLSA Level 1"
                     ],
-                    "justification": [],
-                    "result_type": "PASSED"
+                    "justification": [
+                        "Failed to discover any witness provenance."
+                    ],
+                    "result_type": "FAILED"
                 },
                 {
                     "check_id": "mcn_version_control_system_1",

tests/e2e/expected_results/slsa-verifier/slsa-verifier_cue_PASS.json

@@ -1683,8 +1683,8 @@
         "checks": {
             "summary": {
                 "DISABLED": 0,
-                "FAILED": 0,
-                "PASSED": 9,
+                "FAILED": 1,
+                "PASSED": 8,
                 "SKIPPED": 0,
                 "UNKNOWN": 0
             },
@@ -1777,8 +1777,10 @@
                         "Provenance content - Identifies artifacts - SLSA Level 1",
                         "Provenance content - Identifies builder - SLSA Level 1"
                     ],
-                    "justification": [],
-                    "result_type": "PASSED"
+                    "justification": [
+                        "Failed to discover any witness provenance."
+                    ],
+                    "result_type": "FAILED"
                 },
                 {
                     "check_id": "mcn_trusted_builder_level_three_1",

tests/e2e/expected_results/urllib3/urllib3.json

@@ -271,8 +271,8 @@
         "checks": {
             "summary": {
                 "DISABLED": 0,
-                "FAILED": 1,
-                "PASSED": 8,
+                "FAILED": 2,
+                "PASSED": 7,
                 "SKIPPED": 0,
                 "UNKNOWN": 0
             },
@@ -365,8 +365,10 @@
                         "Provenance content - Identifies artifacts - SLSA Level 1",
                         "Provenance content - Identifies builder - SLSA Level 1"
                     ],
-                    "justification": [],
-                    "result_type": "PASSED"
+                    "justification": [
+                        "Failed to discover any witness provenance."
+                    ],
+                    "result_type": "FAILED"
                 },
                 {
                     "check_id": "mcn_version_control_system_1",

tests/e2e/expected_results/urllib3/urllib3_cue_invalid.json

@@ -271,8 +271,8 @@
         "checks": {
             "summary": {
                 "DISABLED": 0,
-                "FAILED": 1,
-                "PASSED": 7,
+                "FAILED": 2,
+                "PASSED": 6,
                 "SKIPPED": 0,
                 "UNKNOWN": 1
             },
@@ -365,8 +365,10 @@
                         "Provenance content - Identifies artifacts - SLSA Level 1",
                         "Provenance content - Identifies builder - SLSA Level 1"
                     ],
-                    "justification": [],
-                    "result_type": "PASSED"
+                    "justification": [
+                        "Failed to discover any witness provenance."
+                    ],
+                    "result_type": "FAILED"
                 },
                 {
                     "check_id": "mcn_version_control_system_1",