Test with setup-sigstore-env Action

.github/workflows/test-with-setup-sigstore-env.yml

@@ -0,0 +1,44 @@
+name: test-with-setup-sigstore-env
+
+on:
+  push:
+    branches:
+      - main
+      - series/*
+  pull_request:
+  schedule:
+    - cron: '0 12 * * *'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to access the workflow's OIDC identity.
+      id-token: write
+    steps:
+        # TODO: use new release
+        - id: setup-sigstore-env
+          uses: sigstore/scaffolding/actions/setup-sigstore-env@c1697866accfa0fbe21caf7e16ab85f236695428  # main
+        - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+          with:
+            persist-credentials: false
+        - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+          with:
+            # use the version specified by this project.
+            python-version-file: pyproject.toml
+            allow-prereleases: true
+            cache: "pip"
+            cache-dependency-path: pyproject.toml
+        - run: |
+            export TEST_SETUP_SIGSTORE_ENV="any non-empty value"
+            export TRUST_CONFIG=${{ steps.setup-sigstore-env.outputs.trust-config }}
+            export SIGSTORE_IDENTITY_TOKEN_local=$( cat ${{ steps.setup-sigstore-env.outputs.oidc-token }} )
+            export TEST_SIGSTORE_TIMESTAMP_AUTHORITY_URL="${{ steps.setup-sigstore-env.outputs.tsa-url }}/api/v1/timestamp"
+
+            set -o pipefail
+            make test TEST_ARGS=" -rs -vv --showlocals" | tee output
+            ! grep -q "skipping test that use the local environment" output || (echo "ERROR: Found skip message" && exit 1)
+
+            make test TEST_ARGS="-m timestamp_authority -rs -vv --showlocals" | tee output
+            ! grep -q "skipping test that use the local environment" output || (echo "ERROR: Found skip message" && exit 1)
+            ! grep -q "skipping test that requires a Timestamp Authority" output || (echo "ERROR: Found skip message" && exit 1)

test/unit/conftest.py

@@ -48,6 +48,16 @@
 assert _TUF_ASSETS.is_dir()
 
 TEST_CLIENT_ID = "sigstore"
+LOCAL = "local"
+
+
+def _has_setup_sigstore_env() -> bool:
+    """
+    Checks whether the TEST_SETUP_SIGSTORE_ENV variable is set to true,
+    This means we are using the sigstore/scaffolding/actions/setup-sigstore-env
+    that has the sigstore services in containers available for us to use.
+    """
+    return bool(os.getenv("TEST_SETUP_SIGSTORE_ENV", False))
 
 
 @pytest.fixture
@@ -194,6 +204,8 @@ def sign_ctx_and_ident_for_env(
     """
     Returns a SigningContext and IdentityToken for the given environment.
     The SigningContext is behind a callable so that it may be lazily evaluated.
+
+    The local tests require setup by the test-with-setup-sigstore-env.yml workflow.
     """
     if env == "staging":
 
@@ -216,20 +228,45 @@ def ctx_cls():
     return ctx_cls, IdentityToken(token)
 
 
-@pytest.fixture
-def staging() -> tuple[type[SigningContext], type[Verifier], IdentityToken]:
+@pytest.fixture(
+    params=[
+        pytest.param(
+            (ClientTrustConfig.staging, os.getenv("SIGSTORE_IDENTITY_TOKEN_staging")),
+            id="preprod-staging",
+        ),
+        pytest.param(
+            (
+                lambda: ClientTrustConfig.from_json(
+                    Path(os.getenv("TRUST_CONFIG")).read_text()
+                ),
+                os.getenv("SIGSTORE_IDENTITY_TOKEN_local"),
+            ),
+            id="preprod-local",
+            marks=pytest.mark.skipif(
+                not _has_setup_sigstore_env(),
+                reason="skipping test that use the local environment due to unset `TEST_SETUP_SIGSTORE_ENV` env variable",
+            ),
+        ),
+    ]
+)
+def preprod(request) -> tuple[type[SigningContext], type[Verifier], IdentityToken]:
     """
     Returns a SigningContext, Verifier, and IdentityToken for the staging environment.
     The SigningContext and Verifier are both behind callables so that they may be lazily evaluated.
+
+    We paramaterize this fixture so that consuming tests can run multiple times, once for each of
+    the params. https://docs.pytest.org/en/stable/how-to/fixtures.html#fixture-parametrize
     """
+    trust_config_func, token = request.param
+    ctx = SigningContext.from_trust_config(trust_config_func())
 
     def signer():
-        return SigningContext.from_trust_config(ClientTrustConfig.staging())
+        return ctx
 
-    verifier = Verifier.staging
+    def verifier():
+        return Verifier(trusted_root=ctx._trusted_root)
 
     # Detect env variable for local interactive tests.
-    token = os.getenv("SIGSTORE_IDENTITY_TOKEN_staging")
     if not token:
         # If the variable is not defined, try getting an ambient token.
         token = detect_credential(TEST_CLIENT_ID)

test/unit/internal/rekor/test_client_v2.py

@@ -28,10 +28,10 @@
 
 @pytest.mark.staging
 @pytest.mark.ambient_oidc
-def test_rekor_v2_create_entry_dsse(staging):
+def test_rekor_v2_create_entry_dsse(preprod):
     # This is not a real unit test: it requires not only staging rekor but also TUF
     # fulcio and oidc -- maybe useful only until we have real integration tests in place
-    sign_ctx_cls, _, identity = staging
+    sign_ctx_cls, _, identity = preprod
 
     # Hack to run Signer.sign() with staging rekor v2
     sign_ctx = sign_ctx_cls()
@@ -64,10 +64,10 @@ def test_rekor_v2_create_entry_dsse(staging):
 
 @pytest.mark.staging
 @pytest.mark.ambient_oidc
-def test_rekor_v2_create_entry_hashed_rekord(staging):
+def test_rekor_v2_create_entry_hashed_rekord(preprod):
     # This is not a real unit test: it requires not only staging rekor but also TUF
     # fulcio and oidc -- maybe useful only until we have real integration tests in place
-    sign_ctx_cls, _, identity = staging
+    sign_ctx_cls, _, identity = preprod
 
     # Hack to run Signer.sign() with staging rekor v2
     sign_ctx = sign_ctx_cls()

test/unit/test_sign.py

@@ -21,7 +21,6 @@
 
 import sigstore.oidc
 from sigstore._internal.timestamp import TimestampAuthorityClient
-from sigstore._internal.trust import ClientTrustConfig
 from sigstore.dsse import StatementBuilder, Subject
 from sigstore.errors import VerificationError
 from sigstore.hashes import Hashed
@@ -115,8 +114,8 @@ def test_identity_proof_claim_lookup(sign_ctx_and_ident_for_env, monkeypatch):
 
 @pytest.mark.staging
 @pytest.mark.ambient_oidc
-def test_sign_prehashed(staging):
-    sign_ctx_cls, verifier_cls, identity = staging
+def test_sign_prehashed(preprod):
+    sign_ctx_cls, verifier_cls, identity = preprod
 
     sign_ctx = sign_ctx_cls()
     verifier = verifier_cls()
@@ -140,8 +139,8 @@ def test_sign_prehashed(staging):
 
 @pytest.mark.staging
 @pytest.mark.ambient_oidc
-def test_sign_dsse(staging):
-    sign_ctx, _, identity = staging
+def test_sign_dsse(preprod):
+    sign_ctx, _, identity = preprod
 
     ctx = sign_ctx()
     stmt = (
@@ -169,18 +168,15 @@ def test_sign_dsse(staging):
 @pytest.mark.timestamp_authority
 class TestSignWithTSA:
     @pytest.fixture
-    def sig_ctx(self, asset, tsa_url) -> SigningContext:
-        trust_config = ClientTrustConfig.from_json(
-            asset("tsa/trust_config.json").read_text()
-        )
-
-        trust_config._inner.signing_config.tsa_urls[0].url = tsa_url
-
-        return SigningContext.from_trust_config(trust_config)
+    def sign_ctx_and_identity(self, preprod, tsa_url):
+        sign_ctx_func, _, identity = preprod
+        sign_ctx = sign_ctx_func()
+        sign_ctx._tsa_clients[0].url = tsa_url
+        return sign_ctx, identity
 
     @pytest.fixture
-    def identity(self, staging):
-        _, _, identity = staging
+    def identity(self, preprod):
+        _, _, identity = preprod
         return identity
 
     @pytest.fixture
@@ -190,7 +186,8 @@ def hashed(self) -> Hashed:
             digest=hashlib.sha256(input_).digest(), algorithm=HashAlgorithm.SHA2_256
         )
 
-    def test_sign_artifact(self, sig_ctx, identity, hashed):
+    def test_sign_artifact(self, sign_ctx_and_identity, hashed):
+        sig_ctx, identity = sign_ctx_and_identity
         with sig_ctx.signer(identity) as signer:
             bundle = signer.sign_artifact(hashed)
 
@@ -199,7 +196,8 @@ def test_sign_artifact(self, sig_ctx, identity, hashed):
             bundle.verification_material.timestamp_verification_data.rfc3161_timestamps
         )
 
-    def test_sign_dsse(self, sig_ctx, identity):
+    def test_sign_dsse(self, sign_ctx_and_identity):
+        sig_ctx, identity = sign_ctx_and_identity
         stmt = (
             StatementBuilder()
             .subjects(
@@ -226,8 +224,9 @@ def test_sign_dsse(self, sig_ctx, identity):
             bundle.verification_material.timestamp_verification_data.rfc3161_timestamps
         )
 
-    def test_with_timestamp_error(self, sig_ctx, identity, hashed, caplog):
+    def test_with_timestamp_error(self, sign_ctx_and_identity, hashed, caplog):
         # Simulate here an TSA that returns an invalid Timestamp
+        sig_ctx, identity = sign_ctx_and_identity
         sig_ctx._tsa_clients.append(TimestampAuthorityClient("invalid-url"))
 
         with caplog.at_level(logging.WARNING, logger="sigstore.sign"):

test/unit/verify/test_verifier.py

@@ -170,8 +170,8 @@ def test_verifier_fail_expiry(signing_materials, null_policy, monkeypatch):
 
 @pytest.mark.staging
 @pytest.mark.ambient_oidc
-def test_verifier_dsse_roundtrip(staging):
-    signer_cls, verifier_cls, identity = staging
+def test_verifier_dsse_roundtrip(preprod):
+    signer_cls, verifier_cls, identity = preprod
 
     ctx = signer_cls()
     stmt = (