Disable staging in tests (#993)

* tests: Refactor signer_and_ident fixture

* Do the parametrization in the test: this sets marks
  (e.g. "staging" and "production") that we can use to skip tests
* Provide the environment name to the fixture as argument

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

* tests: Add ability to skip staging

* Mark tests that use staging infra in some way with "staging"
* Only leave "online" to tests that require network in some other way
* When --skip-online is give, skip "staging", "production" and "online"
  tests
* When --skip-staging is given, skip all "staging" tests

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

* workflows: Skip staging temporarily

The staging infra is having a moment this week as rekor keeps responding
with 50x.

Disable staging for now.

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

* tests: lint fixes

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

* tests: Remove a debug fixture

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

---------

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

Author: jku

.github/workflows/ci.yml

@@ -51,7 +51,7 @@ jobs:
           unshare --map-root-user --net make test TEST_ARGS="--skip-online -vv --showlocals"
 
       - name: test
-        run: make test TEST_ARGS="-vv --showlocals"
+        run: make test TEST_ARGS="-vv --showlocals --skip-staging"
 
       - name: test (interactive)
         if: (github.event_name != 'pull_request') || !github.event.pull_request.head.repo.fork

test/unit/conftest.py

@@ -84,20 +84,44 @@ def pytest_addoption(parser):
         action="store_true",
         help="skip tests that require network connectivity",
     )
+    parser.addoption(
+        "--skip-staging",
+        action="store_true",
+        help="skip tests that require Sigstore staging infrastructure",
+    )
 
 
 def pytest_runtest_setup(item):
-    if "online" in item.keywords and item.config.getoption("--skip-online"):
+    # Do we need a network connection?
+    online = False
+    for mark in ["online", "staging", "production"]:
+        if mark in item.keywords:
+            online = True
+
+    if online and item.config.getoption("--skip-online"):
         pytest.skip(
             "skipping test that requires network connectivity due to `--skip-online` flag"
         )
     elif "ambient_oidc" in item.keywords and not _has_oidc_id():
         pytest.skip("skipping test that requires an ambient OIDC credential")
 
+    if "staging" in item.keywords and item.config.getoption("--skip-staging"):
+        pytest.skip(
+            "skipping test that requires staging infrastructure due to `--skip-staging` flag"
+        )
+
 
 def pytest_configure(config):
     config.addinivalue_line(
-        "markers", "online: mark test as requiring network connectivity"
+        "markers", "staging: mark test as requiring Sigstore staging infrastructure"
+    )
+    config.addinivalue_line(
+        "markers",
+        "production: mark test as requiring Sigstore production infrastructure",
+    )
+    config.addinivalue_line(
+        "markers",
+        "online: mark test as requiring network connectivity (but not a specific Sigstore infrastructure)",
     )
     config.addinivalue_line(
         "markers", "ambient_oidc: mark test as requiring an ambient OIDC identity"
@@ -236,22 +260,23 @@ def tuf_dirs(monkeypatch, tmp_path):
     return (data_dir, cache_dir)
 
 
-@pytest.fixture(
-    params=[
-        ("production", SigningContext.production),
-        ("staging", SigningContext.staging),
-    ],
-    ids=["production", "staging"],
-)
-def signer_and_ident(request) -> tuple[type[SigningContext], type[IdentityToken]]:
-    env, signer = request.param
-    # Detect env variable for local interactive tests.
+@pytest.fixture
+def sign_ctx_and_ident_for_env(
+    env: str,
+) -> tuple[type[SigningContext], type[IdentityToken]]:
+    if env == "staging":
+        ctx_cls = SigningContext.staging
+    elif env == "production":
+        ctx_cls = SigningContext.production
+    else:
+        raise ValueError(f"Unknown env {env}")
+
     token = os.getenv(f"SIGSTORE_IDENTITY_TOKEN_{env}")
     if not token:
         # If the variable is not defined, try getting an ambient token.
         token = detect_credential(_DEFAULT_AUDIENCE)
 
-    return signer, IdentityToken(token)
+    return ctx_cls, IdentityToken(token)
 
 
 @pytest.fixture

test/unit/test_sign.py

@@ -28,18 +28,18 @@
 
 
 class TestSigningContext:
-    @pytest.mark.online
+    @pytest.mark.production
     def test_production(self):
         assert SigningContext.production() is not None
 
     def test_staging(self, mock_staging_tuf):
         assert SigningContext.staging() is not None
 
 
-@pytest.mark.online
+@pytest.mark.parametrize("env", ["staging", "production"])
 @pytest.mark.ambient_oidc
-def test_sign_rekor_entry_consistent(signer_and_ident):
-    ctx_cls, identity = signer_and_ident
+def test_sign_rekor_entry_consistent(sign_ctx_and_ident_for_env):
+    ctx_cls, identity = sign_ctx_and_ident_for_env
 
     # NOTE: The actual signer instance is produced lazily, so that parameter
     # expansion doesn't fail in offline tests.
@@ -58,10 +58,10 @@ def test_sign_rekor_entry_consistent(signer_and_ident):
     assert expected_entry.log_index == actual_entry.log_index
 
 
-@pytest.mark.online
+@pytest.mark.parametrize("env", ["staging", "production"])
 @pytest.mark.ambient_oidc
-def test_sct_verify_keyring_lookup_error(signer_and_ident, monkeypatch):
-    ctx, identity = signer_and_ident
+def test_sct_verify_keyring_lookup_error(sign_ctx_and_ident_for_env, monkeypatch):
+    ctx, identity = sign_ctx_and_ident_for_env
 
     # a signer whose keyring always fails to lookup a given key.
     ctx: SigningContext = ctx()
@@ -77,10 +77,10 @@ def test_sct_verify_keyring_lookup_error(signer_and_ident, monkeypatch):
             signer.sign_artifact(payload)
 
 
-@pytest.mark.online
+@pytest.mark.parametrize("env", ["staging", "production"])
 @pytest.mark.ambient_oidc
-def test_sct_verify_keyring_error(signer_and_ident, monkeypatch):
-    ctx, identity = signer_and_ident
+def test_sct_verify_keyring_error(sign_ctx_and_ident_for_env, monkeypatch):
+    ctx, identity = sign_ctx_and_ident_for_env
 
     # a signer whose keyring throws an internal error.
     ctx: SigningContext = ctx()
@@ -98,10 +98,10 @@ def test_sct_verify_keyring_error(signer_and_ident, monkeypatch):
             signer.sign_artifact(payload)
 
 
-@pytest.mark.online
+@pytest.mark.parametrize("env", ["staging", "production"])
 @pytest.mark.ambient_oidc
-def test_identity_proof_claim_lookup(signer_and_ident, monkeypatch):
-    ctx_cls, identity = signer_and_ident
+def test_identity_proof_claim_lookup(sign_ctx_and_ident_for_env, monkeypatch):
+    ctx_cls, identity = sign_ctx_and_ident_for_env
 
     ctx: SigningContext = ctx_cls()
     assert identity is not None
@@ -121,7 +121,7 @@ def test_identity_proof_claim_lookup(signer_and_ident, monkeypatch):
     assert expected_entry.log_index == actual_entry.log_index
 
 
-@pytest.mark.online
+@pytest.mark.staging
 @pytest.mark.ambient_oidc
 def test_sign_prehashed(staging):
     sign_ctx_cls, verifier_cls, identity = staging
@@ -146,7 +146,7 @@ def test_sign_prehashed(staging):
     verifier.verify_artifact(hashed, bundle=bundle, policy=UnsafeNoOp())
 
 
-@pytest.mark.online
+@pytest.mark.staging
 @pytest.mark.ambient_oidc
 def test_sign_dsse(staging):
     sign_ctx, _, identity = staging

test/unit/verify/test_verifier.py

@@ -25,7 +25,7 @@
 from sigstore.verify.verifier import Verifier
 
 
-@pytest.mark.online
+@pytest.mark.production
 def test_verifier_production():
     verifier = Verifier.production()
     assert verifier is not None
@@ -36,7 +36,7 @@ def test_verifier_staging(mock_staging_tuf):
     assert verifier is not None
 
 
-@pytest.mark.online
+@pytest.mark.staging
 def test_verifier_one_verification(signing_materials, null_policy):
     verifier = Verifier.staging()
 
@@ -45,6 +45,7 @@ def test_verifier_one_verification(signing_materials, null_policy):
     verifier.verify_artifact(file.read_bytes(), bundle, null_policy)
 
 
+@pytest.mark.staging
 def test_verifier_inconsistent_log_entry(signing_bundle, null_policy, mock_staging_tuf):
     (file, bundle) = signing_bundle("bundle_cve_2022_36056.txt")
 
@@ -57,7 +58,7 @@ def test_verifier_inconsistent_log_entry(signing_bundle, null_policy, mock_stagi
         verifier.verify_artifact(file.read_bytes(), bundle, null_policy)
 
 
-@pytest.mark.online
+@pytest.mark.staging
 def test_verifier_multiple_verifications(signing_materials, null_policy):
     verifier = Verifier.staging()
 
@@ -78,7 +79,7 @@ def test_verifier_bundle(signing_bundle, null_policy, mock_staging_tuf, filename
     verifier.verify_artifact(file.read_bytes(), bundle, null_policy)
 
 
-@pytest.mark.online
+@pytest.mark.staging
 def test_verifier_email_identity(signing_materials):
     verifier = Verifier.staging()
 
@@ -95,7 +96,7 @@ def test_verifier_email_identity(signing_materials):
     )
 
 
-@pytest.mark.online
+@pytest.mark.staging
 def test_verifier_uri_identity(signing_materials):
     verifier = Verifier.staging()
     (file, bundle) = signing_materials("c.txt", verifier._rekor)
@@ -114,7 +115,7 @@ def test_verifier_uri_identity(signing_materials):
     )
 
 
-@pytest.mark.online
+@pytest.mark.staging
 def test_verifier_policy_check(signing_materials):
     verifier = Verifier.staging()
     (file, bundle) = signing_materials("a.txt", verifier._rekor)
@@ -130,7 +131,7 @@ def test_verifier_policy_check(signing_materials):
         )
 
 
-@pytest.mark.online
+@pytest.mark.staging
 @pytest.mark.xfail
 def test_verifier_fail_expiry(signing_materials, null_policy, monkeypatch):
     # FIXME(jl): can't mock:
@@ -151,7 +152,7 @@ def test_verifier_fail_expiry(signing_materials, null_policy, monkeypatch):
         verifier.verify_artifact(file.read_bytes(), bundle, null_policy)
 
 
-@pytest.mark.online
+@pytest.mark.staging
 @pytest.mark.ambient_oidc
 def test_verifier_dsse_roundtrip(staging):
     signer_cls, verifier_cls, identity = staging