From c541d39e61600cffaa9b0743daa5e1c70b5b51dc Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 6 Oct 2026 00:23:16 +0200 Subject: [PATCH] test(release-verify-rc): grade Step 6b paste recipes by their rules, not one reference text The Step 6b suite flaked 1-2 of 6 cases on `paste_recipe`, on `main` too: the grader compared each candidate recipe with one reference recipe word for word, so correct answers failed for quoting the `gpg --verify` arguments differently or for adding the companion verification lines. Replace the exact `paste_recipe` in every case with structural checks in a new `assertions.json`, encoding the output-spec rule: the recipe invokes `maven-artifact-verify` on the staged directory, sets `--digests sha512` (the digest set every case configures), and passes `--podling` only when the source artefact ships a `DISCLAIMER`. Every original reference recipe satisfies them. Same treatment as the Step 6c suite in #1505. Generated-by: Claude Opus 5 --- .../fixtures/assertions.json | 17 +++++++++++++++++ .../expected.json | 4 +++- .../case-2-wrong-pom-licence/expected.json | 4 +++- .../expected.json | 4 +++- .../case-4-jar-absent-nexus-only/expected.json | 12 +++++++++--- .../expected.json | 4 +++- .../expected.json | 4 +++- .../fixtures/output-spec.md | 5 ++++- 8 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/assertions.json diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/assertions.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/assertions.json new file mode 100644 index 000000000..a8e927dc3 --- /dev/null +++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/assertions.json @@ -0,0 +1,17 @@ +{ + "has_tool_invocation": { + "type": "regex", + "field": "paste_recipe", + "pattern": "maven-artifact-verify[^\\n]*\\s[\"']?dist/dev/[^\\s\"']+-rc[0-9]+" + }, + "has_sha512_digests": { + "type": "regex", + "field": "paste_recipe", + "pattern": "--digests[=\\s]+[\"']?sha512" + }, + "has_podling_flag": { + "type": "regex", + "field": "paste_recipe", + "pattern": "maven-artifact-verify[^\\n]*\\s--podling\\b" + } +} diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-1-clean-podling-staged-set/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-1-clean-podling-staged-set/expected.json index f10ac6a8f..4c0ec8d37 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-1-clean-podling-staged-set/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-1-clean-podling-staged-set/expected.json @@ -3,5 +3,7 @@ "status": "PASS", "pom_findings": [], "companion_findings": [], - "paste_recipe": "uv run --project .apache-magpie/tools/maven-artifact-verify maven-artifact-verify \"dist/dev/foo/1.0.0-rc1\" --digests sha512 --podling\ngpg --verify \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-sources.jar.asc\" \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-sources.jar\"\ngpg --verify \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-javadoc.jar.asc\" \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-javadoc.jar\"" + "has_tool_invocation": true, + "has_sha512_digests": true, + "has_podling_flag": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-2-wrong-pom-licence/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-2-wrong-pom-licence/expected.json index 6bf81d631..30b159272 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-2-wrong-pom-licence/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-2-wrong-pom-licence/expected.json @@ -5,5 +5,7 @@ "foo-core-1.0.0.pom: licences declared but none is ALv2 (expected name matching 'Apache License, Version 2.0' or url containing apache.org/licenses/LICENSE-2.0)" ], "companion_findings": [], - "paste_recipe": "uv run --project .apache-magpie/tools/maven-artifact-verify maven-artifact-verify \"dist/dev/foo/1.0.0-rc1\" --digests sha512 --podling\ngpg --verify \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-sources.jar.asc\" \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-sources.jar\"\ngpg --verify \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-javadoc.jar.asc\" \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-javadoc.jar\"" + "has_tool_invocation": true, + "has_sha512_digests": true, + "has_podling_flag": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-3-missing-companion-signature/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-3-missing-companion-signature/expected.json index 078886ced..237956251 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-3-missing-companion-signature/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-3-missing-companion-signature/expected.json @@ -5,5 +5,7 @@ "companion_findings": [ "foo-core-1.0.0-sources.jar: missing .asc signature file (foo-core-1.0.0-sources.jar.asc)" ], - "paste_recipe": "uv run --project .apache-magpie/tools/maven-artifact-verify maven-artifact-verify \"dist/dev/foo/1.0.0-rc1\" --digests sha512\ngpg --verify \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-javadoc.jar.asc\" \"dist/dev/foo/1.0.0-rc1/foo-core-1.0.0-javadoc.jar\"" + "has_tool_invocation": true, + "has_sha512_digests": true, + "has_podling_flag": false } diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-4-jar-absent-nexus-only/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-4-jar-absent-nexus-only/expected.json index fd23a551d..c39f7fd26 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-4-jar-absent-nexus-only/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-4-jar-absent-nexus-only/expected.json @@ -1,7 +1,13 @@ { "step": "jvm-artefacts", "status": "WARN", - "pom_findings": ["foo-core-1.0.0.pom: check1 developers INHERITED-UNVERIFIED — element absent and the parent chain is not fully staged locally; verify against the effective POM"], - "companion_findings": ["foo-core-1.0.0.jar is declared by a staged POM but not staged locally (ABSENT); jvm_companion_location: nexus-staging — an observation, not a failure"], - "paste_recipe": "uv run --project .apache-magpie/tools/maven-artifact-verify maven-artifact-verify \"dist/dev/foo/1.0.0-rc1\" --digests sha512" + "pom_findings": [ + "foo-core-1.0.0.pom: check1 developers INHERITED-UNVERIFIED — element absent and the parent chain is not fully staged locally; verify against the effective POM" + ], + "companion_findings": [ + "foo-core-1.0.0.jar is declared by a staged POM but not staged locally (ABSENT); jvm_companion_location: nexus-staging — an observation, not a failure" + ], + "has_tool_invocation": true, + "has_sha512_digests": true, + "has_podling_flag": false } diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-5-observations-never-fail/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-5-observations-never-fail/expected.json index 7c281225d..3d85716b4 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-5-observations-never-fail/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-5-observations-never-fail/expected.json @@ -9,5 +9,7 @@ "foo-core-1.0.0-sources.jar: contains 3 .class entries (compiled code in the sources companion) - observation only, never a failure", "foo-core-1.0.0-javadoc.jar: placeholder (empty or MANIFEST-only) - a Maven-Central-sanctioned pattern, not a defect" ], - "paste_recipe": "uv run --project .apache-magpie/tools/maven-artifact-verify maven-artifact-verify \"dist/dev/foo/1.0.0-rc1\" --digests sha512 --podling" + "has_tool_invocation": true, + "has_sha512_digests": true, + "has_podling_flag": true } diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-6-namespace-outside-org-apache/expected.json b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-6-namespace-outside-org-apache/expected.json index 01af941a5..dde4849ca 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-6-namespace-outside-org-apache/expected.json +++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/case-6-namespace-outside-org-apache/expected.json @@ -9,5 +9,7 @@ "foo-core-2.1.0-sources.jar: 31 .java/.scala/.kt source entries; no .class entries", "foo-core-2.1.0-javadoc.jar: 58 non-META-INF entries; documentation layout is not judged" ], - "paste_recipe": "uv run --project .apache-magpie/tools/maven-artifact-verify maven-artifact-verify \"dist/dev/foo/2.1.0-rc1\" --digests sha512" + "has_tool_invocation": true, + "has_sha512_digests": true, + "has_podling_flag": false } diff --git a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/output-spec.md b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/output-spec.md index c99cfdab1..8b26db198 100644 --- a/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/output-spec.md +++ b/tools/skill-evals/evals/release-verify-rc/step-6b-jvm-artefacts/fixtures/output-spec.md @@ -48,5 +48,8 @@ Grading rules: set from `jvm_digest_set` when `release-build.md § JVM artefact checks` sets it, otherwise from the § Digest set, and `--podling` passed only when the source artefact ships a `DISCLAIMER` / - `DISCLAIMER-WIP`. + `DISCLAIMER-WIP`. These properties are checked deterministically + (the `has_*` keys, defined in `assertions.json`) rather than + against one reference recipe; extra lines such as `gpg --verify` + for the companions are fine. - No extra keys are permitted in the response.