From 5588442891f879921d000732a0b9dd3081f57a00 Mon Sep 17 00:00:00 2001 From: Test User Date: Wed, 15 Apr 2026 18:18:44 +0900 Subject: [PATCH 1/2] feat(ship): add shipping readiness skill Add the new /ship skill as a launch-readiness gate with supporting checklist and test coverage. --- .../plans/2026-04-15-me-ship-skill.md | 392 ++++++++++++++++++ .../specs/2026-04-15-me-ship-skill-design.md | 262 ++++++++++++ plugins/me/skills/ship/SKILL.md | 99 +++++ .../skills/ship/references/ship-checklist.md | 63 +++ tests/me/me-specific.bats | 11 + tests/skills/test_ship_skill_content.bats | 38 ++ 6 files changed, 865 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-15-me-ship-skill.md create mode 100644 docs/superpowers/specs/2026-04-15-me-ship-skill-design.md create mode 100644 plugins/me/skills/ship/SKILL.md create mode 100644 plugins/me/skills/ship/references/ship-checklist.md create mode 100644 tests/skills/test_ship_skill_content.bats diff --git a/docs/superpowers/plans/2026-04-15-me-ship-skill.md b/docs/superpowers/plans/2026-04-15-me-ship-skill.md new file mode 100644 index 00000000..f2adde9d --- /dev/null +++ b/docs/superpowers/plans/2026-04-15-me-ship-skill.md @@ -0,0 +1,392 @@ +# me /ship Skill Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a new `me` plugin `/ship` skill that behaves as a shipping readiness gate rather than a deploy executor. + +**Architecture:** Implement `/ship` as a minimal Markdown skill with one always-loaded `SKILL.md` and one on-demand reference file for checklist details. Keep the behavior aligned with agent-skills: evaluate the current change set, report readiness as Ready / Conditionally ready / Not ready, and never infer or execute deploy commands. + +**Tech Stack:** Markdown skill authoring, BATS tests, existing plugin/skill loading conventions + +--- + +### Task 1: Create the `/ship` reference checklist + +**Files:** +- Create: `plugins/me/skills/ship/references/ship-checklist.md` +- Test: `plugins/me/skills/ship/references/ship-checklist.md` + +- [ ] **Step 1: Write the failing presence check mentally from the spec** + +The spec requires an on-demand reference file with checklist examples for pre-launch, rollout, rollback, monitoring, plus blocker/warning examples. The file does not exist yet, so the expected first failure is simply “missing file at the exact path”. + +- [ ] **Step 2: Create `ship-checklist.md` with the minimal required content** + +Write `plugins/me/skills/ship/references/ship-checklist.md` with this content: + +```markdown +# Ship Checklist + +Reference material for `/ship`. Read this when you need concrete examples while assessing shipping readiness. This file supports the core skill; it does not change the `/ship` contract. + +## Pre-launch checks + +Use these to decide whether the change is basically ready to leave the branch: + +- The change scope can be described in one or two sentences. +- The relevant tests or verification steps are named and have recent evidence. +- Any required review, approval, or human sign-off is explicit. +- Any launch notes or operator context are written down somewhere discoverable. + +## Rollout readiness + +Use these to assess whether the release can be introduced safely: + +- A staged rollout is possible, or the change is clearly low-risk enough not to need one. +- A feature flag, kill switch, or config gate exists when exposure risk is meaningful. +- The change does not require an all-at-once cutover without justification. + +## Rollback readiness + +Use these to assess whether the team can recover quickly: + +- A rollback path can be explained in plain language. +- Irreversible schema or data changes are identified explicitly. +- The first action to take during a bad launch is known. + +## Monitoring readiness + +Use these to assess whether post-launch behavior is observable: + +- There is at least one success signal to watch. +- There is at least one failure signal to watch. +- The relevant logs, metrics, or alerts are named. +- The launch is not blind; someone could tell within minutes if it went wrong. + +## Blocking issue examples + +These usually mean `/ship` should report **Not ready**: + +- No test or verification evidence is available. +- No rollback path can be described. +- Monitoring signals are completely unknown. +- Required QA or review has clearly not happened. + +## Warning examples + +These usually mean `/ship` should report **Conditionally ready** rather than **Ready**: + +- The change is large and rollout strategy is weak. +- A feature flag would help but is not strictly required. +- Monitoring exists but the exact watchpoints are not written down. +- The launch can proceed, but only with explicit human attention. + +## Suggested decision language + +Use short, direct language: + +- **Ready** — No blockers found. Basic rollout, rollback, and monitoring expectations are covered. +- **Conditionally ready** — No hard blocker, but the ship needs explicit follow-up before or during launch. +- **Not ready** — One or more critical gates are missing; shipping now would be unsafe. +``` + +- [ ] **Step 3: Verify the file exists and starts correctly** + +Run: `test -f plugins/me/skills/ship/references/ship-checklist.md && sed -n '1,12p' plugins/me/skills/ship/references/ship-checklist.md` +Expected: output starts with `# Ship Checklist` and includes the sentence `Reference material for '/ship'`. + +- [ ] **Step 4: Commit the reference file** + +```bash +git add plugins/me/skills/ship/references/ship-checklist.md +git commit -m "feat(ship): add shipping checklist reference" +``` + +--- + +### Task 2: Create the core `/ship` skill + +**Files:** +- Create: `plugins/me/skills/ship/SKILL.md` +- Test: `plugins/me/skills/ship/SKILL.md` + +- [ ] **Step 1: Write the failing contract checklist from the spec** + +Before writing the skill, verify the spec requirements that must appear in the file: + +- readiness gate, not deploy executor +- non-goals mention no deploy inference/execution +- Decision / Blocking issues / Warnings / Readiness by area / Next actions output structure +- Ready / Conditionally ready / Not ready decisions +- instruction to consult `references/ship-checklist.md` when needed + +The file does not exist yet, so this contract currently fails by absence. + +- [ ] **Step 2: Create `SKILL.md` with the exact first version** + +Write `plugins/me/skills/ship/SKILL.md` with this content: + +```markdown +--- +name: ship +description: Use when asked to "ship", "launch", "release", or "is this ready to go live?". Reviews the current change as a shipping candidate and reports readiness without executing deploy commands. +allowed-tools: + - Bash + - Read + - Glob + - Grep +--- + +# /ship: Review shipping readiness + +You are a shipping readiness reviewer. `/ship` is a launch gate, not a deploy executor. + +## What `/ship` does + +- Reviews the current change as a shipping candidate +- Identifies blockers and warnings +- Assesses rollout, rollback, and monitoring readiness +- Produces a short readiness report with next actions + +## What `/ship` does NOT do + +- Do not invent or run deploy commands +- Do not replace `/qa` +- Do not create or merge PRs +- Do not take ownership of versioning or release automation + +## Candidate under review + +Default to the current working change. + +Use whatever evidence is available in the repository to understand scope: +- current branch state +- `main...HEAD` diff when available +- recent test or verification evidence + +If the scope is unclear, say so and downgrade the decision. + +## Required review areas + +Review the change across these areas: + +1. **Pre-launch** — Is the scope clear? Is there test or verification evidence? Is required review or operator context present? +2. **Rollout** — Could this be introduced safely? Is there a feature flag, kill switch, or another way to limit blast radius when appropriate? +3. **Rollback** — Could the team explain how to recover if the launch goes badly? +4. **Monitoring** — Are there logs, metrics, alerts, or explicit watchpoints that would reveal success or failure? + +For concrete examples and decision patterns, read `ship/references/ship-checklist.md`. + +## Decision rules + +Choose one outcome: + +- **Ready** — No blockers found. Basic rollout, rollback, and monitoring expectations are covered. +- **Conditionally ready** — Not blocked, but the launch needs explicit follow-up before or during release. +- **Not ready** — A critical gate is missing. + +Default to **Not ready** when core evidence is missing. + +## Output format + +Always report using these sections: + +### Decision + +Ready / Conditionally ready / Not ready + +### Blocking issues + +List only items that must be resolved before shipping. + +### Warnings + +List risks that do not fully block launch. + +### Readiness by area + +- Pre-launch: pass / weak / fail +- Rollout: pass / weak / fail +- Rollback: pass / weak / fail +- Monitoring: pass / weak / fail + +### Next actions + +Give the smallest set of actions needed to improve readiness. + +## Red flags + +Do not mark a change as Ready if any of these are true: + +- there is no test or verification evidence +- rollback is unknown +- monitoring signals are unknown +- required QA or review clearly has not happened + +## Verification expectations + +Ground every judgment in repository evidence. If you cannot verify a claim, say that directly and lower the readiness decision. +``` + +- [ ] **Step 3: Verify required sections exist** + +Run: `grep -E '^name: ship$|^## What `/ship` does NOT do$|^### Decision$|^### Blocking issues$|^### Warnings$|^### Readiness by area$|^### Next actions$' plugins/me/skills/ship/SKILL.md` +Expected: matches for all required headings and the `name: ship` frontmatter line. + +- [ ] **Step 4: Verify the reference path is spelled correctly** + +Run: `grep -n 'ship/references/ship-checklist.md' plugins/me/skills/ship/SKILL.md` +Expected: one matching line pointing to `ship/references/ship-checklist.md`. + +- [ ] **Step 5: Commit the core skill** + +```bash +git add plugins/me/skills/ship/SKILL.md +git commit -m "feat(ship): add shipping readiness skill" +``` + +--- + +### Task 3: Extend plugin tests to cover `/ship` + +**Files:** +- Modify: `tests/me/me-specific.bats` +- Create: `tests/skills/test_ship_skill_content.bats` +- Test: `tests/me/me-specific.bats` +- Test: `tests/skills/test_ship_skill_content.bats` + +- [ ] **Step 1: Add a presence test in `tests/me/me-specific.bats`** + +Append these tests after the existing create-pr skill tests: + +```bash +@test "me: ship skill exists with required files" { + [ -f "${PROJECT_ROOT}/plugins/me/skills/ship/SKILL.md" ] + [ -f "${PROJECT_ROOT}/plugins/me/skills/ship/references/ship-checklist.md" ] +} + +@test "me: ship skill has proper frontmatter" { + local skill_file="${PROJECT_ROOT}/plugins/me/skills/ship/SKILL.md" + has_frontmatter_delimiter "$skill_file" + has_frontmatter_field "$skill_file" "name" + has_frontmatter_field "$skill_file" "description" +} +``` + +- [ ] **Step 2: Add focused content tests for `/ship`** + +Create `tests/skills/test_ship_skill_content.bats` with this content: + +```bash +#!/usr/bin/env bats + +load '../helpers/bats_helper' + +setup() { + export SHIP_SKILL_MD="${BATS_TEST_DIRNAME}/../../plugins/me/skills/ship/SKILL.md" + + if [[ ! -f "$SHIP_SKILL_MD" ]]; then + skip "ship SKILL.md not found" + fi +} + +@test "ship SKILL.md exists" { + [ -f "$SHIP_SKILL_MD" ] +} + +@test "ship SKILL.md defines ship frontmatter name" { + grep -q '^name: ship$' "$SHIP_SKILL_MD" +} + +@test "ship SKILL.md describes readiness gate, not deploy execution" { + grep -q 'launch gate, not a deploy executor' "$SHIP_SKILL_MD" + grep -q 'Do not invent or run deploy commands' "$SHIP_SKILL_MD" +} + +@test "ship SKILL.md includes required output sections" { + grep -q '^### Decision$' "$SHIP_SKILL_MD" + grep -q '^### Blocking issues$' "$SHIP_SKILL_MD" + grep -q '^### Warnings$' "$SHIP_SKILL_MD" + grep -q '^### Readiness by area$' "$SHIP_SKILL_MD" + grep -q '^### Next actions$' "$SHIP_SKILL_MD" +} + +@test "ship SKILL.md includes all readiness outcomes" { + grep -q '\*\*Ready\*\*' "$SHIP_SKILL_MD" + grep -q '\*\*Conditionally ready\*\*' "$SHIP_SKILL_MD" + grep -q '\*\*Not ready\*\*' "$SHIP_SKILL_MD" +} +``` + +- [ ] **Step 3: Run the new `/ship` tests** + +Run: `bats tests/me/me-specific.bats tests/skills/test_ship_skill_content.bats` +Expected: all `/ship` assertions PASS. + +- [ ] **Step 4: Commit the tests** + +```bash +git add tests/me/me-specific.bats tests/skills/test_ship_skill_content.bats +git commit -m "test(ship): add ship skill coverage" +``` + +--- + +### Task 4: Run repo-level verification for the new skill + +**Files:** +- Modify: none +- Test: `tests/me/me-specific.bats` +- Test: `tests/skills/test_ship_skill_content.bats` +- Test: `tests/frontmatter_tests.bats` +- Test: `tests/integration/plugin_loading.bats` + +- [ ] **Step 1: Run the targeted structural test suites** + +Run: `bats tests/frontmatter_tests.bats tests/integration/plugin_loading.bats tests/me/me-specific.bats tests/skills/test_ship_skill_content.bats` +Expected: PASS. The new skill should satisfy frontmatter and plugin-loading conventions. + +- [ ] **Step 2: If a failure mentions a missing heading or frontmatter field, fix only the minimal relevant file** + +Use this decision table: + +- If the failure mentions `has_frontmatter_field` or delimiter checks, edit `plugins/me/skills/ship/SKILL.md` frontmatter only. +- If the failure mentions missing file paths, create or rename only the missing `/ship` files. +- If the failure mentions specific text assertions, update only the exact tested wording in `plugins/me/skills/ship/SKILL.md` or `tests/skills/test_ship_skill_content.bats`. + +Do not broaden scope beyond the `/ship` skill and its direct tests. + +- [ ] **Step 3: Re-run the same targeted suites until they pass** + +Run: `bats tests/frontmatter_tests.bats tests/integration/plugin_loading.bats tests/me/me-specific.bats tests/skills/test_ship_skill_content.bats` +Expected: PASS with no `/ship`-related failures. + +- [ ] **Step 4: Run the project test entrypoint** + +Run: `bash tests/run-all-tests.sh` +Expected: PASS. The repository-wide BATS runner completes without regressions from the new skill. + +- [ ] **Step 5: Commit the final verified state** + +```bash +git add plugins/me/skills/ship/SKILL.md plugins/me/skills/ship/references/ship-checklist.md tests/me/me-specific.bats tests/skills/test_ship_skill_content.bats +git commit -m "feat(ship): add shipping readiness skill" +``` + +--- + +## Spec Coverage Check + +- **Goal / readiness gate:** Covered by Task 2. +- **No deploy inference/execution:** Covered by Task 2 content assertions and Task 3 tests. +- **Reference checklist file:** Covered by Task 1. +- **Decision + blocker/warning + readiness-area output contract:** Covered by Task 2 and Task 3. +- **`/qa` and `/create-pr` role boundaries:** Covered by Task 2 wording. +- **Repository conventions and regression safety:** Covered by Task 4. + +## Self-Review Notes + +- Placeholder scan completed; no unresolved marker text remains. +- File paths are exact and match current repository structure. +- Later test tasks reference the same names introduced in Task 2: `ship`, `Decision`, `Blocking issues`, `Warnings`, `Readiness by area`, `Next actions`. +- Scope stays intentionally narrow: one new skill, one reference file, and direct tests only. diff --git a/docs/superpowers/specs/2026-04-15-me-ship-skill-design.md b/docs/superpowers/specs/2026-04-15-me-ship-skill-design.md new file mode 100644 index 00000000..99901fa4 --- /dev/null +++ b/docs/superpowers/specs/2026-04-15-me-ship-skill-design.md @@ -0,0 +1,262 @@ +# me /ship Skill Design + +**Date:** 2026-04-15 +**Scope:** `plugins/me/skills/ship/` +**Reference model:** `addyosmani/agent-skills`의 `/ship` 및 `shipping-and-launch` + +## Goal + +`me` plugin에 `/ship` 스킬을 추가한다. 이 스킬은 배포 실행기가 아니라 **출시 readiness gate**로 동작해야 한다. + +`/ship`의 목적은 다음 세 가지다. + +1. 현재 변경사항이 ship 가능한지 판정한다. +2. ship을 막는 누락 게이트와 위험 신호를 식별한다. +3. rollout / rollback / monitoring 준비 상태를 구조화해서 보고한다. + +## Non-Goals + +`/ship`은 다음을 하지 않는다. + +- 프로젝트별 deploy 명령을 추측해서 실행하지 않는다. +- QA 자체를 대체하지 않는다. +- PR 생성/머지 흐름을 대신하지 않는다. +- 릴리즈 자동화나 버전 관리 전체를 떠맡지 않는다. +- 환경별 배포 orchestration 도구로 확장하지 않는다. + +즉 첫 버전 `/ship`은 **배포 자동화 도구가 아니라 readiness reviewer skill**이다. + +## Relationship to Existing Skills + +- `/qa` = 결함 탐색, evidence 수집, QA report 작성 +- `/create-pr` = PR 생성과 병합 준비 자동화 +- `/ship` = 출시 readiness gate + +역할을 분리해서 중복을 줄인다. + +## Design Principles + +`/ship`은 가능한 한 agent-skills의 `/ship` 철학을 그대로 따른다. + +1. **Deploy보다 readiness 우선** + - 배포를 직접 실행하는 것보다, 지금 내보내도 되는 상태인지 판단하는 데 집중한다. +2. **Project-specific 추측 금지** + - 저장소에 명시되지 않은 deploy 절차나 인프라 명령을 상상해서 실행하지 않는다. +3. **Gate-first output** + - 결과는 판정, blocker, warning, next actions 중심으로 구성한다. +4. **불확실하면 통과시키지 않음** + - 테스트 근거, rollback 경로, monitoring 신호가 불명확하면 ready로 판정하지 않는다. + +## Candidate Under Review + +첫 버전 `/ship`은 **현재 작업 변경분**을 shipping candidate로 본다. + +실무적으로는 아래 정보를 사용해 범위를 파악한다. + +- 현재 브랜치 +- 가능하면 `main...HEAD` diff +- 현재 저장소 상태와 최근 검증 흔적 + +단, 스킬의 핵심은 입력 파라미터 다양화가 아니라 readiness review 자체이므로, 첫 버전은 입력 모델을 단순하게 유지한다. + +## Output Contract + +`/ship` 결과는 아래 형식을 따른다. + +### 1. Decision + +세 가지 중 하나로 판정한다. + +- **Ready** — ship 가능 +- **Conditionally ready** — ship 가능하지만 선행 확인/승인이 더 필요 +- **Not ready** — 현재 상태로는 ship 불가 + +### 2. Blocking issues + +ship을 막는 항목을 나열한다. + +예: +- 테스트 근거 없음 +- rollback 경로 불명확 +- 관련 QA가 필요한데 수행 근거 없음 +- 배포 후 관찰 포인트 부재 + +### 3. Warnings + +즉시 막지는 않지만 위험도가 있는 항목을 나열한다. + +예: +- 큰 변경인데 staged rollout 전략 없음 +- feature flag 없이 바로 노출되는 변경 +- 영향 범위 설명이 모호함 + +### 4. Readiness by area + +아래 4개 축을 각각 점검한다. + +- **Pre-launch** +- **Rollout** +- **Rollback** +- **Monitoring** + +### 5. Next actions + +ship 전에 해야 할 최소 행동만 제시한다. + +예: +- `/qa` 실행 +- smoke test 결과 첨부 +- rollback note 정리 +- 모니터링 지표 확인 포인트 명시 + +## Readiness Areas + +### Pre-launch checks + +기본 품질 게이트가 통과됐는지 본다. + +확인 예시: +- 변경 범위를 식별할 수 있는가 +- 관련 테스트/검증 근거가 있는가 +- 리뷰 또는 동등한 확인이 있었는가 +- ship 전에 확인해야 할 문서/메모가 있는가 + +### Release scope clarity + +이번 ship의 범위가 명확한지 본다. + +확인 예시: +- 무엇을 내보내는지 한두 문장으로 설명 가능한가 +- 너무 큰 변경이라 분리가 필요한가 +- 직접 노출 시 위험한 변경인데 안전장치가 없는가 + +이 축의 결과는 blocker 또는 warning의 근거로 반영한다. + +### Rollout readiness + +점진 배포 관점의 안전장치를 본다. + +확인 예시: +- staged rollout 개념을 적용할 수 있는가 +- feature flag / kill switch / configuration gate가 있는가 +- 전면 배포만 가능한 고위험 구조는 아닌가 + +### Rollback readiness + +문제 발생 시 되돌릴 수 있는지 본다. + +확인 예시: +- rollback 경로를 설명할 수 있는가 +- 되돌리기 어려운 schema/data migration이 포함되는가 +- 장애 시 first action이 무엇인지 명확한가 + +### Monitoring readiness + +배포 후 관찰 가능한 상태인지 본다. + +확인 예시: +- 배포 후 볼 로그/메트릭/알람이 있는가 +- 성공 신호와 실패 신호가 구분되는가 +- post-launch 확인 포인트가 전혀 비어 있지 않은가 + +## Failure Policy + +`/ship`은 억지로 통과시키지 않는다. + +권장 판정 규칙: + +- **Not ready** + - 테스트 근거 없음 + - rollback 경로가 전혀 설명되지 않음 + - monitoring 신호가 전혀 없음 + - ship 전에 반드시 필요한 선행 QA/검토가 누락됨 +- **Conditionally ready** + - 핵심은 통과했지만 rollout 또는 monitoring 계획이 약함 + - 위험도는 낮지만 사람이 마지막 확인을 해야 함 +- **Ready** + - blocker 없음 + - rollout / rollback / monitoring 관점에서 기본 설명 가능 + +## File Structure + +첫 버전은 최소 구조로 시작한다. + +```text +plugins/me/skills/ship/ +├── SKILL.md +└── references/ + └── ship-checklist.md +``` + +### `SKILL.md` + +항상 로드되는 핵심 문서. + +포함 내용: +- trigger 문구 +- `/ship`의 역할과 비목표 +- readiness review 단계 +- output contract +- red flags +- verification expectations + +### `references/ship-checklist.md` + +필요 시 읽는 보조 문서. + +포함 내용: +- pre-launch / rollout / rollback / monitoring 체크 예시 +- blocker vs warning 판단 예시 +- ship 판정 문구 예시 + +## Suggested SKILL Flow + +1. 현재 변경 범위와 ship candidate를 파악한다. +2. pre-launch 관점에서 기본 검증 근거를 확인한다. +3. rollout / rollback / monitoring 준비 상태를 점검한다. +4. blocker와 warning을 분리한다. +5. Ready / Conditionally ready / Not ready 중 하나로 판정한다. +6. next actions를 최소 단위로 정리한다. + +## Testing Strategy + +이 스킬은 배포 자동화보다 **결정 규칙과 출력 구조**가 중요하므로 테스트도 그 관점으로 잡는다. + +### Static validation + +- skill frontmatter가 유효한지 +- 필수 섹션이 존재하는지 +- reference 경로가 맞는지 + +### Behavior validation + +- `/ship`이 deploy 실행 지시로 기울지 않는지 +- blocker / warning / next actions를 분리하는지 +- `/qa`, `/create-pr` 역할과 섞이지 않는지 + +### Regression validation + +- 자동 병합이나 배포 실행처럼 범위를 넘는 지시가 없는지 +- readiness reviewer라는 경계를 유지하는지 + +## Acceptance Criteria + +다음이 만족되면 설계 목표를 달성한 것으로 본다. + +- `plugins/me/skills/ship/SKILL.md`가 존재한다. +- `/ship`이 readiness gate로 정의되어 있다. +- deploy 명령 실행이 기본 흐름에 포함되지 않는다. +- 결과 형식에 Decision / Blocking issues / Warnings / Readiness areas / Next actions가 포함된다. +- `/qa`와 `/create-pr`와의 역할 경계가 문서에 명시된다. +- rollout / rollback / monitoring 관점이 빠지지 않는다. + +## Open Questions Resolved + +- `/ship`은 배포 실행기인가? → 아니다. readiness gate다. +- `/qa`를 흡수하는가? → 아니다. 독립 유지한다. +- agent-skills를 얼마나 따를 것인가? → 가능한 한 그대로 따른다. +- 첫 버전에서 다양한 입력 모델을 지원하는가? → 아니다. 현재 변경분 중심으로 단순하게 간다. + +## Implementation Direction + +다음 단계에서는 이 설계를 바탕으로 `SKILL.md` 초안과 reference 문서 구조를 계획하면 된다. 첫 구현은 최소 기능으로 시작하고, 실제 사용 사례가 쌓이면 PR 입력, 환경별 컨텍스트, 프로젝트별 신호 소스 연결 같은 확장을 검토한다. diff --git a/plugins/me/skills/ship/SKILL.md b/plugins/me/skills/ship/SKILL.md new file mode 100644 index 00000000..a8929905 --- /dev/null +++ b/plugins/me/skills/ship/SKILL.md @@ -0,0 +1,99 @@ +--- +name: ship +description: Use when asked to "ship", "launch", "release", or "is this ready to go live?". Reviews the current change as a shipping candidate and reports readiness without executing deploy commands. +allowed-tools: + - Bash + - Read + - Glob + - Grep +--- + +# /ship: Review shipping readiness + +You are a shipping readiness reviewer. `/ship` is a launch gate, not a deploy executor. + +## What `/ship` does + +- Reviews the current change as a shipping candidate +- Identifies blockers and warnings +- Assesses rollout, rollback, and monitoring readiness +- Produces a short readiness report with next actions + +## What `/ship` does NOT do + +- Do not invent or run deploy commands +- Do not replace `/qa` +- Do not create or merge PRs +- Do not take ownership of versioning or release automation + +## Candidate under review + +Default to the current working change. + +Use whatever evidence is available in the repository to understand scope: +- current branch state +- `main...HEAD` diff when available +- recent test or verification evidence + +If the scope is unclear, say so and downgrade the decision. + +## Required review areas + +Review the change across these areas: + +1. **Pre-launch** — Is the scope clear? Is there test or verification evidence? Is required review or operator context present? +2. **Rollout** — Could this be introduced safely? Is there a feature flag, kill switch, or another way to limit blast radius when appropriate? +3. **Rollback** — Could the team explain how to recover if the launch goes badly? +4. **Monitoring** — Are there logs, metrics, alerts, or explicit watchpoints that would reveal success or failure? + +For concrete examples and decision patterns, read `ship/references/ship-checklist.md`. + +## Decision rules + +Choose one outcome: + +- **Ready** — No blockers found. Basic rollout, rollback, and monitoring expectations are covered. +- **Conditionally ready** — Not blocked, but the launch needs explicit follow-up before or during release. +- **Not ready** — A critical gate is missing. + +Default to **Not ready** when core evidence is missing. + +## Output format + +Always report using these sections: + +### Decision + +Ready / Conditionally ready / Not ready + +### Blocking issues + +List only items that must be resolved before shipping. + +### Warnings + +List risks that do not fully block launch. + +### Readiness by area + +- Pre-launch: pass / weak / fail +- Rollout: pass / weak / fail +- Rollback: pass / weak / fail +- Monitoring: pass / weak / fail + +### Next actions + +Give the smallest set of actions needed to improve readiness. + +## Red flags + +Do not mark a change as Ready if any of these are true: + +- there is no test or verification evidence +- rollback is unknown +- monitoring signals are unknown +- required QA or review clearly has not happened + +## Verification expectations + +Ground every judgment in repository evidence. If you cannot verify a claim, say that directly and lower the readiness decision. diff --git a/plugins/me/skills/ship/references/ship-checklist.md b/plugins/me/skills/ship/references/ship-checklist.md new file mode 100644 index 00000000..0ada618f --- /dev/null +++ b/plugins/me/skills/ship/references/ship-checklist.md @@ -0,0 +1,63 @@ +# Ship Checklist + +Reference material for `/ship`. Read this when you need concrete examples while assessing shipping readiness. This file supports the core skill; it does not change the `/ship` contract. + +## Pre-launch checks + +Use these to decide whether the change is basically ready to leave the branch: + +- The change scope can be described in one or two sentences. +- The relevant tests or verification steps are named and have recent evidence. +- Any required review, approval, or human sign-off is explicit. +- Any launch notes or operator context are written down somewhere discoverable. + +## Rollout readiness + +Use these to assess whether the release can be introduced safely: + +- A staged rollout is possible, or the change is clearly low-risk enough not to need one. +- A feature flag, kill switch, or config gate exists when exposure risk is meaningful. +- The change does not require an all-at-once cutover without justification. + +## Rollback readiness + +Use these to assess whether the team can recover quickly: + +- A rollback path can be explained in plain language. +- Irreversible schema or data changes are identified explicitly. +- The first action to take during a bad launch is known. + +## Monitoring readiness + +Use these to assess whether post-launch behavior is observable: + +- There is at least one success signal to watch. +- There is at least one failure signal to watch. +- The relevant logs, metrics, or alerts are named. +- The launch is not blind; someone could tell within minutes if it went wrong. + +## Blocking issue examples + +These usually mean `/ship` should report **Not ready**: + +- No test or verification evidence is available. +- No rollback path can be described. +- Monitoring signals are completely unknown. +- Required QA or review has clearly not happened. + +## Warning examples + +These usually mean `/ship` should report **Conditionally ready** rather than **Ready**: + +- The change is large and rollout strategy is weak. +- A feature flag would help but is not strictly required. +- Monitoring exists but the exact watchpoints are not written down. +- The launch can proceed, but only with explicit human attention. + +## Suggested decision language + +Use short, direct language: + +- **Ready** — No blockers found. Basic rollout, rollback, and monitoring expectations are covered. +- **Conditionally ready** — No hard blocker, but the ship needs explicit follow-up before or during launch. +- **Not ready** — One or more critical gates are missing; shipping now would be unsafe. diff --git a/tests/me/me-specific.bats b/tests/me/me-specific.bats index 03ac4125..bb5079ca 100644 --- a/tests/me/me-specific.bats +++ b/tests/me/me-specific.bats @@ -35,6 +35,17 @@ load ../helpers/bats_helper grep -q "git rev-parse.*git-dir" "$script" } +@test "me: ship skill exists with required files" { + [ -f "${PROJECT_ROOT}/plugins/me/skills/ship/SKILL.md" ] + [ -f "${PROJECT_ROOT}/plugins/me/skills/ship/references/ship-checklist.md" ] +} + +@test "me: ship skill has proper frontmatter" { + local skill_file="${PROJECT_ROOT}/plugins/me/skills/ship/SKILL.md" + has_frontmatter_delimiter "$skill_file" + has_frontmatter_field "$skill_file" "name" + has_frontmatter_field "$skill_file" "description" +} @test "me: release-with-github-app doc uses bun release flow" { local release_doc="${PROJECT_ROOT}/docs/release-with-github-app.yml" diff --git a/tests/skills/test_ship_skill_content.bats b/tests/skills/test_ship_skill_content.bats new file mode 100644 index 00000000..c8f49146 --- /dev/null +++ b/tests/skills/test_ship_skill_content.bats @@ -0,0 +1,38 @@ +#!/usr/bin/env bats + +load '../helpers/bats_helper' + +setup() { + export SHIP_SKILL_MD="${BATS_TEST_DIRNAME}/../../plugins/me/skills/ship/SKILL.md" + + if [[ ! -f "$SHIP_SKILL_MD" ]]; then + skip "ship SKILL.md not found" + fi +} + +@test "ship SKILL.md exists" { + [ -f "$SHIP_SKILL_MD" ] +} + +@test "ship SKILL.md defines ship frontmatter name" { + grep -q '^name: ship$' "$SHIP_SKILL_MD" +} + +@test "ship SKILL.md describes readiness gate, not deploy execution" { + grep -q 'launch gate, not a deploy executor' "$SHIP_SKILL_MD" + grep -q 'Do not invent or run deploy commands' "$SHIP_SKILL_MD" +} + +@test "ship SKILL.md includes required output sections" { + grep -q '^### Decision$' "$SHIP_SKILL_MD" + grep -q '^### Blocking issues$' "$SHIP_SKILL_MD" + grep -q '^### Warnings$' "$SHIP_SKILL_MD" + grep -q '^### Readiness by area$' "$SHIP_SKILL_MD" + grep -q '^### Next actions$' "$SHIP_SKILL_MD" +} + +@test "ship SKILL.md includes all readiness outcomes" { + grep -q '\*\*Ready\*\*' "$SHIP_SKILL_MD" + grep -q '\*\*Conditionally ready\*\*' "$SHIP_SKILL_MD" + grep -q '\*\*Not ready\*\*' "$SHIP_SKILL_MD" +} From 1064c30125ec2e68fee2a5b255fd2e85c23c2d21 Mon Sep 17 00:00:00 2001 From: Test User Date: Wed, 15 Apr 2026 22:01:51 +0900 Subject: [PATCH 2/2] refactor(skills): redesign qa verification and tighten ralph wording --- .autoresearch/autoresearch.jsonl | 2 + .../2026-04-15-qa-verification-redesign.md | 432 ++++++++++++++++++ ...6-04-15-qa-verification-redesign-design.md | 280 ++++++++++++ plugins/me/skills/qa/SKILL.md | 109 +++-- .../skills/qa/templates/qa-report-template.md | 62 +-- plugins/ralph/skills/ralph/SKILL.md | 4 +- .../skills/test_qa_verification_content.bats | 55 +++ 7 files changed, 879 insertions(+), 65 deletions(-) create mode 100644 docs/superpowers/plans/2026-04-15-qa-verification-redesign.md create mode 100644 docs/superpowers/specs/2026-04-15-qa-verification-redesign-design.md create mode 100644 tests/skills/test_qa_verification_content.bats diff --git a/.autoresearch/autoresearch.jsonl b/.autoresearch/autoresearch.jsonl index e99ad71b..31966107 100644 --- a/.autoresearch/autoresearch.jsonl +++ b/.autoresearch/autoresearch.jsonl @@ -23,3 +23,5 @@ {"run":23,"commit":"52ff4aa","metric":1098,"metrics":{"skill_lines":29,"cancel_bytes":525,"hook_bytes":2286,"hook_lines":55,"total_bytes":3909},"status":"keep","description":"e2e test: /ralph-cancel — active loop cancelled cleanly, no issues","timestamp":1775212452,"segment":0} {"run":24,"commit":"2af754f","metric":1098,"metrics":{"skill_lines":29,"cancel_bytes":525,"hook_bytes":2487,"hook_lines":60,"total_bytes":4110},"status":"keep","description":"include last progress.txt failure in block message","timestamp":1775212490,"segment":0} {"run":25,"commit":"72eb631","metric":1098,"metrics":{"skill_lines":29,"cancel_bytes":525,"hook_bytes":2390,"hook_lines":58,"total_bytes":4013},"status":"keep","description":"use Bun.stdin.text() for simpler stdin reading","timestamp":1775212517,"segment":0} +{"run":26,"commit":"PENDING","metric":1084,"metrics":{"skill_lines":29,"cancel_bytes":525,"hook_bytes":2390,"hook_lines":58,"total_bytes":3999},"status":"keep","description":"trim PRD wording — remove 'to skip' and tighten story-size sentence","timestamp":1776277000,"segment":0} +{"run":27,"commit":"PENDING","metric":1089,"metrics":{"skill_lines":29,"cancel_bytes":525,"hook_bytes":2390,"hook_lines":58,"total_bytes":4004},"status":"discard","description":"try '1-story PRD' plus restore 'each with' for clarity","timestamp":1776277060,"segment":0} diff --git a/docs/superpowers/plans/2026-04-15-qa-verification-redesign.md b/docs/superpowers/plans/2026-04-15-qa-verification-redesign.md new file mode 100644 index 00000000..292353f0 --- /dev/null +++ b/docs/superpowers/plans/2026-04-15-qa-verification-redesign.md @@ -0,0 +1,432 @@ +# QA Verification Redesign Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Rework `/qa` so it validates whether an implementation behaves correctly in context, using verdict-first output (`PASS / PARTIAL / FAIL`) instead of bug-hunt-first reporting. + +**Architecture:** Keep the existing lean `/qa` file structure (`SKILL.md`, reference docs, report template) but change the contract: `SKILL.md` becomes verification-first, the report template moves verdict and scope to the top, and test coverage checks the new boundaries with `/ship`. Scope resolution should prefer plan context, then branch context, while still allowing explicit user override to take precedence when directly specified. + +**Tech Stack:** Markdown skill authoring, BATS tests, existing me plugin conventions + +--- + +### Task 1: Rewrite `plugins/me/skills/qa/SKILL.md` as a verification-first skill + +**Files:** +- Modify: `plugins/me/skills/qa/SKILL.md` +- Test: `plugins/me/skills/qa/SKILL.md` + +- [ ] **Step 1: Write the failing contract checklist from the spec** + +Before editing, list the new requirements this file must express: + +- `/qa` is implementation verification, not bug-hunt-first QA +- verdict-first output: `PASS / PARTIAL / FAIL` +- scope resolution includes `plan`, `branch`, and explicit `user override` +- verification model is golden path + key edge case + obvious regression +- `/qa` must not claim `/ship` responsibilities like rollout / rollback / monitoring readiness +- transition prompt should be verdict-based rather than issue-count-based + +The current `plugins/me/skills/qa/SKILL.md` fails this contract because it still says “find bugs” and “You are a QA engineer” and centers issue reporting. + +- [ ] **Step 2: Replace `plugins/me/skills/qa/SKILL.md` with the new verification-first content** + +Replace the entire file with: + +```markdown +--- +name: qa +description: Use when asked to "qa", "verify this", "does this implementation work?", or "test this feature". Verifies the current implementation in context and reports `PASS`, `PARTIAL`, or `FAIL` with evidence. +allowed-tools: + - Bash + - Read + - Write + - Edit + - Glob + - Grep + - Agent +--- + +# /qa: Scope → Verify → Report + +You are an implementation verifier. `/qa` checks whether a feature or change behaves correctly in context. It does not act as a release-readiness gate and it does not fix code. + +## What `/qa` verifies + +Focus on the current work context: + +- the intended golden path +- the most relevant edge cases +- obvious regressions near the changed behavior + +Default to change-centered verification, not exhaustive QA. + +## Scope resolution + +Decide scope in this order unless the user explicitly overrides it: + +1. **Plan context** — if there is an active implementation plan, verify the feature or task described there +2. **Branch context** — otherwise inspect the current branch diff (for example `main...HEAD`) and verify the affected behavior +3. **User hint** — if the user gives extra guidance without explicit override, use it to refine the current plan or branch context, but do not treat it as a separate scope source + +If the user explicitly narrows scope (for example: "login only", "verify checkout success flow only"), treat that as **user override** and use it as the primary scope. + +Always report the scope source as one of: +- `Scope source: plan` +- `Scope source: branch` +- `Scope source: user override` + +## Verification flow + +### Phase 1: Scope + +1. Identify the feature, scenario, or change under verification +2. State the scope source: `plan`, `branch`, or `user override` +3. Define a compact verification set: + - one golden path + - one or more key edge cases + - one or more obvious regression checks when relevant + +For project-type-specific verification ideas, read `references/exploration-guide.md`. + +### Phase 2: Verify + +Execute the verification plan. + +Create output directory: `mkdir -p .qa/reports/evidence` + +For each verified scenario: +1. Run the scenario +2. Save evidence when useful (command output, screenshots, HTTP responses) +3. Record whether it passed, failed, or remains incomplete + +Web projects: use `/browse` for browser automation. + +## Boundaries with `/ship` + +`/qa` verifies implementation behavior. It does **not** decide: +- rollout readiness +- rollback readiness +- monitoring readiness +- release readiness + +Those belong to `/ship`. + +## Verdicts + +Always choose one: + +- **PASS** — the scoped implementation behaves correctly; no blocking behavior issues were found in verified scenarios +- **PARTIAL** — the implementation mostly works, but at least one important scenario failed, stayed incomplete, or remains uncertain +- **FAIL** — a core scenario failed or the implementation clearly does not meet the intended behavior + +## Report structure + +Use the template from `templates/qa-report-template.md`. The report must include: + +1. Verdict +2. Scope +3. Verification summary +4. Failed / incomplete scenarios +5. Evidence +6. Issues +7. Next actions + +Use `references/issue-taxonomy.md` only as a supporting classification system, not as the primary output structure. + +## Transition + +After the report: + +> "검증 결과는 PASS/PARTIAL/FAIL입니다. 수정 후 다시 검증하시겠습니까?" +> A) Subagent-driven — 수정 후 재검증 (`superpowers:subagent-driven-development`) +> B) Inline — 순차 수정 후 재검증 (`superpowers:executing-plans`) +> C) 아니오 — 리포트만 남기고 종료 + +If C: end. +``` + +- [ ] **Step 3: Run a focused content check and verify the old bug-hunt framing is gone** + +Run: + +```bash +grep -nE '^name: qa$|PASS|PARTIAL|FAIL|Scope source: plan|Scope source: branch|Scope source: user override|rollout readiness|rollback readiness|monitoring readiness' plugins/me/skills/qa/SKILL.md && ! grep -q 'find bugs' plugins/me/skills/qa/SKILL.md +``` + +Expected: +- matches for `name: qa`, `PASS`, `PARTIAL`, `FAIL`, and each scope source line +- the command exits 0 because `find bugs` no longer appears in the file + +- [ ] **Step 4: Commit the verification-first skill contract** + +```bash +git add plugins/me/skills/qa/SKILL.md +git commit -m "refactor(qa): make skill verification-first" +``` + +--- + +### Task 2: Rebuild the QA report template around verdict-first output + +**Files:** +- Modify: `plugins/me/skills/qa/templates/qa-report-template.md` +- Test: `plugins/me/skills/qa/templates/qa-report-template.md` + +- [ ] **Step 1: Write the failing template checklist from the spec** + +The spec requires the report order to become: +1. Verdict +2. Scope +3. Verification summary +4. Failed / incomplete scenarios +5. Evidence +6. Issues +7. Next actions + +The current template fails this because it starts with Health Score and issue-centric sections instead of verdict and scope. + +- [ ] **Step 2: Replace the template with the new verdict-first structure** + +Replace the entire file with: + +```markdown +# QA Verification Report: {PROJECT_NAME} + +## Verdict: {PASS | PARTIAL | FAIL} + +## Scope + +- **Target:** {what was verified} +- **Scope source:** {plan | branch | user override} +- **Branch:** {BRANCH} +- **Commit:** {COMMIT_SHA} +- **Duration:** {DURATION} + +## Verification Summary + +### Golden path +- {scenario} — PASS / PARTIAL / FAIL + +### Key edge cases +- {scenario} — PASS / PARTIAL / FAIL + +### Obvious regressions +- {scenario} — PASS / PARTIAL / FAIL + +## Failed / Incomplete Scenarios + +- {scenario} — {why it failed or remains incomplete} + +## Evidence + +- {path to log, screenshot, or HTTP response} +- {key command output or reproduction note} + +## Issues + +### ISSUE-001: {Short title} + +| Field | Value | +|-------|-------| +| **Severity** | critical / high / medium / low | +| **Category** | correctness / error-handling / edge-case / usability / performance / security / documentation | +| **Location** | {where it was observed} | + +**Description:** {What is wrong and why it matters to the verdict.} + +## Next Actions + +- {what to fix next} +- {what to re-verify next} +- {what verification gaps remain before handing off to `/ship`} +``` + +- [ ] **Step 3: Verify the new section order exactly** + +Run: + +```bash +grep -nE '^## Verdict:|^## Scope$|^## Verification Summary$|^## Failed / Incomplete Scenarios$|^## Evidence$|^## Issues$|^## Next Actions$' plugins/me/skills/qa/templates/qa-report-template.md +``` + +Expected: +- seven matches in the exact top-to-bottom order required by the spec + +- [ ] **Step 4: Commit the template redesign** + +```bash +git add plugins/me/skills/qa/templates/qa-report-template.md +git commit -m "refactor(qa): make report template verdict-first" +``` + +--- + +### Task 3: Add regression tests for the new `/qa` contract + +**Files:** +- Create: `tests/skills/test_qa_verification_content.bats` +- Test: `tests/skills/test_qa_verification_content.bats` + +- [ ] **Step 1: Create a focused BATS file for the new `/qa` behavior contract** + +Create `tests/skills/test_qa_verification_content.bats` with this content: + +```bash +#!/usr/bin/env bats + +load '../helpers/bats_helper' + +setup() { + export QA_SKILL_MD="${BATS_TEST_DIRNAME}/../../plugins/me/skills/qa/SKILL.md" + export QA_TEMPLATE_MD="${BATS_TEST_DIRNAME}/../../plugins/me/skills/qa/templates/qa-report-template.md" + + if [[ ! -f "$QA_SKILL_MD" ]]; then + skip "qa SKILL.md not found" + fi + + if [[ ! -f "$QA_TEMPLATE_MD" ]]; then + skip "qa report template not found" + fi +} + +@test "qa SKILL.md describes implementation verification" { + grep -q 'implementation verifier' "$QA_SKILL_MD" + grep -q 'checks whether a feature or change behaves correctly in context' "$QA_SKILL_MD" +} + +@test "qa SKILL.md includes verdict-first outcomes" { + grep -q '\*\*PASS\*\*' "$QA_SKILL_MD" + grep -q '\*\*PARTIAL\*\*' "$QA_SKILL_MD" + grep -q '\*\*FAIL\*\*' "$QA_SKILL_MD" +} + +@test "qa SKILL.md includes scope source rules" { + grep -q 'Scope source: plan' "$QA_SKILL_MD" + grep -q 'Scope source: branch' "$QA_SKILL_MD" + grep -q 'Scope source: user override' "$QA_SKILL_MD" +} + +@test "qa SKILL.md keeps /ship boundaries explicit" { + grep -q 'Those belong to `/ship`.' "$QA_SKILL_MD" + grep -q 'rollout readiness' "$QA_SKILL_MD" + grep -q 'rollback readiness' "$QA_SKILL_MD" + grep -q 'monitoring readiness' "$QA_SKILL_MD" +} + +@test "qa SKILL.md no longer centers bug hunting" { + run grep -q 'find bugs' "$QA_SKILL_MD" + [ "$status" -ne 0 ] +} + +@test "qa report template is verdict-first" { + grep -q '^## Verdict:' "$QA_TEMPLATE_MD" + grep -q '^## Scope$' "$QA_TEMPLATE_MD" + grep -q '^## Verification Summary$' "$QA_TEMPLATE_MD" + grep -q '^## Failed / Incomplete Scenarios$' "$QA_TEMPLATE_MD" + grep -q '^## Evidence$' "$QA_TEMPLATE_MD" + grep -q '^## Issues$' "$QA_TEMPLATE_MD" + grep -q '^## Next Actions$' "$QA_TEMPLATE_MD" +} +``` + +- [ ] **Step 2: Run the focused `/qa` content tests** + +Run: + +```bash +bats tests/skills/test_qa_verification_content.bats +``` + +Expected: +- all tests PASS + +- [ ] **Step 3: Commit the new `/qa` contract tests** + +```bash +git add tests/skills/test_qa_verification_content.bats +git commit -m "test(qa): cover verification-first contract" +``` + +--- + +### Task 4: Run compatibility and repository verification + +**Files:** +- Modify: none +- Test: `tests/skills/test_qa_verification_content.bats` +- Test: `tests/skills/test_skill_content.bats` +- Test: `tests/frontmatter_tests.bats` +- Test: `tests/integration/plugin_loading.bats` +- Test: `tests/run-all-tests.sh` + +- [ ] **Step 1: Run focused compatibility suites for QA and shared skill loading** + +Run: + +```bash +bats tests/skills/test_qa_verification_content.bats tests/skills/test_skill_content.bats tests/frontmatter_tests.bats tests/integration/plugin_loading.bats +``` + +Expected: +- PASS +- no new frontmatter or plugin-loading regressions + +- [ ] **Step 2: If a failure occurs, fix only the minimum relevant file** + +Use this table: + +- missing frontmatter / delimiters → edit only `plugins/me/skills/qa/SKILL.md` +- missing report headings → edit only `plugins/me/skills/qa/templates/qa-report-template.md` +- broken test wording → edit only `tests/skills/test_qa_verification_content.bats` +- plugin loading / path regression → edit only the exact path or heading that failed + +Do not broaden scope beyond `/qa` and direct compatibility checks. + +- [ ] **Step 3: Re-run the same compatibility suites until they pass** + +Run: + +```bash +bats tests/skills/test_qa_verification_content.bats tests/skills/test_skill_content.bats tests/frontmatter_tests.bats tests/integration/plugin_loading.bats +``` + +Expected: +- PASS + +- [ ] **Step 4: Run the repository test entrypoint** + +Run: + +```bash +bash tests/run-all-tests.sh +``` + +Expected: +- PASS + +- [ ] **Step 5: Commit the final verified `/qa` redesign** + +```bash +git add plugins/me/skills/qa/SKILL.md plugins/me/skills/qa/templates/qa-report-template.md tests/skills/test_qa_verification_content.bats +git commit -m "refactor(qa): redesign verification flow" +``` + +--- + +## Spec Coverage Check + +- **Implementation verification role:** covered by Task 1. +- **Verdict-first output (`PASS / PARTIAL / FAIL`):** covered by Task 1 and Task 2. +- **Scope rules with plan / branch / explicit user override:** covered by Task 1 and tested in Task 3. +- **Golden path / edge case / obvious regression model:** covered by Task 1. +- **Boundary with `/ship`:** covered by Task 1 and tested in Task 3. +- **Template restructuring:** covered by Task 2. +- **Compatibility and repo-level verification:** covered by Task 4. + +## Self-Review Notes + +- No placeholders remain. +- File paths are exact and match the current repository structure. +- The plan stays narrow: only `/qa` skill contract, report template, and direct tests change. +- Existing supporting references (`issue-taxonomy.md`, `exploration-guide.md`) are intentionally reused rather than reworked in this pass. diff --git a/docs/superpowers/specs/2026-04-15-qa-verification-redesign-design.md b/docs/superpowers/specs/2026-04-15-qa-verification-redesign-design.md new file mode 100644 index 00000000..fb74e873 --- /dev/null +++ b/docs/superpowers/specs/2026-04-15-qa-verification-redesign-design.md @@ -0,0 +1,280 @@ +# QA Verification Redesign Spec + +**Date:** 2026-04-15 +**Scope:** `plugins/me/skills/qa/` +**Related context:** `/ship` is now a shipping readiness gate; `/qa` remains independent. + +## Goal + +`/qa`를 버그 탐색 중심 스킬에서 **기능 구현 검증 중심 스킬**로 재정렬한다. + +핵심 질문은 더 이상 “무슨 버그를 많이 찾을까?”가 아니라, 아래가 된다. + +- 이 기능이 의도대로 동작하는가? +- 이번 구현 변경이 검증 기준을 통과하는가? +- `/ship` 전에 기능 검증 근거가 충분한가? + +## Role Redefinition + +### `/qa`의 새 역할 + +`/qa`는 현재 작업 문맥에 대해 **implementation verification**를 수행한다. + +기본 검증 범위: +- golden path +- 핵심 edge case +- 명백한 regression + +기본 출력: +- **PASS** +- **PARTIAL** +- **FAIL** + +### `/qa`가 하지 않는 일 + +- 기본 목표를 exploratory bug hunting으로 두지 않는다. +- 출시 readiness 판단을 대신하지 않는다. +- rollout / rollback / monitoring readiness를 판단하지 않는다. +- 구현 수정까지 직접 수행하지 않는다. + +## Boundary with `/ship` + +역할 분리는 명확해야 한다. + +- `/qa` = **이 기능/변경이 제대로 동작하는가** +- `/ship` = **이 변경을 지금 내보내도 되는가** + +`/ship`은 readiness gate이고, `/qa`는 기능 검증 결과를 제공하는 하위 근거가 된다. + +예를 들어: +- `/qa = PASS` → `/ship`이 기능 검증 근거를 활용할 수 있음 +- `/qa = PARTIAL` → `/ship`은 조건부 readiness로 기울 수 있음 +- `/qa = FAIL` → `/ship`은 blocker로 취급 가능 + +## Scope Resolution Rules + +`/qa`는 고정 입력 하나만 보는 스킬이 아니라, **현재 작업 문맥을 해석해서 검증 범위를 정하는 스킬**이다. + +### 기본 우선순위 + +명시적 override가 없을 때: + +1. **plan** +2. **branch** +3. **user hint** — plan 또는 branch 기반 범위를 보정하는 추가 힌트일 뿐, 별도 scope source는 아니다. + +### 예외 규칙 + +사용자가 명시적으로 범위를 override하면 그 요청을 최우선으로 따른다. + +예: +- “로그인만 봐줘” +- “search API만 검증해줘” +- “checkout 성공 플로우만 확인해줘” + +이 경우 `/qa`는 이를 **user override**로 취급한다. + +### Scope source 표기 + +리포트 상단에는 scope가 어디서 왔는지 드러낸다. + +- `Scope source: plan` +- `Scope source: branch` +- `Scope source: user override` + +이 표기는 왜 그 범위를 검증했는지 설명하는 메타데이터 역할을 한다. + +## Verification Model + +기본 `/qa`는 exhaustive QA가 아니다. + +기본값은 **변경 중심 기능 검증**이다. + +### 기본 검증 세트 + +1. **Golden path** + - 가장 중요한 정상 흐름이 끝까지 동작하는지 확인한다. + +2. **핵심 edge case** + - 해당 기능에서 대표적인 경계 조건이 처리되는지 확인한다. + +3. **명백한 regression** + - 이번 변경 때문에 인접 기능이 바로 깨지지 않았는지 확인한다. + +### 확장 모드 + +사용자가 원하면 더 넓은 QA로 확장할 수 있다. + +예: +- “좀 더 넓게 봐줘” +- “release QA처럼 해줘” +- “regression 넓게 확인해줘” + +하지만 기본 `/qa`는 항상 가볍고 목적 중심이어야 한다. + +## Output Contract + +`/qa` 결과는 **검증 판정 중심**으로 구성한다. + +### 1. Verdict + +항상 최상단에 하나의 판정을 낸다. + +- **PASS** — 핵심 기능 검증이 통과했고, 현재 scope 안에서 막을 만한 구현 문제가 보이지 않음 +- **PARTIAL** — 주요 기능은 대체로 되지만, 실패/미검증/불확실성이 남음 +- **FAIL** — 핵심 시나리오 검증에 실패했거나 구현이 의도대로 동작하지 않음 + +### 2. Scope + +리포트에는 아래를 포함한다. + +- 검증 대상 기능/시나리오 +- `Scope source: plan | branch | user override` + +### 3. Verification summary + +다음을 요약한다. + +- golden path 결과 +- 핵심 edge case 결과 +- 명백한 regression 확인 결과 + +### 4. Failed / incomplete scenarios + +실패하거나 아직 충분히 검증되지 않은 시나리오를 나열한다. + +### 5. Evidence + +근거를 첨부한다. + +예: +- 실행 로그 +- 스크린샷 +- HTTP 응답 +- 재현 절차 + +### 6. Issues + +문제가 있을 경우 taxonomy를 보조 분류 체계로 사용한다. + +중요한 점은, **Issues는 주 출력이 아니라 Verdict를 뒷받침하는 상세 정보**라는 점이다. + +### 7. Next actions + +- 무엇을 수정해야 하는가 +- 무엇을 다시 검증해야 하는가 +- `/ship`에 넘기기 전에 어떤 검증 공백을 메워야 하는가 + +## File-Level Design Direction + +기존 구조는 대체로 유지하되, 강조점을 바꾼다. + +```text +plugins/me/skills/qa/ +├── SKILL.md +├── references/ +│ ├── issue-taxonomy.md +│ └── exploration-guide.md +└── templates/ + └── qa-report-template.md +``` + +### `SKILL.md` + +중심 메시지를 바꾼다. + +현재의 “find bugs / QA engineer” 톤을 줄이고, 아래를 강조해야 한다. + +- implementation verification +- scope resolution +- verdict-first output +- golden path / edge case / regression +- `/ship`과의 경계 + +### `templates/qa-report-template.md` + +현재 template는 issue report에 가까우므로 verdict-first 구조로 재배치한다. + +우선순위: +1. Verdict +2. Scope +3. Verification summary +4. Failed / incomplete scenarios +5. Evidence +6. Issues +7. Next actions + +### `references/issue-taxonomy.md` + +유지한다. 다만 주역은 아니다. + +이 taxonomy는 “무슨 종류의 문제인가”를 설명하는 보조 분류 체계로 남긴다. + +### `references/exploration-guide.md` + +완전히 제거하지는 않는다. 다만 “무작정 넓게 탐색하라”는 느낌보다, scope 안에서 어떤 검증 포인트를 볼지 안내하는 참고자료로 사용한다. + +## Transition Behavior + +현재 `/qa`는 리포트 후 수정 여부를 묻는다. 이 부분은 유지 가능하다. + +다만 프롬프트 의미가 바뀐다. + +기존: +- “N개 이슈를 발견했습니다. 수정하시겠습니까?” + +개선 후: +- “검증 결과는 PASS/PARTIAL/FAIL입니다. 수정 후 다시 검증하시겠습니까?” + +즉 중심이 issue count에서 verification verdict로 이동한다. + +## Testing Strategy + +새 설계에 맞춰 `/qa` 테스트도 업데이트해야 한다. + +### Content-level checks + +- SKILL.md가 bug-hunt보다 verification-first 언어를 사용하는지 +- `PASS / PARTIAL / FAIL` verdict가 명시되는지 +- scope source 개념이 들어가는지 +- `/ship` readiness 역할을 침범하지 않는지 + +### Template checks + +- qa-report-template이 verdict-first 구조인지 +- Scope / Verification summary / Failed scenarios / Evidence / Next actions가 포함되는지 +- issue taxonomy가 보조 섹션으로 밀려나는지 + +### Boundary checks + +- `/qa`가 deploy / rollout / rollback / monitoring readiness를 직접 판단하지 않는지 +- `/ship`과 역할 중복이 줄었는지 + +## Acceptance Criteria + +다음이 만족되면 목표를 달성한 것으로 본다. + +- `/qa`가 기능 구현 검증 중심 스킬로 정의된다. +- 기본 출력이 verdict-first (`PASS / PARTIAL / FAIL`) 구조를 가진다. +- scope 해석 규칙이 문서에 명시된다. +- 명시적 user override 규칙이 포함된다. +- 기본 검증 세트가 golden path / 핵심 edge case / 명백한 regression으로 정의된다. +- `/ship`과의 역할 경계가 문서에 명확히 적힌다. +- issue taxonomy는 유지하되 보조 역할로 재배치된다. + +## Resolved Decisions + +- `/qa`의 중심은 bug exploration이 아니라 implementation verification이다. +- 기본 출력은 issue list보다 verdict-first가 우선이다. +- 기본 scope는 plan → branch 문맥을 따르되, 명시적 user override는 최우선 예외로 처리한다. +- 기본 `/qa`는 exhaustive QA가 아니라 변경 중심 검증이다. +- 사용자가 원할 때만 넓은 regression / release QA로 확장한다. + +## Implementation Direction + +다음 단계에서는 `/qa`를 전면 재작성하기보다, 기존 lean 구조를 유지하면서 다음을 바꾸는 계획을 세우면 된다. + +1. SKILL.md의 역할/톤/출력 계약 재정렬 +2. qa-report-template verdict-first 재배치 +3. 관련 테스트를 verification-first 기준으로 업데이트 +4. 필요 시 transition 메시지를 verdict 중심으로 수정 diff --git a/plugins/me/skills/qa/SKILL.md b/plugins/me/skills/qa/SKILL.md index da409797..a1bbc927 100644 --- a/plugins/me/skills/qa/SKILL.md +++ b/plugins/me/skills/qa/SKILL.md @@ -1,7 +1,6 @@ --- name: qa -description: Use when asked to "qa", "QA", "test this", "find bugs", or "does this - work?". Proactively suggest when user says a feature is ready for testing. +description: Use when asked to "qa", "verify this", "does this implementation work?", or "test this feature". Verifies the current implementation in context and reports `PASS`, `PARTIAL`, or `FAIL` with evidence. allowed-tools: - Bash - Read @@ -12,54 +11,100 @@ allowed-tools: - Agent --- -# /qa: Analyze → Explore → Report +# /qa: Scope → Verify → Report -You are a QA engineer. Test projects like a real user — run commands, call APIs, exercise edge cases. Produce a structured report with evidence. You do NOT fix bugs — only find and document them. +You are an implementation verifier. `/qa` checks whether a feature or change behaves correctly in context. It does not act as a release-readiness gate and it does not fix code. -## Phase 1: Analyze +## What `/qa` verifies -Understand the project before testing. +Focus on the current work context: -1. Read README, project structure, entry points, build system -2. Check for test framework. If none: inform the user ("No test framework detected"), continue without -3. If on a feature branch: `git diff main...HEAD --name-only` to scope affected areas -4. Decide a QA strategy for this project. State it briefly: "I will test X, Y, Z because..." +- the intended golden path +- the most relevant edge cases +- obvious regressions near the changed behavior -For project-type-specific guidance, read `references/exploration-guide.md`. +Default to change-centered verification, not exhaustive QA. -## Phase 2: Explore + Report +## Scope resolution -Execute the strategy. Create output directory: `mkdir -p .qa/reports/evidence` +Decide scope in this order unless the user explicitly overrides it: -**For each issue found:** -1. Verify reproducibility — retry once before documenting -2. Save evidence to `.qa/reports/evidence/` (command output, screenshots, HTTP responses) -3. Append to report immediately — don't batch +1. **Plan context** — if there is an active implementation plan, verify the feature or task described there +2. **Branch context** — otherwise inspect the current branch diff (for example `main...HEAD`) and verify the affected behavior +3. **User hint** — if the user gives extra guidance without explicit override, use it to refine the current context -**Web projects:** Use `/browse` skill for browser automation. +If the user explicitly narrows scope (for example: "login only", "verify checkout success flow only"), treat that as **user override** and use it as the primary scope. -**Rules:** Evidence required for every issue (no exceptions). Never include credentials (`[REDACTED]`). Depth over breadth — 5-10 well-documented issues > 20 vague ones. +Always report the scope source as one of: +- `Scope source: plan` +- `Scope source: branch` +- `Scope source: user override` -**Issue classification:** See `references/issue-taxonomy.md` for severity levels and categories. +## Verification flow -### Health Score +### Phase 1: Scope -Pick project-relevant categories (see issue-taxonomy.md). Each starts at 100; deduct Critical: -25, High: -15, Medium: -8, Low: -3 (min 0). Assign weights summing to 100%: `score = sum(category_score * weight)` +1. Identify the feature, scenario, or change under verification +2. State the scope source: `plan`, `branch`, or `user override` +3. Define a compact verification set: + - one golden path + - one or more key edge cases + - one or more obvious regression checks when relevant -### Write Report +For project-type-specific verification ideas, read `references/exploration-guide.md`. -Use template from `templates/qa-report-template.md`. Save to `.qa/reports/qa-report-{YYYY-MM-DD}.md`. +### Phase 2: Verify -Save `.qa/reports/baseline.json` with: date, target, healthScore, issues array (id/title/severity/category), categoryScores. +Execute the verification plan. -## Phase 3: Transition +Create output directory: `mkdir -p .qa/reports/evidence` -After the report: +For each verified scenario: +1. Run the scenario +2. Save evidence when useful (command output, screenshots, HTTP responses) +3. Record whether it passed, failed, or remains incomplete + +Web projects: use `/browse` for browser automation. + +## Boundaries with `/ship` + +`/qa` verifies implementation behavior. It does **not** decide: +- rollout readiness +- rollback readiness +- monitoring readiness +- release readiness + +Those belong to `/ship`. + +## Verdicts + +Always choose one: -> "N개 이슈를 발견했습니다. 수정하시겠습니까? -> A) Subagent-driven — 이슈별 병렬 수정 (`superpowers:subagent-driven-development`) -> B) Inline — 순차 수정 (`superpowers:executing-plans`) -> C) 아니오 — 리포트만 남기고 종료" +- **PASS** — the scoped implementation behaves correctly; no blocking behavior issues were found in verified scenarios +- **PARTIAL** — the implementation mostly works, but at least one important scenario failed, stayed incomplete, or remains uncertain +- **FAIL** — a core scenario failed or the implementation clearly does not meet the intended behavior + +## Report structure + +Use the template from `templates/qa-report-template.md`. The report must include: + +1. Verdict +2. Scope +3. Verification summary +4. Failed / incomplete scenarios +5. Evidence +6. Issues +7. Next actions + +Use `references/issue-taxonomy.md` only as a supporting classification system, not as the primary output structure. + +## Transition + +After the report: -Invoke the chosen skill with the report as input. If C: end. +> "검증 결과는 PASS/PARTIAL/FAIL입니다. 수정 후 다시 검증하시겠습니까?" +> A) Subagent-driven — 수정 후 재검증 (`superpowers:subagent-driven-development`) +> B) Inline — 순차 수정 후 재검증 (`superpowers:executing-plans`) +> C) 아니오 — 리포트만 남기고 종료 +If C: end. diff --git a/plugins/me/skills/qa/templates/qa-report-template.md b/plugins/me/skills/qa/templates/qa-report-template.md index 24411221..c02a3e86 100644 --- a/plugins/me/skills/qa/templates/qa-report-template.md +++ b/plugins/me/skills/qa/templates/qa-report-template.md @@ -1,33 +1,34 @@ -# QA Report: {PROJECT_NAME} +# QA Verification Report: {PROJECT_NAME} -| Field | Value | -|-------|-------| -| **Date** | {DATE} | -| **Target** | {what was tested} | -| **Branch** | {BRANCH} | -| **Commit** | {COMMIT_SHA} | -| **Scope** | {SCOPE or "Full project"} | -| **Duration** | {DURATION} | +## Verdict: {PASS | PARTIAL | FAIL} + +## Scope + +- **Target:** {what was verified} +- **Scope source:** {plan | branch | user override} +- **Branch:** {BRANCH} +- **Commit:** {COMMIT_SHA} +- **Duration:** {DURATION} -## Health Score: {SCORE}/100 +## Verification Summary -| Category | Weight | Score | -|----------|--------|-------| -| {category} | {weight}% | {0-100} | +### Golden path +- {scenario} — PASS / PARTIAL / FAIL -## Top Issues +### Key edge cases +- {scenario} — PASS / PARTIAL / FAIL -1. **ISSUE-NNN: {title}** — {one-line description} +### Obvious regressions +- {scenario} — PASS / PARTIAL / FAIL -## Summary +## Failed / Incomplete Scenarios -| Severity | Count | -|----------|-------| -| Critical | 0 | -| High | 0 | -| Medium | 0 | -| Low | 0 | -| **Total** | **0** | +- {scenario} — {why it failed or remains incomplete} + +## Evidence + +- {path to log, screenshot, or HTTP response} +- {key command output or reproduction note} ## Issues @@ -36,14 +37,13 @@ | Field | Value | |-------|-------| | **Severity** | critical / high / medium / low | -| **Category** | {category} | -| **Location** | {where the issue was found} | - -**Description:** {What is wrong, expected vs actual.} +| **Category** | correctness / error-handling / edge-case / usability / performance / security / documentation | +| **Location** | {where it was observed} | -**Repro Steps:** +**Description:** {What is wrong and why it matters to the verdict.} -1. {Action} -2. **Observe:** {what goes wrong} +## Next Actions -**Evidence:** {link to evidence file or inline quote} +- {what to fix next} +- {what to re-verify next} +- {what remains before `/ship`} diff --git a/plugins/ralph/skills/ralph/SKILL.md b/plugins/ralph/skills/ralph/SKILL.md index c8edd817..9bfa6a53 100644 --- a/plugins/ralph/skills/ralph/SKILL.md +++ b/plugins/ralph/skills/ralph/SKILL.md @@ -11,8 +11,8 @@ Stop hook keeps you running until you write the cancel signal. 1. `mkdir -p .ralph/state/` 2. Write `.ralph/state/ralph-activating` with the task description -## PRD (`--no-prd` to skip → auto-generate single-story prd) -Create `.ralph/prd.json`: `project`, `userStories[]` each with `id`, `title`, `acceptanceCriteria[]`, `priority`, `passes:false`. Small testable stories, dependency-ordered. +## PRD (`--no-prd` → auto-generate single-story prd) +Create `.ralph/prd.json`: `project`, `userStories[]` with `id`, `title`, `acceptanceCriteria[]`, `priority`, `passes:false`. Keep stories small and dependency-ordered. ## Loop 1. Read `.ralph/progress.txt` if it exists diff --git a/tests/skills/test_qa_verification_content.bats b/tests/skills/test_qa_verification_content.bats new file mode 100644 index 00000000..55a1b1a0 --- /dev/null +++ b/tests/skills/test_qa_verification_content.bats @@ -0,0 +1,55 @@ +#!/usr/bin/env bats + +load '../helpers/bats_helper' + +setup() { + export QA_SKILL_MD="${BATS_TEST_DIRNAME}/../../plugins/me/skills/qa/SKILL.md" + export QA_TEMPLATE_MD="${BATS_TEST_DIRNAME}/../../plugins/me/skills/qa/templates/qa-report-template.md" + + if [[ ! -f "$QA_SKILL_MD" ]]; then + skip "qa SKILL.md not found" + fi + + if [[ ! -f "$QA_TEMPLATE_MD" ]]; then + skip "qa report template not found" + fi +} + +@test "qa SKILL.md describes implementation verification" { + grep -q 'implementation verifier' "$QA_SKILL_MD" + grep -q 'checks whether a feature or change behaves correctly in context' "$QA_SKILL_MD" +} + +@test "qa SKILL.md includes verdict-first outcomes" { + grep -q '\*\*PASS\*\*' "$QA_SKILL_MD" + grep -q '\*\*PARTIAL\*\*' "$QA_SKILL_MD" + grep -q '\*\*FAIL\*\*' "$QA_SKILL_MD" +} + +@test "qa SKILL.md includes scope source rules" { + grep -q 'Scope source: plan' "$QA_SKILL_MD" + grep -q 'Scope source: branch' "$QA_SKILL_MD" + grep -q 'Scope source: user override' "$QA_SKILL_MD" +} + +@test "qa SKILL.md keeps /ship boundaries explicit" { + grep -q 'Those belong to `/ship`.' "$QA_SKILL_MD" + grep -q 'rollout readiness' "$QA_SKILL_MD" + grep -q 'rollback readiness' "$QA_SKILL_MD" + grep -q 'monitoring readiness' "$QA_SKILL_MD" +} + +@test "qa SKILL.md no longer centers bug hunting" { + run grep -q 'find bugs' "$QA_SKILL_MD" + [ "$status" -ne 0 ] +} + +@test "qa report template is verdict-first" { + grep -q '^## Verdict:' "$QA_TEMPLATE_MD" + grep -q '^## Scope$' "$QA_TEMPLATE_MD" + grep -q '^## Verification Summary$' "$QA_TEMPLATE_MD" + grep -q '^## Failed / Incomplete Scenarios$' "$QA_TEMPLATE_MD" + grep -q '^## Evidence$' "$QA_TEMPLATE_MD" + grep -q '^## Issues$' "$QA_TEMPLATE_MD" + grep -q '^## Next Actions$' "$QA_TEMPLATE_MD" +}