diff --git a/README.md b/README.md index 32c182f..f6e06d1 100644 --- a/README.md +++ b/README.md @@ -45,3 +45,33 @@ pytest -q 数据契约、导出命令和服务器 smoke-test 入口见 [`docs/SFT_BASELINE.md`](docs/SFT_BASELINE.md);公共 seam 与 RL 接入条件见 [`docs/RL_FRAMEWORK.md`](docs/RL_FRAMEWORK.md)。 + +## 文档与产物结构 + +`docs/` 存放项目文档,`artifacts/generated/` 存放可由脚本重新生成的分析产物。 + +```text +docs/ 文档:设计说明、分析结论、阶段报告 +├── design/ 设计文档(data_contract / prompt_interface) +├── phase_reports/ 阶段报告 +├── data_alignment.md +├── prompt_length_analysis.md +├── SFT_BASELINE.md / RL_FRAMEWORK.md / 新数据集运行说明.md + +artifacts/generated/ 生成产物:统计结果、对齐报告等可复现输出 +├── alignment/ 对齐报告(data_alignment_report.json/.md) +└── prompt_stats/ 统计产物 + ├── canonical_id_baseline/ 旧 canonical-id 统计(baseline) + └── choice_id/ choice-protocol 新统计 +``` + +关键文档: + +| 文档 | 内容 | +|---|---| +| [`docs/design/data_contract.md`](docs/design/data_contract.md) | canonical category_id 数据契约(registry/corpus/target、身份策略、训练入口规则) | +| [`docs/design/prompt_interface.md`](docs/design/prompt_interface.md) | prompt-facing choice protocol(choice id ↔ canonical id、共享解码层) | +| [`docs/prompt_length_analysis.md`](docs/prompt_length_analysis.md) | prompt token 分析(old∽new、Stage1 收益、worst-case 2689/32768) | +| [`docs/data_alignment.md`](docs/data_alignment.md) | dataset ↔ corpus/standard 对齐人工结论 | + +`docs/reports/` 为本地 gitignored 调研/smoke scratch,不入库。 diff --git a/artifacts/data_alignment_report.json b/artifacts/generated/alignment/data_alignment_report.json similarity index 100% rename from artifacts/data_alignment_report.json rename to artifacts/generated/alignment/data_alignment_report.json diff --git a/artifacts/data_alignment_report.md b/artifacts/generated/alignment/data_alignment_report.md similarity index 100% rename from artifacts/data_alignment_report.md rename to artifacts/generated/alignment/data_alignment_report.md diff --git a/artifacts/prompt_stats_finance.json b/artifacts/generated/prompt_stats/canonical_id_baseline/prompt_stats_finance.json similarity index 100% rename from artifacts/prompt_stats_finance.json rename to artifacts/generated/prompt_stats/canonical_id_baseline/prompt_stats_finance.json diff --git a/artifacts/prompt_stats_shougang.json b/artifacts/generated/prompt_stats/canonical_id_baseline/prompt_stats_shougang.json similarity index 100% rename from artifacts/prompt_stats_shougang.json rename to artifacts/generated/prompt_stats/canonical_id_baseline/prompt_stats_shougang.json diff --git a/artifacts/prompt_token_stats_finance.json b/artifacts/generated/prompt_stats/canonical_id_baseline/prompt_token_stats_finance.json similarity index 100% rename from artifacts/prompt_token_stats_finance.json rename to artifacts/generated/prompt_stats/canonical_id_baseline/prompt_token_stats_finance.json diff --git a/artifacts/prompt_token_stats_infra.json b/artifacts/generated/prompt_stats/canonical_id_baseline/prompt_token_stats_infra.json similarity index 100% rename from artifacts/prompt_token_stats_infra.json rename to artifacts/generated/prompt_stats/canonical_id_baseline/prompt_token_stats_infra.json diff --git a/artifacts/prompt_token_stats_pers_info.json b/artifacts/generated/prompt_stats/canonical_id_baseline/prompt_token_stats_pers_info.json similarity index 100% rename from artifacts/prompt_token_stats_pers_info.json rename to artifacts/generated/prompt_stats/canonical_id_baseline/prompt_token_stats_pers_info.json diff --git a/artifacts/prompt_token_stats_shougang.json b/artifacts/generated/prompt_stats/canonical_id_baseline/prompt_token_stats_shougang.json similarity index 100% rename from artifacts/prompt_token_stats_shougang.json rename to artifacts/generated/prompt_stats/canonical_id_baseline/prompt_token_stats_shougang.json diff --git a/artifacts/prompt_token_stats_summary.json b/artifacts/generated/prompt_stats/canonical_id_baseline/prompt_token_stats_summary.json similarity index 100% rename from artifacts/prompt_token_stats_summary.json rename to artifacts/generated/prompt_stats/canonical_id_baseline/prompt_token_stats_summary.json diff --git a/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_finance.json b/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_finance.json new file mode 100644 index 0000000..c36a95a --- /dev/null +++ b/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_finance.json @@ -0,0 +1,139 @@ +{ + "tokenizer": "Qwen/Qwen2.5-7B-Instruct", + "splits": { + "train": { + "stage1": { + "tokens": { + "rows": 403, + "max": 2553, + "p95": 2543, + "mean": 2535.4 + }, + "chars": { + "rows": 403, + "max": 5048, + "p95": 5033, + "mean": 5023.1 + } + }, + "stage2": { + "tokens": { + "rows": 403, + "max": 740, + "p95": 560, + "mean": 494.0 + }, + "chars": { + "rows": 403, + "max": 1579, + "p95": 1261, + "mean": 1163.3 + } + } + }, + "val": { + "stage1": { + "tokens": { + "rows": 69, + "max": 2543, + "p95": 2538, + "mean": 2534.6 + }, + "chars": { + "rows": 69, + "max": 5031, + "p95": 5027, + "mean": 5020.7 + } + }, + "stage2": { + "tokens": { + "rows": 69, + "max": 556, + "p95": 554, + "mean": 453.7 + }, + "chars": { + "rows": 69, + "max": 1254, + "p95": 1249, + "mean": 1097.1 + } + } + }, + "test": { + "stage1": { + "tokens": { + "rows": 57, + "max": 2539, + "p95": 2537, + "mean": 2535.3 + }, + "chars": { + "rows": 57, + "max": 5030, + "p95": 5026, + "mean": 5022.3 + } + }, + "stage2": { + "tokens": { + "rows": 57, + "max": 560, + "p95": 558, + "mean": 495.1 + }, + "chars": { + "rows": 57, + "max": 1258, + "p95": 1257, + "mean": 1160.8 + } + } + } + }, + "prompt_only": { + "train": { + "stage1": { + "rows": 403, + "max": 2534, + "p95": 2524, + "mean": 2515.9 + }, + "stage2": { + "rows": 403, + "max": 730, + "p95": 550, + "mean": 484.0 + } + }, + "val": { + "stage1": { + "rows": 69, + "max": 2523, + "p95": 2518, + "mean": 2514.6 + }, + "stage2": { + "rows": 69, + "max": 546, + "p95": 544, + "mean": 443.7 + } + }, + "test": { + "stage1": { + "rows": 57, + "max": 2520, + "p95": 2518, + "mean": 2515.8 + }, + "stage2": { + "rows": 57, + "max": 550, + "p95": 548, + "mean": 485.1 + } + } + } +} \ No newline at end of file diff --git a/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_infra.json b/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_infra.json new file mode 100644 index 0000000..a5e119c --- /dev/null +++ b/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_infra.json @@ -0,0 +1,139 @@ +{ + "tokenizer": "Qwen/Qwen2.5-7B-Instruct", + "splits": { + "train": { + "stage1": { + "tokens": { + "rows": 40, + "max": 2588, + "p95": 2587, + "mean": 2581.6 + }, + "chars": { + "rows": 40, + "max": 4978, + "p95": 4977, + "mean": 4966.6 + } + }, + "stage2": { + "tokens": { + "rows": 40, + "max": 316, + "p95": 315, + "mean": 294.7 + }, + "chars": { + "rows": 40, + "max": 884, + "p95": 883, + "mean": 854.8 + } + } + }, + "val": { + "stage1": { + "tokens": { + "rows": 15, + "max": 2589, + "p95": 2583, + "mean": 2579.9 + }, + "chars": { + "rows": 15, + "max": 4973, + "p95": 4973, + "mean": 4965.1 + } + }, + "stage2": { + "tokens": { + "rows": 15, + "max": 311, + "p95": 305, + "mean": 301.9 + }, + "chars": { + "rows": 15, + "max": 876, + "p95": 876, + "mean": 868.1 + } + } + }, + "test": { + "stage1": { + "tokens": { + "rows": 9, + "max": 2593, + "p95": 2590, + "mean": 2588.1 + }, + "chars": { + "rows": 9, + "max": 4982, + "p95": 4980, + "mean": 4976.1 + } + }, + "stage2": { + "tokens": { + "rows": 9, + "max": 321, + "p95": 318, + "mean": 316.1 + }, + "chars": { + "rows": 9, + "max": 888, + "p95": 886, + "mean": 882.1 + } + } + } + }, + "prompt_only": { + "train": { + "stage1": { + "rows": 40, + "max": 2570, + "p95": 2569, + "mean": 2562.7 + }, + "stage2": { + "rows": 40, + "max": 306, + "p95": 305, + "mean": 284.7 + } + }, + "val": { + "stage1": { + "rows": 15, + "max": 2571, + "p95": 2565, + "mean": 2561.9 + }, + "stage2": { + "rows": 15, + "max": 301, + "p95": 295, + "mean": 291.9 + } + }, + "test": { + "stage1": { + "rows": 9, + "max": 2575, + "p95": 2572, + "mean": 2570.1 + }, + "stage2": { + "rows": 9, + "max": 311, + "p95": 308, + "mean": 306.1 + } + } + } +} \ No newline at end of file diff --git a/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_pers_info.json b/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_pers_info.json new file mode 100644 index 0000000..34b244a --- /dev/null +++ b/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_pers_info.json @@ -0,0 +1,139 @@ +{ + "tokenizer": "Qwen/Qwen2.5-7B-Instruct", + "splits": { + "train": { + "stage1": { + "tokens": { + "rows": 140, + "max": 266, + "p95": 260, + "mean": 255.5 + }, + "chars": { + "rows": 140, + "max": 747, + "p95": 740, + "mean": 730.0 + } + }, + "stage2": { + "tokens": { + "rows": 140, + "max": 202, + "p95": 196, + "mean": 191.3 + }, + "chars": { + "rows": 140, + "max": 707, + "p95": 700, + "mean": 689.0 + } + } + }, + "val": { + "stage1": { + "tokens": { + "rows": 18, + "max": 260, + "p95": 259, + "mean": 256.6 + }, + "chars": { + "rows": 18, + "max": 742, + "p95": 742, + "mean": 735.7 + } + }, + "stage2": { + "tokens": { + "rows": 18, + "max": 196, + "p95": 195, + "mean": 192.7 + }, + "chars": { + "rows": 18, + "max": 702, + "p95": 702, + "mean": 695.6 + } + } + }, + "test": { + "stage1": { + "tokens": { + "rows": 18, + "max": 259, + "p95": 258, + "mean": 255.4 + }, + "chars": { + "rows": 18, + "max": 739, + "p95": 738, + "mean": 732.9 + } + }, + "stage2": { + "tokens": { + "rows": 18, + "max": 196, + "p95": 195, + "mean": 191.9 + }, + "chars": { + "rows": 18, + "max": 698, + "p95": 698, + "mean": 692.3 + } + } + } + }, + "prompt_only": { + "train": { + "stage1": { + "rows": 140, + "max": 248, + "p95": 241, + "mean": 237.3 + }, + "stage2": { + "rows": 140, + "max": 192, + "p95": 186, + "mean": 181.3 + } + }, + "val": { + "stage1": { + "rows": 18, + "max": 242, + "p95": 241, + "mean": 238.6 + }, + "stage2": { + "rows": 18, + "max": 186, + "p95": 185, + "mean": 182.7 + } + }, + "test": { + "stage1": { + "rows": 18, + "max": 241, + "p95": 240, + "mean": 237.3 + }, + "stage2": { + "rows": 18, + "max": 186, + "p95": 185, + "mean": 181.9 + } + } + } +} \ No newline at end of file diff --git a/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_shougang.json b/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_shougang.json new file mode 100644 index 0000000..f77f4ec --- /dev/null +++ b/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_shougang.json @@ -0,0 +1,139 @@ +{ + "tokenizer": "Qwen/Qwen2.5-7B-Instruct", + "splits": { + "train": { + "stage1": { + "tokens": { + "rows": 14521, + "max": 2672, + "p95": 2588, + "mean": 2582.5 + }, + "chars": { + "rows": 14521, + "max": 5075, + "p95": 4980, + "mean": 4968.6 + } + }, + "stage2": { + "tokens": { + "rows": 14521, + "max": 376, + "p95": 313, + "mean": 293.8 + }, + "chars": { + "rows": 14521, + "max": 959, + "p95": 883, + "mean": 853.8 + } + } + }, + "val": { + "stage1": { + "tokens": { + "rows": 1926, + "max": 2626, + "p95": 2590, + "mean": 2582.5 + }, + "chars": { + "rows": 1926, + "max": 5016, + "p95": 4982, + "mean": 4968.7 + } + }, + "stage2": { + "tokens": { + "rows": 1926, + "max": 363, + "p95": 316, + "mean": 293.3 + }, + "chars": { + "rows": 1926, + "max": 942, + "p95": 886, + "mean": 853.6 + } + } + }, + "test": { + "stage1": { + "tokens": { + "rows": 1946, + "max": 2689, + "p95": 2590, + "mean": 2582.7 + }, + "chars": { + "rows": 1946, + "max": 5093, + "p95": 4981, + "mean": 4968.8 + } + }, + "stage2": { + "tokens": { + "rows": 1946, + "max": 404, + "p95": 320, + "mean": 297.4 + }, + "chars": { + "rows": 1946, + "max": 981, + "p95": 888, + "mean": 858.0 + } + } + } + }, + "prompt_only": { + "train": { + "stage1": { + "rows": 14521, + "max": 2653, + "p95": 2569, + "mean": 2563.1 + }, + "stage2": { + "rows": 14521, + "max": 366, + "p95": 303, + "mean": 283.8 + } + }, + "val": { + "stage1": { + "rows": 1926, + "max": 2607, + "p95": 2571, + "mean": 2563.2 + }, + "stage2": { + "rows": 1926, + "max": 353, + "p95": 306, + "mean": 283.3 + } + }, + "test": { + "stage1": { + "rows": 1946, + "max": 2670, + "p95": 2572, + "mean": 2563.5 + }, + "stage2": { + "rows": 1946, + "max": 394, + "p95": 310, + "mean": 287.4 + } + } + } +} \ No newline at end of file diff --git a/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_summary.json b/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_summary.json new file mode 100644 index 0000000..52703bb --- /dev/null +++ b/artifacts/generated/prompt_stats/choice_id/prompt_token_stats_summary.json @@ -0,0 +1,473 @@ +{ + "tokenizer": "Qwen/Qwen2.5-7B-Instruct", + "datasets": { + "finance": { + "splits": { + "train": { + "stage1": { + "tokens": { + "rows": 403, + "max": 2553, + "p95": 2543, + "mean": 2535.4 + }, + "chars": { + "rows": 403, + "max": 5048, + "p95": 5033, + "mean": 5023.1 + } + }, + "stage2": { + "tokens": { + "rows": 403, + "max": 740, + "p95": 560, + "mean": 494.0 + }, + "chars": { + "rows": 403, + "max": 1579, + "p95": 1261, + "mean": 1163.3 + } + } + }, + "val": { + "stage1": { + "tokens": { + "rows": 69, + "max": 2543, + "p95": 2538, + "mean": 2534.6 + }, + "chars": { + "rows": 69, + "max": 5031, + "p95": 5027, + "mean": 5020.7 + } + }, + "stage2": { + "tokens": { + "rows": 69, + "max": 556, + "p95": 554, + "mean": 453.7 + }, + "chars": { + "rows": 69, + "max": 1254, + "p95": 1249, + "mean": 1097.1 + } + } + }, + "test": { + "stage1": { + "tokens": { + "rows": 57, + "max": 2539, + "p95": 2537, + "mean": 2535.3 + }, + "chars": { + "rows": 57, + "max": 5030, + "p95": 5026, + "mean": 5022.3 + } + }, + "stage2": { + "tokens": { + "rows": 57, + "max": 560, + "p95": 558, + "mean": 495.1 + }, + "chars": { + "rows": 57, + "max": 1258, + "p95": 1257, + "mean": 1160.8 + } + } + } + }, + "vs_legacy_train": { + "stage1": { + "old_mean": 5940.8, + "new_mean": 2535.4, + "old_p95": 5950, + "new_p95": 2543, + "old_max": 5959, + "new_max": 2553, + "p95_reduction_pct": 57.3, + "max_reduction_pct": 57.2 + }, + "stage2": { + "old_mean": 542.9, + "new_mean": 494.0, + "old_p95": 603, + "new_p95": 560, + "old_max": 794, + "new_max": 740, + "p95_reduction_pct": 7.1, + "max_reduction_pct": 6.8 + } + }, + "train_stage2_gt_position_hist": null + }, + "infra": { + "splits": { + "train": { + "stage1": { + "tokens": { + "rows": 40, + "max": 2588, + "p95": 2587, + "mean": 2581.6 + }, + "chars": { + "rows": 40, + "max": 4978, + "p95": 4977, + "mean": 4966.6 + } + }, + "stage2": { + "tokens": { + "rows": 40, + "max": 316, + "p95": 315, + "mean": 294.7 + }, + "chars": { + "rows": 40, + "max": 884, + "p95": 883, + "mean": 854.8 + } + } + }, + "val": { + "stage1": { + "tokens": { + "rows": 15, + "max": 2589, + "p95": 2583, + "mean": 2579.9 + }, + "chars": { + "rows": 15, + "max": 4973, + "p95": 4973, + "mean": 4965.1 + } + }, + "stage2": { + "tokens": { + "rows": 15, + "max": 311, + "p95": 305, + "mean": 301.9 + }, + "chars": { + "rows": 15, + "max": 876, + "p95": 876, + "mean": 868.1 + } + } + }, + "test": { + "stage1": { + "tokens": { + "rows": 9, + "max": 2593, + "p95": 2590, + "mean": 2588.1 + }, + "chars": { + "rows": 9, + "max": 4982, + "p95": 4980, + "mean": 4976.1 + } + }, + "stage2": { + "tokens": { + "rows": 9, + "max": 321, + "p95": 318, + "mean": 316.1 + }, + "chars": { + "rows": 9, + "max": 888, + "p95": 886, + "mean": 882.1 + } + } + } + }, + "vs_legacy_train": { + "stage1": { + "old_mean": 5066.6, + "new_mean": 2581.6, + "old_p95": 5073, + "new_p95": 2587, + "old_max": 5074, + "new_max": 2588, + "p95_reduction_pct": 49.0, + "max_reduction_pct": 49.0 + }, + "stage2": { + "old_mean": 326.7, + "new_mean": 294.7, + "old_p95": 347, + "new_p95": 315, + "old_max": 348, + "new_max": 316, + "p95_reduction_pct": 9.2, + "max_reduction_pct": 9.2 + } + }, + "train_stage2_gt_position_hist": null + }, + "pers_info": { + "splits": { + "train": { + "stage1": { + "tokens": { + "rows": 140, + "max": 266, + "p95": 260, + "mean": 255.5 + }, + "chars": { + "rows": 140, + "max": 747, + "p95": 740, + "mean": 730.0 + } + }, + "stage2": { + "tokens": { + "rows": 140, + "max": 202, + "p95": 196, + "mean": 191.3 + }, + "chars": { + "rows": 140, + "max": 707, + "p95": 700, + "mean": 689.0 + } + } + }, + "val": { + "stage1": { + "tokens": { + "rows": 18, + "max": 260, + "p95": 259, + "mean": 256.6 + }, + "chars": { + "rows": 18, + "max": 742, + "p95": 742, + "mean": 735.7 + } + }, + "stage2": { + "tokens": { + "rows": 18, + "max": 196, + "p95": 195, + "mean": 192.7 + }, + "chars": { + "rows": 18, + "max": 702, + "p95": 702, + "mean": 695.6 + } + } + }, + "test": { + "stage1": { + "tokens": { + "rows": 18, + "max": 259, + "p95": 258, + "mean": 255.4 + }, + "chars": { + "rows": 18, + "max": 739, + "p95": 738, + "mean": 732.9 + } + }, + "stage2": { + "tokens": { + "rows": 18, + "max": 196, + "p95": 195, + "mean": 191.9 + }, + "chars": { + "rows": 18, + "max": 698, + "p95": 698, + "mean": 692.3 + } + } + } + }, + "vs_legacy_train": { + "stage1": { + "old_mean": 512.3, + "new_mean": 255.5, + "old_p95": 517, + "new_p95": 260, + "old_max": 523, + "new_max": 266, + "p95_reduction_pct": 49.7, + "max_reduction_pct": 49.1 + }, + "stage2": { + "old_mean": 225.6, + "new_mean": 191.3, + "old_p95": 232, + "new_p95": 196, + "old_max": 236, + "new_max": 202, + "p95_reduction_pct": 15.5, + "max_reduction_pct": 14.4 + } + }, + "train_stage2_gt_position_hist": null + }, + "shougang": { + "splits": { + "train": { + "stage1": { + "tokens": { + "rows": 14521, + "max": 2672, + "p95": 2588, + "mean": 2582.5 + }, + "chars": { + "rows": 14521, + "max": 5075, + "p95": 4980, + "mean": 4968.6 + } + }, + "stage2": { + "tokens": { + "rows": 14521, + "max": 376, + "p95": 313, + "mean": 293.8 + }, + "chars": { + "rows": 14521, + "max": 959, + "p95": 883, + "mean": 853.8 + } + } + }, + "val": { + "stage1": { + "tokens": { + "rows": 1926, + "max": 2626, + "p95": 2590, + "mean": 2582.5 + }, + "chars": { + "rows": 1926, + "max": 5016, + "p95": 4982, + "mean": 4968.7 + } + }, + "stage2": { + "tokens": { + "rows": 1926, + "max": 363, + "p95": 316, + "mean": 293.3 + }, + "chars": { + "rows": 1926, + "max": 942, + "p95": 886, + "mean": 853.6 + } + } + }, + "test": { + "stage1": { + "tokens": { + "rows": 1946, + "max": 2689, + "p95": 2590, + "mean": 2582.7 + }, + "chars": { + "rows": 1946, + "max": 5093, + "p95": 4981, + "mean": 4968.8 + } + }, + "stage2": { + "tokens": { + "rows": 1946, + "max": 404, + "p95": 320, + "mean": 297.4 + }, + "chars": { + "rows": 1946, + "max": 981, + "p95": 888, + "mean": 858.0 + } + } + } + }, + "vs_legacy_train": { + "stage1": { + "old_mean": 5067.1, + "new_mean": 2582.5, + "old_p95": 5073, + "new_p95": 2588, + "old_max": 5157, + "new_max": 2672, + "p95_reduction_pct": 49.0, + "max_reduction_pct": 48.2 + }, + "stage2": { + "old_mean": 326.0, + "new_mean": 293.8, + "old_p95": 345, + "new_p95": 313, + "old_max": 408, + "new_max": 376, + "p95_reduction_pct": 9.3, + "max_reduction_pct": 7.8 + } + }, + "train_stage2_gt_position_hist": null + } + } +} \ No newline at end of file diff --git a/docs/data_alignment.md b/docs/data_alignment.md new file mode 100644 index 0000000..34cfc3e --- /dev/null +++ b/docs/data_alignment.md @@ -0,0 +1,67 @@ +# 数据对齐结论(dataset ↔ corpus/standard alignment) + +> 人类维护的结论性文档。机器可复现的完整报告(含逐 dataset 覆盖率矩阵与 +> UNKNOWN 明细)见 `artifacts/generated/alignment/data_alignment_report.{json,md}`, +> 由 `python -m script.analysis.analyze_dataset_corpus_alignment` 再生产。 +> 本文件的内容更新需人工判断,不随脚本自动更新。 + +## 1. dataset ↔ 对应 corpus/standard + +两种覆盖率口径(均来自生成报告,精确匹配/空白归一化): + +- **unique leaf coverage**:数据集唯一 leaf 中能在语料命中的比例(leaf 维度); +- **sample leaf_exact coverage**:样本级精确匹配率(样本维度)。 + +| dataset | 对齐 source | unique leaf coverage | sample leaf_exact coverage | 备注 | +|---|---|---|---|---| +| finance | `corpus:finance` / `financial_standards_dict.json` | 20/25(80%) | 534/568(94.0%) | 语料 220 leaf ≫ 标注 25;fs 是 3 段路径(L1-L2-leaf),数据集是 4 级,**不把 join(level_1..4) 当身份** | +| infra | `standard:guanji_dict.json` | 4/4(100%) | 64/64(100%) | infra ⊂ shougang,共享 code 空间 | +| pers_info | `standard:education_dict.json` | 4/18(≈22%) | 35/176(19.9%) | 单级分类(仅 level_4);14/18 leaf 无任何语料定义 | +| shougang | `standard:guanji_dict.json` | 192/193(≈99.5%) | 18393/19415(94.7%) | 样本口径下未匹配的 1022 条即 `——` 占位符(structural skip);code 唯一、字母↔L1 100% | + +> 注意:finance/shougang 的样本级匹配率(94.0%/94.7%)与 unique leaf +> 覆盖(80%/≈99.5%)分母不同(样本数 vs leaf 数),不要混读。 + +训练推荐 leaf level 全部为 **level_4**。 + +## 2. 现状数字(canonical 解析后,2026-08) + +| dataset | 样本 | 完整 4 级路径 | level_4 leaf 数 | canonical resolved | trainable | +|---|---|---|---|---|---| +| finance | 568 | 381 | 25 | 531 | 529 | +| infra | 64 | 64 | 4 | 64 | 64 | +| pers_info | 176 | 0(单级) | 18 | 176 | 176 | +| shougang | 19,415 | 19,415 | 193 | 18,393 | 18,393 | + +finance 有 2 条 resolved 记录落在原始 split 边界之外(数据管线事实,未静默补入); +shougang `——` 占位符 1,022 条(5.3%)作为 structural skip 不进训练。 + +## 3. 需要记住的 schema 问题(人工视角) + +- **finance**:187/568 样本 level_3 为空而 level_4 完整(路径不全,属数据事实)。 + 2026-08-19 已修复 `"经营 管理"`→`"经营管理"` whitespace 变体(6 条),只读守护 + test 见 `tests/task/test_label_whitespace_variants.py`。 +- **finance 命名噪声**:`交易清金额信息`(疑似错别字 vs `交易清结算信息`)、 + `基本信息(公开`(截断)等——标注侧定名前不改动。 +- **cross-corpus leaf 冲突**:同名 leaf 出现在多个 standard(如 shougang + `供应商管理数据` 在 guanji 与 shanghai_fta;finance `基本信息` 在 + financial_standards 与 education)——leaf 字符串不能作为身份,必须用 + canonical category_id。 +- **pers_info**:单级分类 + 语料覆盖低(4/18),target 允许 description 缺失。 +- **corpus:finance**:leaf-only(不带 path/code),历史遗留;正式 corpus 已 + 改为 `CorpusCategory`(category_id/name/description/descriptions/examples)。 +- **数据层级 vs data_level 字段**:data_level 分布与分类深度不符,语义未定 + (仅作 provenance,不解释为敏感级别)。 + +## 4. 仍需人工确认(摘要) + +finance 5 个 leaf 无语料定义 / guanji code 数字段与 level_2/3 的对应 / +pers_info 14/18 语料来源 / education code 无层级可对照 / data_level 语义 / +shougang corpus 1 条 malformed 条目(`"nan"`)。完整清单见报告 §4.9。 + +## 5. 相关文档 + +- 身份策略与训练规则:`docs/design/data_contract.md` +- 历史设计记录:`docs/phase_reports/stage2_contract_design.md`、 + `docs/phase_reports/stage3c_report.md` +- 数据维护 SOP:`docs/新数据集运行说明.md` diff --git a/docs/design/data_contract.md b/docs/design/data_contract.md new file mode 100644 index 0000000..a7ac65f --- /dev/null +++ b/docs/design/data_contract.md @@ -0,0 +1,96 @@ +# Canonical Data Contract + +Status: **frozen** (Stage 3C/3B). The canonical identity is the single source of +truth for registry, corpus, ground truth, SFT labels, evaluation and reward. + +## 1. One identity: canonical `category_id` + +- **`target.category_id` is the only canonical semantic ground-truth identity** + across SFT and RL. It is unique within one dataset's `LeafRegistry`. + Model-facing supervision/actions use derived choice ids (see + `docs/design/prompt_interface.md`) — the model never generates canonical ids. +- `classification.level_1..level_4` are **provenance only** — never read as a + label, never fallback. There is deliberately **no `level_4` fallback**. +- Prompt-facing **choice ids are a derived, decoupled view** (see + `docs/design/prompt_interface.md`); they never leak back into canonical + records or reward semantics. + +## 2. Canonical sample schema (`data//canonical/all.json`) + +Input sample keeps the original fields (`classification`, `metadata`, +`data_level`, `label_status`, …) untouched as provenance. The resolver +appends: + +```jsonc +"resolution_status": "resolved", // resolved | missing_leaf | path_mismatch | ... +"target": { + "leaf_level": "level_4", + "leaf_name": "合同归并", // display name; not guaranteed unique + "category_id": "B1-2", // the ONLY training identity (registry-unique) + "category_path": ["生产数据域", "生产合同(订单)", "合同归并"] // provenance, not an ID +} +``` + +## 3. Canonical corpus schema (`cfg/task/corpus/.corpus.json`) + +```jsonc +{ "categories": [ { + "category_id": "A1-1-1", + "name": "科研设备预约管理", + "description": "…", // primary description; may be empty + "descriptions": ["…"], // extra description documents (semantically distinct from examples) + "path": ["研发数据域", "科研设备管理", "科研设备预约管理"], // may be empty + "code": "A1-1-1", // may be empty; opaque stable id, digit groups are NOT level_2/3 + "examples": ["设备预约信息", "审核表"] +} ] } +``` + +- A category may have multiple descriptions/examples or none. +- **Production invariant**: the corpus must cover every registry category so + Stage 2 can always resolve candidates by `category_id` (no registry fallback, + validated by `require_corpus_covers_registry`). + +## 4. Leaf registry (`cfg/task/registry/.registry.json`) + +- The `LeafRegistry` is the **complete leaf universe** of the classification + standard/corpus, built from the corpus — **never derived from training + samples** (samples may cover only a subset). +- Per-dataset registry sizes: finance **233** / infra **233** / shougang **233** + / pers_info **18**; corpus sizes match. + +## 5. Per-dataset identity strategy + +| dataset | strategy | category_id shape | +|---|---|---| +| shougang | code (guanji) | `A1-1-1`, `B1-2`; letter↔level_1 100% (A/B/C); digits are opaque ordinals | +| infra | code, `registry_source=shougang` | shares shougang codes (infra ⊂ shougang) | +| finance | path (L1/L2/L4) | `finance:业务.账户信息.基本信息`; level_3 is provenance, not part of identity | +| pers_info | path (level_4 only) | `pers_info:学籍管理信息`; single-level labels | + +## 6. Training entry rule (SFT + RL) + +Only records with `resolution_status == "resolved"` **and** `target.category_id ∈ +LeafRegistry` **and** `target.leaf_name == registry.get(category_id).name` +enter training; violations fail fast (no silent skip). Split membership follows +the original `data//{train,val,test}.json` by record id. `canonical` +resolved / trainable counts (trainable = resolved within split boundaries): + +| dataset | canonical resolved | trainable | outside splits | +|---|---|---|---| +| finance | 531 | 529 | 2 | +| infra | 64 | 64 | 0 | +| pers_info | 176 | 176 | 0 | +| shougang | 18,393 | 18,393 | 0 | + +## 7. Consumers & files + +- Contracts: `src/agent/task/contracts.py` (`LeafCategory`, `CorpusCategory`, + `SampleTarget`, `LeafRegistry`, `TaskConfig`) +- Resolver: `src/agent/task/resolver.py`, identity: `src/agent/task/identity.py`, + dataset config: `src/agent/task/dataset_config.py` +- Shared training helpers: `src/agent/training/common.py` + (`canonical_target`, `build_candidates`, `require_corpus*`) +- Data-side alignment facts: `docs/data_alignment.md`; generated report: + `artifacts/generated/alignment/data_alignment_report.{json,md}` +- Historical design records: `docs/phase_reports/stage2_contract_design.md`, + `docs/phase_reports/stage3c_report.md` diff --git a/docs/design/prompt_interface.md b/docs/design/prompt_interface.md new file mode 100644 index 0000000..feb7ce6 --- /dev/null +++ b/docs/design/prompt_interface.md @@ -0,0 +1,61 @@ +# Prompt Interface (choice protocol) + +Status: **frozen** (Phase 6 / PR #8, merged on master). The model never sees or +generates canonical `category_id`; it speaks a compact **choice protocol** that +is decoded to canonical ids at the LLM boundary. + +## 1. Why a choice protocol + +- Canonical ids are internal identity (`finance:业务.账户信息.基本信息`) — long, + verbose, and **must not become model actions** (generated ids would be + brittle and leak internal taxonomy structure). +- Choice ids are compact (`"1".."N"`), deterministic, and provide a **compact + action representation** (they do not reduce the number of categories). This + is a **decoupling/identity** decision, not only a token reduction one (see + `docs/prompt_length_analysis.md` §6). + +## 2. Mapping (`src/agent/task/prompt_choices.py`) + +`PromptChoiceRegistry` maps canonical `category_id` ↔ global choice id +(`"1".."N"` following `LeafRegistry.categories` order, stable across runs) +plus a **display name** = shortest unique path suffix (leaf name when unique, +parent-qualified otherwise). Choice ids exist only inside prompts/model outputs; +they are decoded back immediately and never written to canonical records. + +## 3. Stage 1 (retrieve 5 candidates over the full registry) + +- Prompt: catalog of `[choice_id, display_name]` pairs for the whole registry. +- Model output: `{"candidates": ["3", "7", "12", "19", "24"]}` (global choice ids). +- Checks (shared, never raise on model output): JSON/schema → exactly 5 → + unique → choice_id in catalog → decode to canonical `category_id` tuple. + +## 4. Stage 2 (pick one answer from the 5-candidate bundle) + +- Prompt: candidate bundle with **local** ids `"1".."5"` in candidate order + + display name + corpus description/examples. +- Model output: `{"answer": "2"}` (local bundle id). +- Checks (shared): JSON/schema → answer ∈ `"1".."5"` (strict, no coercion) → + positional decode against the canonical candidates. + +## 5. Shared decode layer (Phase 7 / PR #9) + +`agent.task.parser.check_stage1_choices` / `check_stage2_choices` → +`ChoiceParseResult` (format/constraint validity + decoded canonical ids + +`canonical_view()`). **Both evaluation and reward consume this single +implementation** — no second copy of choice validation: + +- `agent.evaluation.evaluate_stage1_choices` / `evaluate_stage2_choices` +- `agent.training.rl.reward_stage1_choices` / `reward_stage2_choices` / + `reward_for_choice_result` (decode first, then the unchanged reward table) + +Stage-2 answer — `docs/design/data_contract.md` §6 — ground truth is NOT +required to be among the candidates (a Stage-1 recall miss is a legal state). + +## 6. Files + +- `src/agent/task/prompt_choices.py` — registry + encode/decode primitives +- `src/agent/task/parser.py` — shared never-raise choice-aware checks +- `src/agent/task/prompts.py` — prompt builders (`build_stage1_prompt`, + `build_stage2_prompt`, `stage1_answer`, `stage2_answer`) +- Exports/layout facts: `docs/SFT_BASELINE.md`, `docs/RL_FRAMEWORK.md`; + historical design: `docs/phase_reports/stage2_contract_design.md` diff --git a/artifacts/stage2_contract_design.md b/docs/phase_reports/stage2_contract_design.md similarity index 98% rename from artifacts/stage2_contract_design.md rename to docs/phase_reports/stage2_contract_design.md index aca7b71..056c029 100644 --- a/artifacts/stage2_contract_design.md +++ b/docs/phase_reports/stage2_contract_design.md @@ -1,6 +1,6 @@ # 阶段 2:最小统一数据 contract 设计 -> 依据:`artifacts/data_alignment_report.md/.json`(阶段 1)。 +> 依据:`artifacts/generated/alignment/data_alignment_report.md/.json`(阶段 1)。 > 原则:不自动修复报告中的 UNKNOWN 数据问题;`classification.level_1~level_4` > 原样保留作为 provenance;训练 ground truth 最终只依赖 `target.category_id`。 diff --git a/artifacts/stage3c_report.md b/docs/phase_reports/stage3c_report.md similarity index 96% rename from artifacts/stage3c_report.md rename to docs/phase_reports/stage3c_report.md index 9b33e2a..72be8a8 100644 --- a/artifacts/stage3c_report.md +++ b/docs/phase_reports/stage3c_report.md @@ -13,7 +13,7 @@ - `script/verl/sft/prompt_stats.py` — prompt 长度统计(字符级 + 可选 tokenizer) - `tests/sft/test_sft_canonical_e2e.py` — 真实数据端到端(CI 无 data/ 时 skip) - `tests/sft/test_prompt_stats.py`、`tests/sft/fixtures/canonical/` -- `artifacts/prompt_stats_{finance,shougang}.json`、`artifacts/prompt_token_stats_{finance,shougang}.json` +- `artifacts/generated/prompt_stats/legacy/prompt_stats_{finance,shougang}.json`、`artifacts/generated/prompt_stats/legacy/prompt_token_stats_{finance,shougang}.json` ## 2. 新 SFT 数据流 diff --git a/artifacts/stage4a_report.md b/docs/phase_reports/stage4a_report.md similarity index 99% rename from artifacts/stage4a_report.md rename to docs/phase_reports/stage4a_report.md index 5940330..5c07104 100644 --- a/artifacts/stage4a_report.md +++ b/docs/phase_reports/stage4a_report.md @@ -20,7 +20,7 @@ - `src/agent/training/rl/__init__.py` / `sample.py` / `reward.py` / `dataset.py` — RL 数据 / reward / VeRL parquet 契约 - `script/verl/rl/__init__.py` / `export.py` / `validate.py` — CLI 适配器 - `tests/rl/test_rl_parser.py` / `test_rl_reward.py` / `test_rl_dataset.py` / `test_rl_canonical_e2e.py` / `test_rl_verl_compat.py` -- `artifacts/stage4a_report.md`(本文件) +- `docs/phase_reports/stage4a_report.md`(本文件) ## 2. RL sample schema diff --git a/docs/prompt_length_analysis.md b/docs/prompt_length_analysis.md new file mode 100644 index 0000000..c5e01a0 --- /dev/null +++ b/docs/prompt_length_analysis.md @@ -0,0 +1,83 @@ +# Prompt Length Analysis (Phase 9) + +Recomputed on the **Phase 8 real SFT/RL prompts** (choice protocol) against the +legacy canonical-id baseline, using the `Qwen/Qwen2.5-7B-Instruct` tokenizer +(snapshot `a09a3545`, tokenizer.json only). Measurement = token count of the +full chat-template-rendered conversation (identical semantics to the legacy +`prompt_stats.py` baseline; p95 uses the legacy definition +`sorted[int(0.95·n)-1]`). Old stats are kept untouched as the legacy baseline. + +Generated stats: `artifacts/generated/prompt_stats/` (canonical_id_baseline/ = +old canonical-id stats, choice_id/ = new choice-protocol stats). + +## 1. Old vs new p95 / max (train split, tokens) + +### p95 +| dataset | stage | old p95 | new p95 | reduction % | +|---|---|---|---|---| +| finance | stage1 | 5950 | 2543 | **57.3%** | +| finance | stage2 | 603 | 560 | 7.1% | +| infra | stage1 | 5073 | 2587 | 49.0% | +| infra | stage2 | 347 | 315 | 9.2% | +| pers_info | stage1 | 517 | 260 | 49.7% | +| pers_info | stage2 | 232 | 196 | 15.5% | +| shougang | stage1 | 5073 | 2588 | 49.0% | +| shougang | stage2 | 345 | 313 | 9.3% | + +### max +| dataset | stage | old max | new max | reduction % | +|---|---|---|---|---| +| finance | stage1 | 5959 | 2553 | **57.2%** | +| finance | stage2 | 794 | 740 | 6.8% | +| infra | stage1 | 5074 | 2588 | 49.0% | +| infra | stage2 | 348 | 316 | 9.2% | +| pers_info | stage1 | 523 | 266 | 49.1% | +| pers_info | stage2 | 236 | 202 | 14.4% | +| shougang | stage1 | 5157 | 2672 | 48.2% | +| shougang | stage2 | 408 | 376 | 7.8% | + +## 2. Stage 1 is the main token-saving source + +Per-record combined mean (train): finance 6483.7 → 3029.4 tokens (**−53.3%**, +abs. saving ≈ 3454); infra/shougang 5393 → 2876 (**−46.7%**); pers_info 737.9 → +446.8 (−39.5%). ~99% of the absolute saving comes from **Stage 1** +(finance/infra/shougang ≈ 98–99%), where the full registry (finance/infra/ +shougang 233 categories; pers_info 18) used to be rendered with verbose +canonical ids and is now a compact `[choice_id, display_name]` catalog. Stage 2 (the 5-candidate bundle) was +already compact, so it gains only 7–15%. + +## 3. finance vs shougang: which is longest now — and a reversal + +- **shougang** is now the longest (global max 2689, test stage1; prompt-only + 2670) vs finance (max 2553). +- This **reverses the old baseline** (finance 5959 > shougang 5157). Reasons: + finance's old ids were long (`finance:客户.个人….`) so choice ids save 57%; + shougang's old ids were short (`A1-1-1`) so they save 49%. The remaining + longest rows are driven by **long metadata text** in the data (shougang's + longest row carries a 17-item enumerated `field_description`; finance's + longest row has tiny metadata), not by the registry. + +## 4. Worst case vs context + +Global worst case = **2689 tokens** (shougang test, Stage 1) ≈ **8.2% of the +32768 context** (prompt-only 2670 ≈ 8.15%). The new prompts are safely below +the context limit with a large margin — the old worst case was ~6000 tokens +(~18%), so the choice protocol roughly halves the headroom consumed. + +## 5. Baselines & reproducibility + +`artifacts/generated/prompt_stats/canonical_id_baseline/` keeps the old +canonical-id stats (`prompt_token_stats_*.json`, `prompt_stats_*.json`, +`prompt_token_stats_summary.json`) unchanged for regressions and comparison; +`artifacts/generated/prompt_stats/choice_id/` holds the new choice-protocol +stats. Regenerate (reproduces the tables above) with +`python -m script.verl.sft.prompt_stats --dataset-dir data/sft/ --model +Qwen/Qwen2.5-7B-Instruct --report .json`. + +## 6. Note: PromptChoice is not only about length + +The choice protocol exists primarily to **decouple the internal canonical +`category_id` from model actions** — ids must never be generated as actions +(fragile, leaks taxonomy). Token savings (≈50% Stage 1) are a welcome side +effect. See `docs/design/prompt_interface.md` and +`docs/design/data_contract.md`. diff --git a/script/analysis/analyze_dataset_corpus_alignment.py b/script/analysis/analyze_dataset_corpus_alignment.py index 22ce188..16d591d 100644 --- a/script/analysis/analyze_dataset_corpus_alignment.py +++ b/script/analysis/analyze_dataset_corpus_alignment.py @@ -7,8 +7,8 @@ - standards: data/knowledge/standards_map/*.json Writes: - - artifacts/data_alignment_report.json - - artifacts/data_alignment_report.md + - artifacts/generated/alignment/data_alignment_report.json + - artifacts/generated/alignment/data_alignment_report.md The script never modifies data/ or src/. """ @@ -25,7 +25,7 @@ PROJECT_ROOT = Path(__file__).resolve().parents[2] DEFAULT_DATA_DIR = PROJECT_ROOT / "data" -DEFAULT_OUT_DIR = PROJECT_ROOT / "artifacts" +DEFAULT_OUT_DIR = PROJECT_ROOT / "artifacts" / "generated" / "alignment" CLASSIFICATION_FIELDS = ("level_1", "level_2", "level_3", "level_4") CODE_RE = re.compile(r"[A-Za-z]+\d*(?:-\d+)*") @@ -1314,15 +1314,15 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description=( "Analyze dataset <-> corpus alignment (exact/normalized matching only) " - "and write artifacts/data_alignment_report.json + .md" + "and write artifacts/generated/alignment/data_alignment_report.json + .md" ) ) parser.add_argument("--overwrite", action="store_true", help="Overwrite outputs") args = parser.parse_args(argv) build_report(overwrite=args.overwrite) - print("wrote: artifacts/data_alignment_report.json") - print("wrote: artifacts/data_alignment_report.md") + print("wrote: artifacts/generated/alignment/data_alignment_report.json") + print("wrote: artifacts/generated/alignment/data_alignment_report.md") return 0 diff --git a/src/agent/task/dataset_config.py b/src/agent/task/dataset_config.py index 647c4b2..c00a672 100644 --- a/src/agent/task/dataset_config.py +++ b/src/agent/task/dataset_config.py @@ -1,6 +1,6 @@ """Per-dataset configuration for the canonical training target. -Decision basis: artifacts/data_alignment_report (stage 1). Each dataset gets +Decision basis: artifacts/generated/alignment/data_alignment_report (stage 1). Each dataset gets an explicit category_id strategy instead of a single global rule: - shougang: guanji codes are stable and unique (233 codes, letter prefix