From 7c4fc1cab33a1736fe5df4e8b8d96f211910fe74 Mon Sep 17 00:00:00 2001 From: Alessio <133716460+As7656431@users.noreply.github.com> Date: Sun, 18 Oct 2026 17:01:31 +0200 Subject: [PATCH 1/2] Add files via upload --- .../README.md | 75 ++ .../results.json | 912 ++++++++++++++++++ 2 files changed, 987 insertions(+) create mode 100644 bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/README.md create mode 100644 bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/results.json diff --git a/bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/README.md b/bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/README.md new file mode 100644 index 00000000000..7ba045bad0a --- /dev/null +++ b/bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/README.md @@ -0,0 +1,75 @@ +# Community benchmark on NVIDIA GeForce RTX 5070 + +Measured on 2026-10-18 with `tools/strata_bench.py` (suite 1, 3 runs per test, medians with ranges). Speeds are the engine's own timings; first-token times are measured by the client over HTTP. Speed only: no correctness check was run. + +## Hardware and software + +- GPU and VRAM: NVIDIA GeForce RTX 5070, 12 GiB VRAM +- CPU: Intel(R) Core(TM) i5-14600KF (14 cores, 20 threads); installed RAM: 31.8 GiB (2 x 16 GB DDR4-3600); storage: the model files are on an aigo P2000 NVMe drive, which read at about 1.4-1.6 GB/s in an earlier measurement on this machine +- PCIe: the GPU is on PCIe 5.0 x16; Resizable BAR is off +- OS: Windows 10 Pro 22H2 (10.0.19045), Chinese locale; GPU driver: 591.74; CUDA: 13.1 +- Strata: project 0.1.45, commit not a git checkout (release zip); engine 0.1.45 (official release binary) +- Background workloads and power limits: not recorded; the GPU also drives the display + +## Model and configuration + +- Model and quantization: qwen3.8-flash-next-iq2_xs +- Context: 32768 tokens; parallel slots: 1 +- Settings: the daily configuration of this PC (low-RAM mode, `--mmap-experts`, because the 35.5 GB of experts do not fit in 31.8 GiB of RAM), with one change: the optional control-vector file that setup offers (`--control-vector-scaled`, not a speed setting) was left out so the numbers compare with other reports; thinking off for the suite; temperature 0 + +```text +strata.exe --pack iq2_xs --native Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf --ple-gguf Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf --expert-profile expert-profile.bin --expert-cache auto --prefill auto --spec 4 --mtp rt --max-context 32768 --kv int8 --mmap-experts --pcie-frac 0.55 --spec-min-p 0.70 +``` + +Environment of the config: `STRATA_FETCH_THREADS=16` + +## Method + +Run by `python tools/strata_bench.py` against the HTTP API of a Strata server (already running); model loading is not timed. One warm-up chat and a 1K prompt are discarded first. Every prompt starts with a unique tag, so the conversation cache cannot help the fresh-prompt tests. + +- short chat: the three questions of `tools/ab_engine.py`, up to 100 new tokens, 3 runs +- 4K / 16K prompts: generated word lists sized by a calibration request, 24 new tokens, 3 fresh prompts each +- agent session: system prompt and four tools, then 5 turns that each add about 1500 words of a project file and continue from the model's own earlier answers (the way agent clients resend a conversation), 64 new tokens per turn, 3 sessions; the cache numbers come from the answer's usage (`cached_tokens`) and timings (`prompt_n`) +- concurrency (when run): N requests sent at the same moment, total tokens over the wall time of the burst, latency over all requests + +Two workarounds were needed to run the 0.1.45 tool on this PC; neither changes what is measured: + +- The server was started separately with `serve/server.py` and the tool was pointed at it with `--url http://127.0.0.1:8080 --config `, because the tool's own server start stops with `NameError: name 'port' is not defined` (already reported in #2864, #2872 and #2908; fixes in #2919 and #2994). +- The tool was run with `PYTHONUTF8=1`, because on a Chinese-locale Windows it cannot read `CMakeLists.txt` (`UnicodeDecodeError: 'gbk' codec`, already reported in #2792; fix in #2879). + +## Results + +| Configuration | Actual prompt tokens | Reused tokens | Generated tokens | Runs | Prompt tok/s median and range | Decode tok/s median and range | TTFT seconds median and range | +| --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| short chat | 46 (43-51) | 0 | 100 (100-100) | 9 | not measured | 20.2 (12.9-43.7) | 1.95 (0.76-2.64) | +| 4K prompt, fresh | 4019 (4019-4019) | 0 (0-0) | 24 cap | 3 | 58.3 (51.7-98.9) | not measured | 68.98 (40.70-77.75) | +| 16K prompt, fresh | 16058 (16058-16058) | 0 (0-0) | 24 cap | 3 | 127.5 (124.3-173.4) | not measured | 125.99 (92.69-129.22) | +| agent session, follow-up turns | 14573 (14566-14576) at the last turn | see below | up to 64 | 3 | see below | 6.4 (0.7-13.5) | 48.24 (33.34-86.87) | + +Ranges are the lowest and highest of the runs; `Runs` counts requests that returned a token. + +### Agent session (conversation cache) + +Cache hit over the follow-up turns: 68.8 % (68.6-68.8); earlier-turn tokens read again per session: 4 (2-74) (earlier turns that the cache did not hold; the 13717 (13717-13717) new tokens of the files are read once); first turn (nothing cached) first token: 20.46 s (19.49-26.17). + +| Turn (run 1) | Prompt tokens | Reused | Read | Prompt tok/s | First token (s) | Decode tok/s | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 0 | 609 | 0 | 609 | 23.4 | 26.166 | 10.4 | +| 1 | 3759 | 639 | 3120 | 83.0 | 37.713 | 5.7 | +| 2 | 5292 | 3823 | 1469 | 36.9 | 39.893 | 9.2 | +| 3 | 8725 | 5356 | 3369 | 65.6 | 51.512 | 4.8 | +| 4 | 11628 | 8766 | 2862 | 60.5 | 47.355 | 3.9 | +| 5 | 14566 | 11667 | 2899 | 45.9 | 63.301 | 6.4 | + +Per-request data: `results.json`. No `engine-log-excerpt.txt`: the server was started separately (see Method), so the tool did not capture its log. + +## Correctness and limitations + +Not measured: this is a speed suite. The conversation uses the model's own replies, so token counts differ a little between runs. Machine details come from the server, `nvidia-smi`/`rocm-smi` and the OS; background load and power limits are not recorded. Total time of the suite: 1709 s. + +## Notes + +- **Low RAM and a slower drive.** The RTX 5070 Ti report with 31.6 GiB of RAM (#2791, 0.1.44.1, Windows 11, WD SN770) reads a 4K prompt at 270 tok/s and a 16K prompt at 448 tok/s; this PC reads them at 58 and 128 tok/s. The GPU (16 GB against 12 GB) and the drive differ, so this run alone does not say which one matters more. +- **What earlier measurements on this PC showed** (0.1.39, with Windows performance counters during a 20K-token prompt): the drive read only 0.2-0.5 GB/s while about 300 MB of RAM stayed available, so the prompt path was limited by page faults on the mapped `experts.bin`, not by the drive's speed. Those pages also pushed the decode's cached experts out of RAM, which matches the slow decode in the agent turns here (6.4 tok/s). +- **The short-chat decode** (20.2 tok/s, range 12.9-43.7) was measured right after the warm-up; in low-RAM mode the first requests after a start read many experts from the drive, so it is lower than this PC's decode once the file cache is warm. +- Not tested here: a context above 32K (`--long`), `--clients`, a RAM budget (`--resident-budget-gib`), other models. diff --git a/bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/results.json b/bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/results.json new file mode 100644 index 00000000000..91fce0de2e9 --- /dev/null +++ b/bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/results.json @@ -0,0 +1,912 @@ +{ + "schema": 1, + "tool": "tools/strata_bench.py", + "date": "2026-10-18", + "params": { + "runs": 3, + "think": false, + "clients": 0, + "long": false, + "started_server": false, + "command": "python tools/strata_bench.py" + }, + "elapsed_s": 1709.464569568634, + "machine": { + "gpu": [ + "NVIDIA GeForce RTX 5070" + ], + "gpu_vram_mib": 12227, + "gpu_driver": "591.74", + "cuda": "13.1", + "cpu": "Intel(R) Core(TM) i5-14600KF", + "cpu_cores": 14, + "cpu_threads": 20, + "ram_gib": 31.8, + "os": "Windows 10 (10.0.19045)", + "python": "3.14.0", + "strata_version": "0.1.45", + "strata_commit": null, + "engine_version": "0.1.45", + "model": "qwen3.8-flash-next-iq2_xs", + "quant": "IQ2_XS", + "context": 32768, + "parallel_slots": 1, + "config": { + "exe": "strata.exe", + "args": [ + "--pack", + "iq2_xs", + "--native", + "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf", + "--ple-gguf", + "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf", + "--expert-profile", + "expert-profile.bin", + "--expert-cache", + "auto", + "--prefill", + "auto", + "--spec", + "4", + "--mtp", + "rt", + "--max-context", + "32768", + "--kv", + "int8", + "--mmap-experts", + "--pcie-frac", + "0.55", + "--spec-min-p", + "0.70" + ], + "model_name": "qwen3.8-flash-next-iq2_xs", + "lib_dirs": [ + "\\Lib\\site-packages\\nvidia\\cu13\\bin\\x86_64" + ], + "port": 8080, + "gpu": 0, + "gpus_asked": true, + "env": { + "STRATA_FETCH_THREADS": "16" + } + }, + "launch": "strata.exe --pack iq2_xs --native Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf --ple-gguf Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf --expert-profile expert-profile.bin --expert-cache auto --prefill auto --spec 4 --mtp rt --max-context 32768 --kv int8 --mmap-experts --pcie-frac 0.55 --spec-min-p 0.70", + "engine_info": { + "model": "qwen3.8-flash-next-iq2_xs", + "max_context": 32768, + "images": false, + "version": "0.1.45", + "logit_bias": 1, + "context": 32768, + "kv": "int8", + "kv_resident": 0, + "expert_slots": 3734, + "expert_cache_mib": 5140, + "expert_slots_primary": 3734, + "expert_cache_primary_mib": 5140, + "spec": 6, + "mtp_max": 4, + "lookup": 3, + "vram_free_mib": 328, + "cvec": 0, + "arena_mib": 0, + "pool_workers": 13, + "pcie_frac": "0.55", + "spec_min_p": "0.70", + "conversation_cache_mib": 0, + "conversation_cache_slots": 4, + "conversation_cache_min_free_mib": 2560, + "conversation_cache_min_tokens": 0, + "tail_role_token": -1, + "vram_elastic": 0, + "prefill_cap": 8192, + "prefill_stages": 1, + "stream_all_min": 1024, + "prefill_pipe": 0, + "engine": "0.1.45" + }, + "vram_after": { + "elastic": false + } + }, + "results": { + "chat": { + "requests": [ + { + "ok": true, + "err": null, + "ttft_s": 0.757, + "total_s": 5.448, + "prompt_tokens": 51, + "reused_tokens": 0, + "read_tokens": 51, + "generated_tokens": 100, + "prompt_tps": 68.7, + "prompt_tps_source": "engine", + "decode_tps": 21.3, + "decode_tps_source": "engine", + "prompt_ms": 742.2, + "decode_ms": 4703.4, + "drafts_offered": 71, + "drafts_accepted": 68, + "run": 0, + "prompt": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 1.884, + "total_s": 7.382, + "prompt_tokens": 43, + "reused_tokens": 0, + "read_tokens": 43, + "generated_tokens": 100, + "prompt_tps": 23.1, + "prompt_tps_source": "engine", + "decode_tps": 18.1, + "decode_tps_source": "engine", + "prompt_ms": 1864.4, + "decode_ms": 5515.0, + "drafts_offered": 72, + "drafts_accepted": 47, + "run": 0, + "prompt": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 2.642, + "total_s": 10.398, + "prompt_tokens": 46, + "reused_tokens": 0, + "read_tokens": 46, + "generated_tokens": 100, + "prompt_tps": 17.6, + "prompt_tps_source": "engine", + "decode_tps": 12.9, + "decode_tps_source": "engine", + "prompt_ms": 2606.3, + "decode_ms": 7776.7, + "drafts_offered": 67, + "drafts_accepted": 50, + "run": 0, + "prompt": 2 + }, + { + "ok": true, + "err": null, + "ttft_s": 2.643, + "total_s": 8.517, + "prompt_tokens": 51, + "reused_tokens": 0, + "read_tokens": 51, + "generated_tokens": 100, + "prompt_tps": 19.8, + "prompt_tps_source": "engine", + "decode_tps": 16.9, + "decode_tps_source": "engine", + "prompt_ms": 2578.7, + "decode_ms": 5904.0, + "drafts_offered": 75, + "drafts_accepted": 66, + "run": 1, + "prompt": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 1.947, + "total_s": 6.733, + "prompt_tokens": 43, + "reused_tokens": 0, + "read_tokens": 43, + "generated_tokens": 100, + "prompt_tps": 22.7, + "prompt_tps_source": "engine", + "decode_tps": 20.8, + "decode_tps_source": "engine", + "prompt_ms": 1894.6, + "decode_ms": 4802.6, + "drafts_offered": 64, + "drafts_accepted": 46, + "run": 1, + "prompt": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 2.416, + "total_s": 8.107, + "prompt_tokens": 46, + "reused_tokens": 0, + "read_tokens": 46, + "generated_tokens": 100, + "prompt_tps": 19.4, + "prompt_tps_source": "engine", + "decode_tps": 17.5, + "decode_tps_source": "engine", + "prompt_ms": 2372.1, + "decode_ms": 5708.4, + "drafts_offered": 64, + "drafts_accepted": 50, + "run": 1, + "prompt": 2 + }, + { + "ok": true, + "err": null, + "ttft_s": 2.384, + "total_s": 7.285, + "prompt_tokens": 51, + "reused_tokens": 0, + "read_tokens": 51, + "generated_tokens": 100, + "prompt_tps": 22.0, + "prompt_tps_source": "engine", + "decode_tps": 20.2, + "decode_tps_source": "engine", + "prompt_ms": 2320.2, + "decode_ms": 4948.1, + "drafts_offered": 71, + "drafts_accepted": 68, + "run": 2, + "prompt": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 1.216, + "total_s": 3.492, + "prompt_tokens": 43, + "reused_tokens": 0, + "read_tokens": 43, + "generated_tokens": 100, + "prompt_tps": 37.1, + "prompt_tps_source": "engine", + "decode_tps": 43.7, + "decode_tps_source": "engine", + "prompt_ms": 1158.1, + "decode_ms": 2289.8, + "drafts_offered": 64, + "drafts_accepted": 46, + "run": 2, + "prompt": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 0.895, + "total_s": 3.376, + "prompt_tokens": 46, + "reused_tokens": 0, + "read_tokens": 46, + "generated_tokens": 100, + "prompt_tps": 52.3, + "prompt_tps_source": "engine", + "decode_tps": 40.1, + "decode_tps_source": "engine", + "prompt_ms": 879.9, + "decode_ms": 2493.4, + "drafts_offered": 67, + "drafts_accepted": 49, + "run": 2, + "prompt": 2 + } + ], + "decode_tps": { + "median": 20.2, + "min": 12.9, + "max": 43.7, + "n": 9 + }, + "ttft_s": { + "median": 1.947, + "min": 0.757, + "max": 2.643, + "n": 9 + }, + "generated_tokens": { + "median": 100, + "min": 100, + "max": 100, + "n": 9 + }, + "prompt_tokens": { + "median": 46, + "min": 43, + "max": 51, + "n": 9 + } + }, + "prompts": { + "4K": { + "target_tokens": 4096, + "requests": [ + { + "ok": true, + "err": null, + "ttft_s": 40.7, + "total_s": 46.762, + "prompt_tokens": 4019, + "reused_tokens": 0, + "read_tokens": 4019, + "generated_tokens": 24, + "prompt_tps": 98.9, + "prompt_tps_source": "engine", + "decode_tps": 3.9, + "decode_tps_source": "engine", + "prompt_ms": 40653.3, + "decode_ms": 6098.4, + "drafts_offered": 17, + "drafts_accepted": 6, + "run": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 77.747, + "total_s": 85.74, + "prompt_tokens": 4019, + "reused_tokens": 0, + "read_tokens": 4019, + "generated_tokens": 24, + "prompt_tps": 51.7, + "prompt_tps_source": "engine", + "decode_tps": 3.0, + "decode_tps_source": "engine", + "prompt_ms": 77677.0, + "decode_ms": 8029.7, + "drafts_offered": 18, + "drafts_accepted": 5, + "run": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 68.983, + "total_s": 71.0, + "prompt_tokens": 4019, + "reused_tokens": 0, + "read_tokens": 4019, + "generated_tokens": 24, + "prompt_tps": 58.3, + "prompt_tps_source": "engine", + "decode_tps": 11.6, + "decode_tps_source": "engine", + "prompt_ms": 68879.4, + "decode_ms": 2074.4, + "drafts_offered": 14, + "drafts_accepted": 8, + "run": 2 + } + ], + "prompt_tps": { + "median": 58.3, + "min": 51.7, + "max": 98.9, + "n": 3 + }, + "ttft_s": { + "median": 68.983, + "min": 40.7, + "max": 77.747, + "n": 3 + }, + "prompt_tokens": { + "median": 4019, + "min": 4019, + "max": 4019, + "n": 3 + }, + "reused_tokens": { + "median": 0, + "min": 0, + "max": 0, + "n": 3 + } + }, + "16K": { + "target_tokens": 16384, + "requests": [ + { + "ok": true, + "err": null, + "ttft_s": 129.224, + "total_s": 131.258, + "prompt_tokens": 16058, + "reused_tokens": 0, + "read_tokens": 16058, + "generated_tokens": 24, + "prompt_tps": 124.3, + "prompt_tps_source": "engine", + "decode_tps": 11.6, + "decode_tps_source": "engine", + "prompt_ms": 129138.6, + "decode_ms": 2063.1, + "drafts_offered": 19, + "drafts_accepted": 11, + "run": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 92.694, + "total_s": 96.933, + "prompt_tokens": 16058, + "reused_tokens": 0, + "read_tokens": 16058, + "generated_tokens": 24, + "prompt_tps": 173.4, + "prompt_tps_source": "engine", + "decode_tps": 5.6, + "decode_tps_source": "engine", + "prompt_ms": 92615.8, + "decode_ms": 4268.6, + "drafts_offered": 13, + "drafts_accepted": 10, + "run": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 125.993, + "total_s": 127.537, + "prompt_tokens": 16058, + "reused_tokens": 0, + "read_tokens": 16058, + "generated_tokens": 24, + "prompt_tps": 127.5, + "prompt_tps_source": "engine", + "decode_tps": 15.3, + "decode_tps_source": "engine", + "prompt_ms": 125931.2, + "decode_ms": 1571.3, + "drafts_offered": 13, + "drafts_accepted": 9, + "run": 2 + } + ], + "prompt_tps": { + "median": 127.5, + "min": 124.3, + "max": 173.4, + "n": 3 + }, + "ttft_s": { + "median": 125.993, + "min": 92.694, + "max": 129.224, + "n": 3 + }, + "prompt_tokens": { + "median": 16058, + "min": 16058, + "max": 16058, + "n": 3 + }, + "reused_tokens": { + "median": 0, + "min": 0, + "max": 0, + "n": 3 + } + } + }, + "agent": { + "sessions": [ + [ + { + "ok": true, + "err": null, + "ttft_s": 26.166, + "total_s": 28.831, + "prompt_tokens": 609, + "reused_tokens": 0, + "read_tokens": 609, + "generated_tokens": 28, + "prompt_tps": 23.4, + "prompt_tps_source": "engine", + "decode_tps": 10.4, + "decode_tps_source": "engine", + "prompt_ms": 26068.8, + "decode_ms": 2688.2, + "drafts_offered": 28, + "drafts_accepted": 17, + "run": 0, + "turn": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 37.713, + "total_s": 48.999, + "prompt_tokens": 3759, + "reused_tokens": 639, + "read_tokens": 3120, + "generated_tokens": 64, + "prompt_tps": 83.0, + "prompt_tps_source": "engine", + "decode_tps": 5.7, + "decode_tps_source": "engine", + "prompt_ms": 37575.5, + "decode_ms": 11310.7, + "drafts_offered": 55, + "drafts_accepted": 38, + "run": 0, + "turn": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 39.893, + "total_s": 46.786, + "prompt_tokens": 5292, + "reused_tokens": 3823, + "read_tokens": 1469, + "generated_tokens": 64, + "prompt_tps": 36.9, + "prompt_tps_source": "engine", + "decode_tps": 9.2, + "decode_tps_source": "engine", + "prompt_ms": 39814.3, + "decode_ms": 6921.5, + "drafts_offered": 52, + "drafts_accepted": 39, + "run": 0, + "turn": 2 + }, + { + "ok": true, + "err": null, + "ttft_s": 51.512, + "total_s": 60.261, + "prompt_tokens": 8725, + "reused_tokens": 5356, + "read_tokens": 3369, + "generated_tokens": 42, + "prompt_tps": 65.6, + "prompt_tps_source": "engine", + "decode_tps": 4.8, + "decode_tps_source": "engine", + "prompt_ms": 51378.0, + "decode_ms": 8783.5, + "drafts_offered": 31, + "drafts_accepted": 20, + "run": 0, + "turn": 3 + }, + { + "ok": true, + "err": null, + "ttft_s": 47.355, + "total_s": 57.539, + "prompt_tokens": 11628, + "reused_tokens": 8766, + "read_tokens": 2862, + "generated_tokens": 40, + "prompt_tps": 60.5, + "prompt_tps_source": "engine", + "decode_tps": 3.9, + "decode_tps_source": "engine", + "prompt_ms": 47290.3, + "decode_ms": 10207.9, + "drafts_offered": 33, + "drafts_accepted": 25, + "run": 0, + "turn": 4 + }, + { + "ok": true, + "err": null, + "ttft_s": 63.301, + "total_s": 72.549, + "prompt_tokens": 14566, + "reused_tokens": 11667, + "read_tokens": 2899, + "generated_tokens": 59, + "prompt_tps": 45.9, + "prompt_tps_source": "engine", + "decode_tps": 6.4, + "decode_tps_source": "engine", + "prompt_ms": 63211.7, + "decode_ms": 9279.9, + "drafts_offered": 49, + "drafts_accepted": 32, + "run": 0, + "turn": 5 + } + ], + [ + { + "ok": true, + "err": null, + "ttft_s": 19.494, + "total_s": 21.522, + "prompt_tokens": 609, + "reused_tokens": 0, + "read_tokens": 609, + "generated_tokens": 27, + "prompt_tps": 31.3, + "prompt_tps_source": "engine", + "decode_tps": 13.2, + "decode_tps_source": "engine", + "prompt_ms": 19435.8, + "decode_ms": 2043.0, + "drafts_offered": 28, + "drafts_accepted": 16, + "run": 1, + "turn": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 38.095, + "total_s": 44.134, + "prompt_tokens": 3758, + "reused_tokens": 638, + "read_tokens": 3120, + "generated_tokens": 64, + "prompt_tps": 82.0, + "prompt_tps_source": "engine", + "decode_tps": 10.6, + "decode_tps_source": "engine", + "prompt_ms": 38046.5, + "decode_ms": 6062.6, + "drafts_offered": 50, + "drafts_accepted": 33, + "run": 1, + "turn": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 56.918, + "total_s": 68.623, + "prompt_tokens": 5291, + "reused_tokens": 3821, + "read_tokens": 1470, + "generated_tokens": 64, + "prompt_tps": 26.1, + "prompt_tps_source": "engine", + "decode_tps": 5.2, + "decode_tps_source": "engine", + "prompt_ms": 56309.2, + "decode_ms": 12227.3, + "drafts_offered": 56, + "drafts_accepted": 37, + "run": 1, + "turn": 2 + }, + { + "ok": true, + "err": null, + "ttft_s": 72.825, + "total_s": 77.07, + "prompt_tokens": 8724, + "reused_tokens": 5284, + "read_tokens": 3440, + "generated_tokens": 41, + "prompt_tps": 47.3, + "prompt_tps_source": "engine", + "decode_tps": 9.6, + "decode_tps_source": "engine", + "prompt_ms": 72750.6, + "decode_ms": 4271.6, + "drafts_offered": 31, + "drafts_accepted": 20, + "run": 1, + "turn": 3 + }, + { + "ok": true, + "err": null, + "ttft_s": 38.257, + "total_s": 43.321, + "prompt_tokens": 11626, + "reused_tokens": 8764, + "read_tokens": 2862, + "generated_tokens": 49, + "prompt_tps": 74.9, + "prompt_tps_source": "engine", + "decode_tps": 9.6, + "decode_tps_source": "engine", + "prompt_ms": 38189.9, + "decode_ms": 5087.7, + "drafts_offered": 42, + "drafts_accepted": 30, + "run": 1, + "turn": 4 + }, + { + "ok": true, + "err": null, + "ttft_s": 59.408, + "total_s": 104.816, + "prompt_tokens": 14573, + "reused_tokens": 11674, + "read_tokens": 2899, + "generated_tokens": 30, + "prompt_tps": 50.1, + "prompt_tps_source": "engine", + "decode_tps": 0.7, + "decode_tps_source": "engine", + "prompt_ms": 57807.9, + "decode_ms": 45978.9, + "drafts_offered": 32, + "drafts_accepted": 11, + "run": 1, + "turn": 5 + } + ], + [ + { + "ok": true, + "err": null, + "ttft_s": 20.463, + "total_s": 22.428, + "prompt_tokens": 609, + "reused_tokens": 0, + "read_tokens": 609, + "generated_tokens": 30, + "prompt_tps": 29.9, + "prompt_tps_source": "engine", + "decode_tps": 15.2, + "decode_tps_source": "engine", + "prompt_ms": 20395.4, + "decode_ms": 1976.7, + "drafts_offered": 27, + "drafts_accepted": 19, + "run": 2, + "turn": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 50.054, + "total_s": 59.881, + "prompt_tokens": 3761, + "reused_tokens": 641, + "read_tokens": 3120, + "generated_tokens": 64, + "prompt_tps": 62.4, + "prompt_tps_source": "engine", + "decode_tps": 6.5, + "decode_tps_source": "engine", + "prompt_ms": 49973.2, + "decode_ms": 9853.0, + "drafts_offered": 57, + "drafts_accepted": 29, + "run": 2, + "turn": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 33.338, + "total_s": 44.452, + "prompt_tokens": 5294, + "reused_tokens": 3824, + "read_tokens": 1470, + "generated_tokens": 64, + "prompt_tps": 44.2, + "prompt_tps_source": "engine", + "decode_tps": 5.7, + "decode_tps_source": "engine", + "prompt_ms": 33275.3, + "decode_ms": 11134.6, + "drafts_offered": 59, + "drafts_accepted": 35, + "run": 2, + "turn": 2 + }, + { + "ok": true, + "err": null, + "ttft_s": 45.058, + "total_s": 53.47, + "prompt_tokens": 8727, + "reused_tokens": 5357, + "read_tokens": 3370, + "generated_tokens": 58, + "prompt_tps": 74.9, + "prompt_tps_source": "engine", + "decode_tps": 6.9, + "decode_tps_source": "engine", + "prompt_ms": 44985.6, + "decode_ms": 8443.5, + "drafts_offered": 54, + "drafts_accepted": 28, + "run": 2, + "turn": 3 + }, + { + "ok": true, + "err": null, + "ttft_s": 86.87, + "total_s": 89.135, + "prompt_tokens": 11646, + "reused_tokens": 8784, + "read_tokens": 2862, + "generated_tokens": 32, + "prompt_tps": 33.0, + "prompt_tps_source": "engine", + "decode_tps": 13.5, + "decode_tps_source": "engine", + "prompt_ms": 86720.8, + "decode_ms": 2376.9, + "drafts_offered": 24, + "drafts_accepted": 17, + "run": 2, + "turn": 4 + }, + { + "ok": true, + "err": null, + "ttft_s": 48.24, + "total_s": 108.329, + "prompt_tokens": 14576, + "reused_tokens": 11677, + "read_tokens": 2899, + "generated_tokens": 60, + "prompt_tps": 60.2, + "prompt_tps_source": "engine", + "decode_tps": 1.0, + "decode_tps_source": "engine", + "prompt_ms": 48150.3, + "decode_ms": 60129.3, + "drafts_offered": 52, + "drafts_accepted": 38, + "run": 2, + "turn": 5 + } + ] + ], + "hit_pct": { + "median": 68.799, + "min": 68.637, + "max": 68.819, + "n": 3 + }, + "reread_tokens": { + "median": 4, + "min": 2, + "max": 74, + "n": 3 + }, + "new_tokens": { + "median": 13717, + "min": 13717, + "max": 13717, + "n": 3 + }, + "turn_ttft_s": { + "median": 48.24, + "min": 33.338, + "max": 86.87, + "n": 15 + }, + "last_turn_prompt_tokens": { + "median": 14573, + "min": 14566, + "max": 14576, + "n": 3 + }, + "turn_decode_tps": { + "median": 6.4, + "min": 0.7, + "max": 13.5, + "n": 15 + }, + "first_turn_ttft_s": { + "median": 20.463, + "min": 19.494, + "max": 26.166, + "n": 3 + } + }, + "clients": {}, + "skipped": {}, + "failures": [] + } +} From 5870264963caa615c6f9498a3ec7c5ecb45e49ab Mon Sep 17 00:00:00 2001 From: As7656431 <133716460+As7656431@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:11:36 +0200 Subject: [PATCH 2/2] bench: RTX 5070 12 GB community report re-measured on 0.2.3.1 (replaces the 0.1.45 report) --- .../README.md | 75 - .../README.md | 77 + .../engine-log-excerpt.txt | 80 + .../results.json | 1826 +++++++++-------- 4 files changed, 1071 insertions(+), 987 deletions(-) delete mode 100644 bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/README.md create mode 100644 bench/results/2026-10-23-community-rtx-5070-qwen3-8-flash-next-iq2-xs/README.md create mode 100644 bench/results/2026-10-23-community-rtx-5070-qwen3-8-flash-next-iq2-xs/engine-log-excerpt.txt rename bench/results/{2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs => 2026-10-23-community-rtx-5070-qwen3-8-flash-next-iq2-xs}/results.json (56%) diff --git a/bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/README.md b/bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/README.md deleted file mode 100644 index 7ba045bad0a..00000000000 --- a/bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Community benchmark on NVIDIA GeForce RTX 5070 - -Measured on 2026-10-18 with `tools/strata_bench.py` (suite 1, 3 runs per test, medians with ranges). Speeds are the engine's own timings; first-token times are measured by the client over HTTP. Speed only: no correctness check was run. - -## Hardware and software - -- GPU and VRAM: NVIDIA GeForce RTX 5070, 12 GiB VRAM -- CPU: Intel(R) Core(TM) i5-14600KF (14 cores, 20 threads); installed RAM: 31.8 GiB (2 x 16 GB DDR4-3600); storage: the model files are on an aigo P2000 NVMe drive, which read at about 1.4-1.6 GB/s in an earlier measurement on this machine -- PCIe: the GPU is on PCIe 5.0 x16; Resizable BAR is off -- OS: Windows 10 Pro 22H2 (10.0.19045), Chinese locale; GPU driver: 591.74; CUDA: 13.1 -- Strata: project 0.1.45, commit not a git checkout (release zip); engine 0.1.45 (official release binary) -- Background workloads and power limits: not recorded; the GPU also drives the display - -## Model and configuration - -- Model and quantization: qwen3.8-flash-next-iq2_xs -- Context: 32768 tokens; parallel slots: 1 -- Settings: the daily configuration of this PC (low-RAM mode, `--mmap-experts`, because the 35.5 GB of experts do not fit in 31.8 GiB of RAM), with one change: the optional control-vector file that setup offers (`--control-vector-scaled`, not a speed setting) was left out so the numbers compare with other reports; thinking off for the suite; temperature 0 - -```text -strata.exe --pack iq2_xs --native Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf --ple-gguf Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf --expert-profile expert-profile.bin --expert-cache auto --prefill auto --spec 4 --mtp rt --max-context 32768 --kv int8 --mmap-experts --pcie-frac 0.55 --spec-min-p 0.70 -``` - -Environment of the config: `STRATA_FETCH_THREADS=16` - -## Method - -Run by `python tools/strata_bench.py` against the HTTP API of a Strata server (already running); model loading is not timed. One warm-up chat and a 1K prompt are discarded first. Every prompt starts with a unique tag, so the conversation cache cannot help the fresh-prompt tests. - -- short chat: the three questions of `tools/ab_engine.py`, up to 100 new tokens, 3 runs -- 4K / 16K prompts: generated word lists sized by a calibration request, 24 new tokens, 3 fresh prompts each -- agent session: system prompt and four tools, then 5 turns that each add about 1500 words of a project file and continue from the model's own earlier answers (the way agent clients resend a conversation), 64 new tokens per turn, 3 sessions; the cache numbers come from the answer's usage (`cached_tokens`) and timings (`prompt_n`) -- concurrency (when run): N requests sent at the same moment, total tokens over the wall time of the burst, latency over all requests - -Two workarounds were needed to run the 0.1.45 tool on this PC; neither changes what is measured: - -- The server was started separately with `serve/server.py` and the tool was pointed at it with `--url http://127.0.0.1:8080 --config `, because the tool's own server start stops with `NameError: name 'port' is not defined` (already reported in #2864, #2872 and #2908; fixes in #2919 and #2994). -- The tool was run with `PYTHONUTF8=1`, because on a Chinese-locale Windows it cannot read `CMakeLists.txt` (`UnicodeDecodeError: 'gbk' codec`, already reported in #2792; fix in #2879). - -## Results - -| Configuration | Actual prompt tokens | Reused tokens | Generated tokens | Runs | Prompt tok/s median and range | Decode tok/s median and range | TTFT seconds median and range | -| --- | ---: | ---: | ---: | ---: | --- | --- | --- | -| short chat | 46 (43-51) | 0 | 100 (100-100) | 9 | not measured | 20.2 (12.9-43.7) | 1.95 (0.76-2.64) | -| 4K prompt, fresh | 4019 (4019-4019) | 0 (0-0) | 24 cap | 3 | 58.3 (51.7-98.9) | not measured | 68.98 (40.70-77.75) | -| 16K prompt, fresh | 16058 (16058-16058) | 0 (0-0) | 24 cap | 3 | 127.5 (124.3-173.4) | not measured | 125.99 (92.69-129.22) | -| agent session, follow-up turns | 14573 (14566-14576) at the last turn | see below | up to 64 | 3 | see below | 6.4 (0.7-13.5) | 48.24 (33.34-86.87) | - -Ranges are the lowest and highest of the runs; `Runs` counts requests that returned a token. - -### Agent session (conversation cache) - -Cache hit over the follow-up turns: 68.8 % (68.6-68.8); earlier-turn tokens read again per session: 4 (2-74) (earlier turns that the cache did not hold; the 13717 (13717-13717) new tokens of the files are read once); first turn (nothing cached) first token: 20.46 s (19.49-26.17). - -| Turn (run 1) | Prompt tokens | Reused | Read | Prompt tok/s | First token (s) | Decode tok/s | -| ---: | ---: | ---: | ---: | ---: | ---: | ---: | -| 0 | 609 | 0 | 609 | 23.4 | 26.166 | 10.4 | -| 1 | 3759 | 639 | 3120 | 83.0 | 37.713 | 5.7 | -| 2 | 5292 | 3823 | 1469 | 36.9 | 39.893 | 9.2 | -| 3 | 8725 | 5356 | 3369 | 65.6 | 51.512 | 4.8 | -| 4 | 11628 | 8766 | 2862 | 60.5 | 47.355 | 3.9 | -| 5 | 14566 | 11667 | 2899 | 45.9 | 63.301 | 6.4 | - -Per-request data: `results.json`. No `engine-log-excerpt.txt`: the server was started separately (see Method), so the tool did not capture its log. - -## Correctness and limitations - -Not measured: this is a speed suite. The conversation uses the model's own replies, so token counts differ a little between runs. Machine details come from the server, `nvidia-smi`/`rocm-smi` and the OS; background load and power limits are not recorded. Total time of the suite: 1709 s. - -## Notes - -- **Low RAM and a slower drive.** The RTX 5070 Ti report with 31.6 GiB of RAM (#2791, 0.1.44.1, Windows 11, WD SN770) reads a 4K prompt at 270 tok/s and a 16K prompt at 448 tok/s; this PC reads them at 58 and 128 tok/s. The GPU (16 GB against 12 GB) and the drive differ, so this run alone does not say which one matters more. -- **What earlier measurements on this PC showed** (0.1.39, with Windows performance counters during a 20K-token prompt): the drive read only 0.2-0.5 GB/s while about 300 MB of RAM stayed available, so the prompt path was limited by page faults on the mapped `experts.bin`, not by the drive's speed. Those pages also pushed the decode's cached experts out of RAM, which matches the slow decode in the agent turns here (6.4 tok/s). -- **The short-chat decode** (20.2 tok/s, range 12.9-43.7) was measured right after the warm-up; in low-RAM mode the first requests after a start read many experts from the drive, so it is lower than this PC's decode once the file cache is warm. -- Not tested here: a context above 32K (`--long`), `--clients`, a RAM budget (`--resident-budget-gib`), other models. diff --git a/bench/results/2026-10-23-community-rtx-5070-qwen3-8-flash-next-iq2-xs/README.md b/bench/results/2026-10-23-community-rtx-5070-qwen3-8-flash-next-iq2-xs/README.md new file mode 100644 index 00000000000..a2269086dcb --- /dev/null +++ b/bench/results/2026-10-23-community-rtx-5070-qwen3-8-flash-next-iq2-xs/README.md @@ -0,0 +1,77 @@ +# Community benchmark on NVIDIA GeForce RTX 5070 + +Measured on 2026-10-23 with `tools/strata_bench.py` (suite 1, 3 runs per test, medians with ranges). Speeds are the engine's own timings; first-token times are measured by the client over HTTP. Speed only: no correctness check was run. + +## Hardware and software + +- GPU and VRAM: NVIDIA GeForce RTX 5070, 12 GiB VRAM +- CPU: Intel(R) Core(TM) i5-14600KF (14 cores, 20 threads); installed RAM: 31.8 GiB (2 x 16 GB DDR4-3600); storage: the model files are on an aigo P2000 NVMe drive, which read at about 1.4-1.6 GB/s in an earlier measurement on this machine +- PCIe: the GPU is on PCIe 5.0 x16; Resizable BAR is off +- OS: Windows 10 Pro 22H2 (10.0.19045), Chinese locale; GPU driver: 591.74; CUDA: 13.1 (driver) +- Strata: release 0.2.3.1 (project 0.1.48.1), commit 7ddc17a3; engine 0.1.48.1 built from that commit on this PC (CUDA 13.0, sm_120, MSVC 2022, `STRATA_PORTABLE=ON`), not the release package's binary +- Background workloads and power limits: not recorded; the GPU also drives the display + +## Model and configuration + +- Model and quantization: qwen3.8-flash-next-iq2_xs (the original Flash-Next IQ2_XS pack, 35.5 GB of experts) +- Context: 32768 tokens; parallel slots: 1 +- Settings: the daily configuration of this PC (low-RAM mode, `--mmap-experts`, because the experts do not fit in 31.8 GiB of RAM), with one change: the optional control-vector file that setup offers (`--control-vector-scaled`, not a speed setting) was left out so the numbers compare with other reports; thinking off for the suite; temperature 0 + +```text +strata.exe --pack iq2_xs --native Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf --ple-gguf Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf --expert-profile expert-profile.bin --expert-cache auto --prefill auto --spec 4 --mtp rt --max-context 32768 --kv int8 --mmap-experts --pcie-frac 0.55 --spec-min-p 0.70 +``` + +Environment of the config: `STRATA_FETCH_THREADS=16` + +## Method + +Run by `python tools/strata_bench.py` against the HTTP API of a Strata server (already running); model loading is not timed. One warm-up chat and a 1K prompt are discarded first. Every prompt starts with a unique tag, so the conversation cache cannot help the fresh-prompt tests. + +- short chat: the three questions of `tools/ab_engine.py`, up to 100 new tokens, 3 runs +- 4K / 16K prompts: generated word lists sized by a calibration request, 24 new tokens, 3 fresh prompts each +- agent session: system prompt and four tools, then 5 turns that each add about 1500 words of a project file and continue from the model's own earlier answers (the way agent clients resend a conversation), 64 new tokens per turn, 3 sessions; the cache numbers come from the answer's usage (`cached_tokens`) and timings (`prompt_n`) +- concurrency (when run): N requests sent at the same moment, total tokens over the wall time of the burst, latency over all requests + +How it was run on this PC; neither changes what is measured: + +- The server was started separately with the release's `serve/server.py` and the tool was pointed at it with `--url http://127.0.0.1:8080 --config `, the same way as the 0.1.45 report of this PC, so the tool did not save the engine log itself. `engine-log-excerpt.txt` was cut from the benchmark run's part of the engine log with the tool's own `log_excerpt()` and `redactor()`, and local folders were replaced with placeholders. +- The tool was run with `PYTHONUTF8=1`: on a Chinese-locale Windows it still reads `CMakeLists.txt` with the default code page (`tools/strata_bench.py`, the `project(strata VERSION ...)` lookup) and stops with `UnicodeDecodeError: 'gbk' codec` (reported in #2792). + +## Results + +| Configuration | Actual prompt tokens | Reused tokens | Generated tokens | Runs | Prompt tok/s median and range | Decode tok/s median and range | TTFT seconds median and range | +| --- | ---: | ---: | ---: | ---: | --- | --- | --- | +| short chat | 46 (43-51) | 0 | 100 (100-100) | 9 | not measured | 55.8 (14.6-66.8) | 0.73 (0.53-2.12) | +| 4K prompt, fresh | 4019 (4019-4019) | 0 (0-0) | 24 cap | 3 | 87.0 (56.9-96.7) | not measured | 46.27 (41.63-70.81) | +| 16K prompt, fresh | 16058 (16058-16058) | 0 (0-0) | 24 cap | 3 | 101.3 (66.7-105.3) | not measured | 158.67 (152.52-240.99) | +| agent session, follow-up turns | 14601 (14581-14621) at the last turn | see below | up to 64 | 3 | see below | 5.7 (0.7-14.4) | 62.82 (41.85-80.74) | + +Ranges are the lowest and highest of the runs; `Runs` counts requests that returned a token. + +### Agent session (conversation cache) + +Cache hit over the follow-up turns: 69.5 % (69.4-69.5); earlier-turn tokens read again per session: 73 (73-144) (earlier turns that the cache did not hold; the 13729 (13729-13730) new tokens of the files are read once); first turn (nothing cached) first token: 24.91 s (24.12-29.21). + +| Turn (run 1) | Prompt tokens | Reused | Read | Prompt tok/s | First token (s) | Decode tok/s | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 0 | 609 | 0 | 609 | 20.9 | 29.207 | 9.8 | +| 1 | 4109 | 641 | 3468 | 55.3 | 62.821 | 9.4 | +| 2 | 5642 | 4173 | 1469 | 31.4 | 46.985 | 2.0 | +| 3 | 9075 | 5635 | 3440 | 64.5 | 53.442 | 0.7 | +| 4 | 11828 | 9120 | 2708 | 52.0 | 52.172 | 1.2 | +| 5 | 14601 | 11884 | 2717 | 34.1 | 80.738 | 2.5 | + +Per-request data: `results.json`. Engine log lines from the benchmark run: `engine-log-excerpt.txt` (see Method). + +## Correctness and limitations + +Not measured: this is a speed suite. The conversation uses the model's own replies, so token counts differ a little between runs. Machine details come from the server, `nvidia-smi`/`rocm-smi` and the OS; background load and power limits are not recorded. Total time of the suite: 2161 s. + +## Notes + +- **Against 0.1.45 on this PC** (the report this one replaces, same settings and tool, 2026-10-18): 4K prompt 58.3 tok/s, 16K prompt 127.5 tok/s, agent follow-up turn first token 48.2 s, agent decode 6.4 tok/s. On 0.2.3.1 the 4K median is higher (87.0) but the ranges overlap (56.9-96.7 against 51.7-98.9); the 16K prompt is slower (101.3, range 66.7-105.3 against 124.3-173.4); the agent turns are not faster. +- **Low RAM and a slower drive.** The RTX 5070 Ti report with 31.6 GiB of RAM (#2791, 0.1.44.1, Windows 11, WD SN770) reads a 4K prompt at 270 tok/s and a 16K prompt at 448 tok/s. The GPU (16 GB against 12 GB) and the drive differ, so this run alone does not say which one matters more. +- **What earlier measurements on this PC showed** (0.1.39, with Windows performance counters during a 20K-token prompt): the drive read only 0.2-0.5 GB/s while about 300 MB of RAM stayed available, so the prompt path was limited by page faults on the mapped `experts.bin`, not by the drive's speed. Those pages also push the decode's cached experts out of RAM, which matches the slow decode in the agent turns (5.7 tok/s). Opt-in changes for this setup and their measurements: #3065. +- **VRAM.** With these settings the engine logs 337 MiB of VRAM free with everything loaded, and warns that the card is nearly full; the suite completed without an error. +- **The short-chat decode** (median 55.8 tok/s) includes the first run right after the warm-up (14.6-21.0 tok/s): in low-RAM mode the first requests after a start read many experts from the drive. +- Not tested here: a context above 32K (`--long`), `--clients`, a RAM budget (`--resident-budget-gib`), the release package's own engine binary, other models. diff --git a/bench/results/2026-10-23-community-rtx-5070-qwen3-8-flash-next-iq2-xs/engine-log-excerpt.txt b/bench/results/2026-10-23-community-rtx-5070-qwen3-8-flash-next-iq2-xs/engine-log-excerpt.txt new file mode 100644 index 00000000000..eda1bee0ca0 --- /dev/null +++ b/bench/results/2026-10-23-community-rtx-5070-qwen3-8-flash-next-iq2-xs/engine-log-excerpt.txt @@ -0,0 +1,80 @@ +[strata] 2026-10-23 17:55:24 engine started: \strata.exe --serve --pack \packs\iq2_xs --native \models\IQ2_XS\Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf --ple-gguf \models\IQ2_XS\Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf --expert-profile \data\expert-profile.bin --expert-cache auto --prefill auto --spec 4 --mtp \mtp\rt --max-context 32768 --kv int8 --mmap-experts --pcie-frac 0.55 --spec-min-p 0.70 +strata generate: profile \data\expert-profile.bin: 24576 ranked pairs, built for 24576 slots +strata mtp: draft layer loaded, 839 MiB of VRAM (experts 675, dense 111), files read in 0.24 s (3307 MiB/s) +strata generate: GPU 0: NVIDIA GeForce RTX 5070, compute capability 12.0 +strata generate: expert cache auto: 6.03 GiB free, 700 MiB reserved (+143 MiB for the draft head) -> 3701 slots +strata generate: only 635 MiB free once the slots are written (reserve 700 MiB); shrinking the expert cache +strata generate: expert cache 3780 slots, 5.08 GiB of VRAM; policy is +strata generate: the GPU computes the experts in the cache; it rounds differently from the CPU, +strata generate: pre-filled 3780 of 3780 slots from the profile in 1.4 s (3951 MB/s); slot 0 verified +strata generate: R4 hit path ON - resident experts are computed on the GPU +strata generate: token graph hit path: 3780 resident experts, decided on the device +strata: the CPU prefill share is not armed by default: the experts it would read are 27.9 GiB of file pages and only 19.7 GiB of RAM is free to keep them. STRATA_PREFILL_CPU_SHARE=auto turns it on anyway +strata: prefill plan: chunk 8192, ring 96 kept: file set 32.1 GiB does not fit the page cache 12.4 GiB -> about 137.7 GB read from the drive per 32768-token prompt (4 chunks); no option does better (the set cannot fit: fewest passes win) +strata: prefill plan inputs: 33.0 GiB experts, 5.1 GiB in the GPU caches, 0.0 GiB RAM copy (0 lend slots kept), file tier 27.9 GiB; 12.4 GiB available for the page cache; PCIe 12.0 GB/s; drive 1.50 GB/s (assumed); a 32768-token prompt +strata serve: the prompt path borrows 2816 CUDA0 cache slots (3.77 GiB) +strata mtp: draft head over 106299 tokens (137.9 MiB) +strata serve: 337 MiB of VRAM free with everything loaded +strata serve: that is little room for the verify windows' buffers under WDDM; if the engine exits right after this line, --prefill auto:8192 (a smaller prompt chunk borrows fewer cache slots) or a larger --vram-reserve-mib leaves more +strata serve: WARNING: only 337 MiB of VRAM free at the start of this request (below 1024 MiB): when the card is this full the driver can page GPU memory to system RAM, decode falls to a few tok/s and a window may never finish (#2065). Add --vram-reserve-mib 1408 to the config's args, or lower --max-context / --batch +strata verify: capturing the 6-token window (337 MiB of VRAM free) +strata verify: capturing the 3-token window (327 MiB of VRAM free) +strata verify: capturing the 1-token window (311 MiB of VRAM free) +strata verify: capturing the 4-token window (293 MiB of VRAM free) +strata verify: capturing the 2-token window (277 MiB of VRAM free) +strata serve: decode expert cache hit rate: 37.7% (4035 hits / 10716 lookups) +strata serve: expert tiers: GPU 4035 hits this request; since the start RAM 0 blobs, files 0 blobs 18633.4 MB read +strata serve: decode expert cache hit rate: 60.1% (235 hits / 391 lookups) +strata serve: expert tiers: GPU 235 hits this request; since the start RAM 0 blobs, files 0 blobs 30085.1 MB read +strata serve: decode expert cache hit rate: 60.6% (238 hits / 393 lookups) +strata serve: expert tiers: GPU 238 hits this request; since the start RAM 0 blobs, files 0 blobs 44342.5 MB read +strata serve: decode expert cache hit rate: 57.8% (1882 hits / 3257 lookups) +strata serve: expert tiers: GPU 1882 hits this request; since the start RAM 0 blobs, files 0 blobs 60129.2 MB read +strata serve: decode expert cache hit rate: 58.8% (26220 hits / 44590 lookups) +strata serve: expert tiers: GPU 26220 hits this request; since the start RAM 0 blobs, files 0 blobs 88506.5 MB read +strata serve: decode expert cache hit rate: 69.4% (35483 hits / 51145 lookups) +strata serve: expert tiers: GPU 35483 hits this request; since the start RAM 0 blobs, files 0 blobs 115436.9 MB read +strata serve: decode expert cache hit rate: 61.5% (30476 hits / 49527 lookups) +strata serve: expert tiers: GPU 30476 hits this request; since the start RAM 0 blobs, files 0 blobs 145735.1 MB read +strata serve: decode expert cache hit rate: 52.9% (22663 hits / 42805 lookups) +strata serve: expert tiers: GPU 22663 hits this request; since the start RAM 0 blobs, files 0 blobs 175808.6 MB read +strata serve: decode expert cache hit rate: 73.6% (37309 hits / 50680 lookups) +strata serve: expert tiers: GPU 37309 hits this request; since the start RAM 0 blobs, files 0 blobs 199174.6 MB read +strata serve: decode expert cache hit rate: 69.1% (34363 hits / 49740 lookups) +strata serve: expert tiers: GPU 34363 hits this request; since the start RAM 0 blobs, files 0 blobs 226448.7 MB read +strata serve: decode expert cache hit rate: 52.1% (22955 hits / 44057 lookups) +strata serve: expert tiers: GPU 22955 hits this request; since the start RAM 0 blobs, files 0 blobs 257122.2 MB read +strata serve: decode expert cache hit rate: 74.6% (39320 hits / 52689 lookups) +strata serve: expert tiers: GPU 39320 hits this request; since the start RAM 0 blobs, files 0 blobs 280624.7 MB read +strata serve: decode expert cache hit rate: 67.5% (35337 hits / 52346 lookups) +strata serve: expert tiers: GPU 35337 hits this request; since the start RAM 0 blobs, files 0 blobs 308138.4 MB read +strata serve: decode expert cache hit rate: 59.5% (8574 hits / 14402 lookups) +strata serve: expert tiers: GPU 8574 hits this request; since the start RAM 0 blobs, files 0 blobs 350392.2 MB read +strata serve: decode expert cache hit rate: 74.7% (9243 hits / 12366 lookups) +strata serve: expert tiers: GPU 9243 hits this request; since the start RAM 0 blobs, files 0 blobs 389506.3 MB read +strata serve: decode expert cache hit rate: 83.4% (10581 hits / 12694 lookups) +strata serve: expert tiers: GPU 10581 hits this request; since the start RAM 0 blobs, files 0 blobs 427433.9 MB read +strata serve: decode expert cache hit rate: 87.5% (13598 hits / 15545 lookups) +strata serve: expert tiers: GPU 13598 hits this request; since the start RAM 0 blobs, files 0 blobs 503008.3 MB read +strata serve: decode expert cache hit rate: 83.9% (10014 hits / 11939 lookups) +strata serve: expert tiers: GPU 10014 hits this request; since the start RAM 0 blobs, files 0 blobs 578166.6 MB read +strata serve: decode expert cache hit rate: 91.4% (14442 hits / 15796 lookups) +strata serve: expert tiers: GPU 14442 hits this request; since the start RAM 0 blobs, files 0 blobs 652834.5 MB read +strata serve: decode expert cache hit rate: 61.0% (12640 hits / 20733 lookups) +strata serve: expert tiers: GPU 12640 hits this request; since the start RAM 0 blobs, files 0 blobs 685675.3 MB read +strata serve: decode expert cache hit rate: 56.0% (19786 hits / 35363 lookups) +strata serve: expert tiers: GPU 19786 hits this request; since the start RAM 0 blobs, files 0 blobs 735612.3 MB read +strata serve: decode expert cache hit rate: 66.6% (21043 hits / 31618 lookups) +strata serve: expert tiers: GPU 21043 hits this request; since the start RAM 0 blobs, files 0 blobs 780461.3 MB read +strata serve: decode expert cache hit rate: 80.8% (23560 hits / 29150 lookups) +strata serve: expert tiers: GPU 23560 hits this request; since the start RAM 0 blobs, files 0 blobs 822446.0 MB read +strata serve: decode expert cache hit rate: 82.8% (31203 hits / 37703 lookups) +strata serve: expert tiers: GPU 31203 hits this request; since the start RAM 0 blobs, files 0 blobs 864264.7 MB read +strata serve: decode expert cache hit rate: 79.7% (30497 hits / 38246 lookups) +strata serve: expert tiers: GPU 30497 hits this request; since the start RAM 0 blobs, files 0 blobs 908571.6 MB read +strata serve: decode expert cache hit rate: 59.5% (9262 hits / 15572 lookups) +strata serve: expert tiers: GPU 9262 hits this request; since the start RAM 0 blobs, files 0 blobs 939876.7 MB read +strata serve: decode expert cache hit rate: 75.7% (26768 hits / 35346 lookups) +strata serve: expert tiers: GPU 26768 hits this request; since the start RAM 0 blobs, files 0 blobs 984763.0 MB read +strata serve: decode expert cache hit rate: 71.2% (23485 hits / 32986 lookups) +strata serve: expert tiers: GPU 23485 hits this request; since the start RAM 0 blobs, files 0 blobs 1029102.6 MB read diff --git a/bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/results.json b/bench/results/2026-10-23-community-rtx-5070-qwen3-8-flash-next-iq2-xs/results.json similarity index 56% rename from bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/results.json rename to bench/results/2026-10-23-community-rtx-5070-qwen3-8-flash-next-iq2-xs/results.json index 91fce0de2e9..1bd48d560ff 100644 --- a/bench/results/2026-10-18-community-rtx-5070-qwen3-8-flash-next-iq2-xs/results.json +++ b/bench/results/2026-10-23-community-rtx-5070-qwen3-8-flash-next-iq2-xs/results.json @@ -1,912 +1,914 @@ -{ - "schema": 1, - "tool": "tools/strata_bench.py", - "date": "2026-10-18", - "params": { - "runs": 3, - "think": false, - "clients": 0, - "long": false, - "started_server": false, - "command": "python tools/strata_bench.py" - }, - "elapsed_s": 1709.464569568634, - "machine": { - "gpu": [ - "NVIDIA GeForce RTX 5070" - ], - "gpu_vram_mib": 12227, - "gpu_driver": "591.74", - "cuda": "13.1", - "cpu": "Intel(R) Core(TM) i5-14600KF", - "cpu_cores": 14, - "cpu_threads": 20, - "ram_gib": 31.8, - "os": "Windows 10 (10.0.19045)", - "python": "3.14.0", - "strata_version": "0.1.45", - "strata_commit": null, - "engine_version": "0.1.45", - "model": "qwen3.8-flash-next-iq2_xs", - "quant": "IQ2_XS", - "context": 32768, - "parallel_slots": 1, - "config": { - "exe": "strata.exe", - "args": [ - "--pack", - "iq2_xs", - "--native", - "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf", - "--ple-gguf", - "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf", - "--expert-profile", - "expert-profile.bin", - "--expert-cache", - "auto", - "--prefill", - "auto", - "--spec", - "4", - "--mtp", - "rt", - "--max-context", - "32768", - "--kv", - "int8", - "--mmap-experts", - "--pcie-frac", - "0.55", - "--spec-min-p", - "0.70" - ], - "model_name": "qwen3.8-flash-next-iq2_xs", - "lib_dirs": [ - "\\Lib\\site-packages\\nvidia\\cu13\\bin\\x86_64" - ], - "port": 8080, - "gpu": 0, - "gpus_asked": true, - "env": { - "STRATA_FETCH_THREADS": "16" - } - }, - "launch": "strata.exe --pack iq2_xs --native Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf --ple-gguf Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf --expert-profile expert-profile.bin --expert-cache auto --prefill auto --spec 4 --mtp rt --max-context 32768 --kv int8 --mmap-experts --pcie-frac 0.55 --spec-min-p 0.70", - "engine_info": { - "model": "qwen3.8-flash-next-iq2_xs", - "max_context": 32768, - "images": false, - "version": "0.1.45", - "logit_bias": 1, - "context": 32768, - "kv": "int8", - "kv_resident": 0, - "expert_slots": 3734, - "expert_cache_mib": 5140, - "expert_slots_primary": 3734, - "expert_cache_primary_mib": 5140, - "spec": 6, - "mtp_max": 4, - "lookup": 3, - "vram_free_mib": 328, - "cvec": 0, - "arena_mib": 0, - "pool_workers": 13, - "pcie_frac": "0.55", - "spec_min_p": "0.70", - "conversation_cache_mib": 0, - "conversation_cache_slots": 4, - "conversation_cache_min_free_mib": 2560, - "conversation_cache_min_tokens": 0, - "tail_role_token": -1, - "vram_elastic": 0, - "prefill_cap": 8192, - "prefill_stages": 1, - "stream_all_min": 1024, - "prefill_pipe": 0, - "engine": "0.1.45" - }, - "vram_after": { - "elastic": false - } - }, - "results": { - "chat": { - "requests": [ - { - "ok": true, - "err": null, - "ttft_s": 0.757, - "total_s": 5.448, - "prompt_tokens": 51, - "reused_tokens": 0, - "read_tokens": 51, - "generated_tokens": 100, - "prompt_tps": 68.7, - "prompt_tps_source": "engine", - "decode_tps": 21.3, - "decode_tps_source": "engine", - "prompt_ms": 742.2, - "decode_ms": 4703.4, - "drafts_offered": 71, - "drafts_accepted": 68, - "run": 0, - "prompt": 0 - }, - { - "ok": true, - "err": null, - "ttft_s": 1.884, - "total_s": 7.382, - "prompt_tokens": 43, - "reused_tokens": 0, - "read_tokens": 43, - "generated_tokens": 100, - "prompt_tps": 23.1, - "prompt_tps_source": "engine", - "decode_tps": 18.1, - "decode_tps_source": "engine", - "prompt_ms": 1864.4, - "decode_ms": 5515.0, - "drafts_offered": 72, - "drafts_accepted": 47, - "run": 0, - "prompt": 1 - }, - { - "ok": true, - "err": null, - "ttft_s": 2.642, - "total_s": 10.398, - "prompt_tokens": 46, - "reused_tokens": 0, - "read_tokens": 46, - "generated_tokens": 100, - "prompt_tps": 17.6, - "prompt_tps_source": "engine", - "decode_tps": 12.9, - "decode_tps_source": "engine", - "prompt_ms": 2606.3, - "decode_ms": 7776.7, - "drafts_offered": 67, - "drafts_accepted": 50, - "run": 0, - "prompt": 2 - }, - { - "ok": true, - "err": null, - "ttft_s": 2.643, - "total_s": 8.517, - "prompt_tokens": 51, - "reused_tokens": 0, - "read_tokens": 51, - "generated_tokens": 100, - "prompt_tps": 19.8, - "prompt_tps_source": "engine", - "decode_tps": 16.9, - "decode_tps_source": "engine", - "prompt_ms": 2578.7, - "decode_ms": 5904.0, - "drafts_offered": 75, - "drafts_accepted": 66, - "run": 1, - "prompt": 0 - }, - { - "ok": true, - "err": null, - "ttft_s": 1.947, - "total_s": 6.733, - "prompt_tokens": 43, - "reused_tokens": 0, - "read_tokens": 43, - "generated_tokens": 100, - "prompt_tps": 22.7, - "prompt_tps_source": "engine", - "decode_tps": 20.8, - "decode_tps_source": "engine", - "prompt_ms": 1894.6, - "decode_ms": 4802.6, - "drafts_offered": 64, - "drafts_accepted": 46, - "run": 1, - "prompt": 1 - }, - { - "ok": true, - "err": null, - "ttft_s": 2.416, - "total_s": 8.107, - "prompt_tokens": 46, - "reused_tokens": 0, - "read_tokens": 46, - "generated_tokens": 100, - "prompt_tps": 19.4, - "prompt_tps_source": "engine", - "decode_tps": 17.5, - "decode_tps_source": "engine", - "prompt_ms": 2372.1, - "decode_ms": 5708.4, - "drafts_offered": 64, - "drafts_accepted": 50, - "run": 1, - "prompt": 2 - }, - { - "ok": true, - "err": null, - "ttft_s": 2.384, - "total_s": 7.285, - "prompt_tokens": 51, - "reused_tokens": 0, - "read_tokens": 51, - "generated_tokens": 100, - "prompt_tps": 22.0, - "prompt_tps_source": "engine", - "decode_tps": 20.2, - "decode_tps_source": "engine", - "prompt_ms": 2320.2, - "decode_ms": 4948.1, - "drafts_offered": 71, - "drafts_accepted": 68, - "run": 2, - "prompt": 0 - }, - { - "ok": true, - "err": null, - "ttft_s": 1.216, - "total_s": 3.492, - "prompt_tokens": 43, - "reused_tokens": 0, - "read_tokens": 43, - "generated_tokens": 100, - "prompt_tps": 37.1, - "prompt_tps_source": "engine", - "decode_tps": 43.7, - "decode_tps_source": "engine", - "prompt_ms": 1158.1, - "decode_ms": 2289.8, - "drafts_offered": 64, - "drafts_accepted": 46, - "run": 2, - "prompt": 1 - }, - { - "ok": true, - "err": null, - "ttft_s": 0.895, - "total_s": 3.376, - "prompt_tokens": 46, - "reused_tokens": 0, - "read_tokens": 46, - "generated_tokens": 100, - "prompt_tps": 52.3, - "prompt_tps_source": "engine", - "decode_tps": 40.1, - "decode_tps_source": "engine", - "prompt_ms": 879.9, - "decode_ms": 2493.4, - "drafts_offered": 67, - "drafts_accepted": 49, - "run": 2, - "prompt": 2 - } - ], - "decode_tps": { - "median": 20.2, - "min": 12.9, - "max": 43.7, - "n": 9 - }, - "ttft_s": { - "median": 1.947, - "min": 0.757, - "max": 2.643, - "n": 9 - }, - "generated_tokens": { - "median": 100, - "min": 100, - "max": 100, - "n": 9 - }, - "prompt_tokens": { - "median": 46, - "min": 43, - "max": 51, - "n": 9 - } - }, - "prompts": { - "4K": { - "target_tokens": 4096, - "requests": [ - { - "ok": true, - "err": null, - "ttft_s": 40.7, - "total_s": 46.762, - "prompt_tokens": 4019, - "reused_tokens": 0, - "read_tokens": 4019, - "generated_tokens": 24, - "prompt_tps": 98.9, - "prompt_tps_source": "engine", - "decode_tps": 3.9, - "decode_tps_source": "engine", - "prompt_ms": 40653.3, - "decode_ms": 6098.4, - "drafts_offered": 17, - "drafts_accepted": 6, - "run": 0 - }, - { - "ok": true, - "err": null, - "ttft_s": 77.747, - "total_s": 85.74, - "prompt_tokens": 4019, - "reused_tokens": 0, - "read_tokens": 4019, - "generated_tokens": 24, - "prompt_tps": 51.7, - "prompt_tps_source": "engine", - "decode_tps": 3.0, - "decode_tps_source": "engine", - "prompt_ms": 77677.0, - "decode_ms": 8029.7, - "drafts_offered": 18, - "drafts_accepted": 5, - "run": 1 - }, - { - "ok": true, - "err": null, - "ttft_s": 68.983, - "total_s": 71.0, - "prompt_tokens": 4019, - "reused_tokens": 0, - "read_tokens": 4019, - "generated_tokens": 24, - "prompt_tps": 58.3, - "prompt_tps_source": "engine", - "decode_tps": 11.6, - "decode_tps_source": "engine", - "prompt_ms": 68879.4, - "decode_ms": 2074.4, - "drafts_offered": 14, - "drafts_accepted": 8, - "run": 2 - } - ], - "prompt_tps": { - "median": 58.3, - "min": 51.7, - "max": 98.9, - "n": 3 - }, - "ttft_s": { - "median": 68.983, - "min": 40.7, - "max": 77.747, - "n": 3 - }, - "prompt_tokens": { - "median": 4019, - "min": 4019, - "max": 4019, - "n": 3 - }, - "reused_tokens": { - "median": 0, - "min": 0, - "max": 0, - "n": 3 - } - }, - "16K": { - "target_tokens": 16384, - "requests": [ - { - "ok": true, - "err": null, - "ttft_s": 129.224, - "total_s": 131.258, - "prompt_tokens": 16058, - "reused_tokens": 0, - "read_tokens": 16058, - "generated_tokens": 24, - "prompt_tps": 124.3, - "prompt_tps_source": "engine", - "decode_tps": 11.6, - "decode_tps_source": "engine", - "prompt_ms": 129138.6, - "decode_ms": 2063.1, - "drafts_offered": 19, - "drafts_accepted": 11, - "run": 0 - }, - { - "ok": true, - "err": null, - "ttft_s": 92.694, - "total_s": 96.933, - "prompt_tokens": 16058, - "reused_tokens": 0, - "read_tokens": 16058, - "generated_tokens": 24, - "prompt_tps": 173.4, - "prompt_tps_source": "engine", - "decode_tps": 5.6, - "decode_tps_source": "engine", - "prompt_ms": 92615.8, - "decode_ms": 4268.6, - "drafts_offered": 13, - "drafts_accepted": 10, - "run": 1 - }, - { - "ok": true, - "err": null, - "ttft_s": 125.993, - "total_s": 127.537, - "prompt_tokens": 16058, - "reused_tokens": 0, - "read_tokens": 16058, - "generated_tokens": 24, - "prompt_tps": 127.5, - "prompt_tps_source": "engine", - "decode_tps": 15.3, - "decode_tps_source": "engine", - "prompt_ms": 125931.2, - "decode_ms": 1571.3, - "drafts_offered": 13, - "drafts_accepted": 9, - "run": 2 - } - ], - "prompt_tps": { - "median": 127.5, - "min": 124.3, - "max": 173.4, - "n": 3 - }, - "ttft_s": { - "median": 125.993, - "min": 92.694, - "max": 129.224, - "n": 3 - }, - "prompt_tokens": { - "median": 16058, - "min": 16058, - "max": 16058, - "n": 3 - }, - "reused_tokens": { - "median": 0, - "min": 0, - "max": 0, - "n": 3 - } - } - }, - "agent": { - "sessions": [ - [ - { - "ok": true, - "err": null, - "ttft_s": 26.166, - "total_s": 28.831, - "prompt_tokens": 609, - "reused_tokens": 0, - "read_tokens": 609, - "generated_tokens": 28, - "prompt_tps": 23.4, - "prompt_tps_source": "engine", - "decode_tps": 10.4, - "decode_tps_source": "engine", - "prompt_ms": 26068.8, - "decode_ms": 2688.2, - "drafts_offered": 28, - "drafts_accepted": 17, - "run": 0, - "turn": 0 - }, - { - "ok": true, - "err": null, - "ttft_s": 37.713, - "total_s": 48.999, - "prompt_tokens": 3759, - "reused_tokens": 639, - "read_tokens": 3120, - "generated_tokens": 64, - "prompt_tps": 83.0, - "prompt_tps_source": "engine", - "decode_tps": 5.7, - "decode_tps_source": "engine", - "prompt_ms": 37575.5, - "decode_ms": 11310.7, - "drafts_offered": 55, - "drafts_accepted": 38, - "run": 0, - "turn": 1 - }, - { - "ok": true, - "err": null, - "ttft_s": 39.893, - "total_s": 46.786, - "prompt_tokens": 5292, - "reused_tokens": 3823, - "read_tokens": 1469, - "generated_tokens": 64, - "prompt_tps": 36.9, - "prompt_tps_source": "engine", - "decode_tps": 9.2, - "decode_tps_source": "engine", - "prompt_ms": 39814.3, - "decode_ms": 6921.5, - "drafts_offered": 52, - "drafts_accepted": 39, - "run": 0, - "turn": 2 - }, - { - "ok": true, - "err": null, - "ttft_s": 51.512, - "total_s": 60.261, - "prompt_tokens": 8725, - "reused_tokens": 5356, - "read_tokens": 3369, - "generated_tokens": 42, - "prompt_tps": 65.6, - "prompt_tps_source": "engine", - "decode_tps": 4.8, - "decode_tps_source": "engine", - "prompt_ms": 51378.0, - "decode_ms": 8783.5, - "drafts_offered": 31, - "drafts_accepted": 20, - "run": 0, - "turn": 3 - }, - { - "ok": true, - "err": null, - "ttft_s": 47.355, - "total_s": 57.539, - "prompt_tokens": 11628, - "reused_tokens": 8766, - "read_tokens": 2862, - "generated_tokens": 40, - "prompt_tps": 60.5, - "prompt_tps_source": "engine", - "decode_tps": 3.9, - "decode_tps_source": "engine", - "prompt_ms": 47290.3, - "decode_ms": 10207.9, - "drafts_offered": 33, - "drafts_accepted": 25, - "run": 0, - "turn": 4 - }, - { - "ok": true, - "err": null, - "ttft_s": 63.301, - "total_s": 72.549, - "prompt_tokens": 14566, - "reused_tokens": 11667, - "read_tokens": 2899, - "generated_tokens": 59, - "prompt_tps": 45.9, - "prompt_tps_source": "engine", - "decode_tps": 6.4, - "decode_tps_source": "engine", - "prompt_ms": 63211.7, - "decode_ms": 9279.9, - "drafts_offered": 49, - "drafts_accepted": 32, - "run": 0, - "turn": 5 - } - ], - [ - { - "ok": true, - "err": null, - "ttft_s": 19.494, - "total_s": 21.522, - "prompt_tokens": 609, - "reused_tokens": 0, - "read_tokens": 609, - "generated_tokens": 27, - "prompt_tps": 31.3, - "prompt_tps_source": "engine", - "decode_tps": 13.2, - "decode_tps_source": "engine", - "prompt_ms": 19435.8, - "decode_ms": 2043.0, - "drafts_offered": 28, - "drafts_accepted": 16, - "run": 1, - "turn": 0 - }, - { - "ok": true, - "err": null, - "ttft_s": 38.095, - "total_s": 44.134, - "prompt_tokens": 3758, - "reused_tokens": 638, - "read_tokens": 3120, - "generated_tokens": 64, - "prompt_tps": 82.0, - "prompt_tps_source": "engine", - "decode_tps": 10.6, - "decode_tps_source": "engine", - "prompt_ms": 38046.5, - "decode_ms": 6062.6, - "drafts_offered": 50, - "drafts_accepted": 33, - "run": 1, - "turn": 1 - }, - { - "ok": true, - "err": null, - "ttft_s": 56.918, - "total_s": 68.623, - "prompt_tokens": 5291, - "reused_tokens": 3821, - "read_tokens": 1470, - "generated_tokens": 64, - "prompt_tps": 26.1, - "prompt_tps_source": "engine", - "decode_tps": 5.2, - "decode_tps_source": "engine", - "prompt_ms": 56309.2, - "decode_ms": 12227.3, - "drafts_offered": 56, - "drafts_accepted": 37, - "run": 1, - "turn": 2 - }, - { - "ok": true, - "err": null, - "ttft_s": 72.825, - "total_s": 77.07, - "prompt_tokens": 8724, - "reused_tokens": 5284, - "read_tokens": 3440, - "generated_tokens": 41, - "prompt_tps": 47.3, - "prompt_tps_source": "engine", - "decode_tps": 9.6, - "decode_tps_source": "engine", - "prompt_ms": 72750.6, - "decode_ms": 4271.6, - "drafts_offered": 31, - "drafts_accepted": 20, - "run": 1, - "turn": 3 - }, - { - "ok": true, - "err": null, - "ttft_s": 38.257, - "total_s": 43.321, - "prompt_tokens": 11626, - "reused_tokens": 8764, - "read_tokens": 2862, - "generated_tokens": 49, - "prompt_tps": 74.9, - "prompt_tps_source": "engine", - "decode_tps": 9.6, - "decode_tps_source": "engine", - "prompt_ms": 38189.9, - "decode_ms": 5087.7, - "drafts_offered": 42, - "drafts_accepted": 30, - "run": 1, - "turn": 4 - }, - { - "ok": true, - "err": null, - "ttft_s": 59.408, - "total_s": 104.816, - "prompt_tokens": 14573, - "reused_tokens": 11674, - "read_tokens": 2899, - "generated_tokens": 30, - "prompt_tps": 50.1, - "prompt_tps_source": "engine", - "decode_tps": 0.7, - "decode_tps_source": "engine", - "prompt_ms": 57807.9, - "decode_ms": 45978.9, - "drafts_offered": 32, - "drafts_accepted": 11, - "run": 1, - "turn": 5 - } - ], - [ - { - "ok": true, - "err": null, - "ttft_s": 20.463, - "total_s": 22.428, - "prompt_tokens": 609, - "reused_tokens": 0, - "read_tokens": 609, - "generated_tokens": 30, - "prompt_tps": 29.9, - "prompt_tps_source": "engine", - "decode_tps": 15.2, - "decode_tps_source": "engine", - "prompt_ms": 20395.4, - "decode_ms": 1976.7, - "drafts_offered": 27, - "drafts_accepted": 19, - "run": 2, - "turn": 0 - }, - { - "ok": true, - "err": null, - "ttft_s": 50.054, - "total_s": 59.881, - "prompt_tokens": 3761, - "reused_tokens": 641, - "read_tokens": 3120, - "generated_tokens": 64, - "prompt_tps": 62.4, - "prompt_tps_source": "engine", - "decode_tps": 6.5, - "decode_tps_source": "engine", - "prompt_ms": 49973.2, - "decode_ms": 9853.0, - "drafts_offered": 57, - "drafts_accepted": 29, - "run": 2, - "turn": 1 - }, - { - "ok": true, - "err": null, - "ttft_s": 33.338, - "total_s": 44.452, - "prompt_tokens": 5294, - "reused_tokens": 3824, - "read_tokens": 1470, - "generated_tokens": 64, - "prompt_tps": 44.2, - "prompt_tps_source": "engine", - "decode_tps": 5.7, - "decode_tps_source": "engine", - "prompt_ms": 33275.3, - "decode_ms": 11134.6, - "drafts_offered": 59, - "drafts_accepted": 35, - "run": 2, - "turn": 2 - }, - { - "ok": true, - "err": null, - "ttft_s": 45.058, - "total_s": 53.47, - "prompt_tokens": 8727, - "reused_tokens": 5357, - "read_tokens": 3370, - "generated_tokens": 58, - "prompt_tps": 74.9, - "prompt_tps_source": "engine", - "decode_tps": 6.9, - "decode_tps_source": "engine", - "prompt_ms": 44985.6, - "decode_ms": 8443.5, - "drafts_offered": 54, - "drafts_accepted": 28, - "run": 2, - "turn": 3 - }, - { - "ok": true, - "err": null, - "ttft_s": 86.87, - "total_s": 89.135, - "prompt_tokens": 11646, - "reused_tokens": 8784, - "read_tokens": 2862, - "generated_tokens": 32, - "prompt_tps": 33.0, - "prompt_tps_source": "engine", - "decode_tps": 13.5, - "decode_tps_source": "engine", - "prompt_ms": 86720.8, - "decode_ms": 2376.9, - "drafts_offered": 24, - "drafts_accepted": 17, - "run": 2, - "turn": 4 - }, - { - "ok": true, - "err": null, - "ttft_s": 48.24, - "total_s": 108.329, - "prompt_tokens": 14576, - "reused_tokens": 11677, - "read_tokens": 2899, - "generated_tokens": 60, - "prompt_tps": 60.2, - "prompt_tps_source": "engine", - "decode_tps": 1.0, - "decode_tps_source": "engine", - "prompt_ms": 48150.3, - "decode_ms": 60129.3, - "drafts_offered": 52, - "drafts_accepted": 38, - "run": 2, - "turn": 5 - } - ] - ], - "hit_pct": { - "median": 68.799, - "min": 68.637, - "max": 68.819, - "n": 3 - }, - "reread_tokens": { - "median": 4, - "min": 2, - "max": 74, - "n": 3 - }, - "new_tokens": { - "median": 13717, - "min": 13717, - "max": 13717, - "n": 3 - }, - "turn_ttft_s": { - "median": 48.24, - "min": 33.338, - "max": 86.87, - "n": 15 - }, - "last_turn_prompt_tokens": { - "median": 14573, - "min": 14566, - "max": 14576, - "n": 3 - }, - "turn_decode_tps": { - "median": 6.4, - "min": 0.7, - "max": 13.5, - "n": 15 - }, - "first_turn_ttft_s": { - "median": 20.463, - "min": 19.494, - "max": 26.166, - "n": 3 - } - }, - "clients": {}, - "skipped": {}, - "failures": [] - } -} +{ + "schema": 1, + "tool": "tools/strata_bench.py", + "date": "2026-10-23", + "params": { + "runs": 3, + "think": false, + "clients": 0, + "long": false, + "started_server": false, + "command": "python tools/strata_bench.py" + }, + "elapsed_s": 2160.999336004257, + "machine": { + "gpu": [ + "NVIDIA GeForce RTX 5070" + ], + "gpu_vram_mib": 12227, + "gpu_driver": "591.74", + "cuda": "13.1", + "cpu": "Intel(R) Core(TM) i5-14600KF", + "cpu_cores": 14, + "cpu_threads": 20, + "ram_gib": 31.8, + "os": "Windows 10 (10.0.19045)", + "python": "3.14.0", + "strata_version": "0.1.48.1", + "strata_commit": "7ddc17a3", + "engine_version": "0.1.48.1", + "model": "qwen3.8-flash-next-iq2_xs", + "quant": "IQ2_XS", + "context": 32768, + "parallel_slots": 1, + "config": { + "exe": "strata.exe", + "args": [ + "--pack", + "iq2_xs", + "--native", + "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf", + "--ple-gguf", + "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf", + "--expert-profile", + "expert-profile.bin", + "--expert-cache", + "auto", + "--prefill", + "auto", + "--spec", + "4", + "--mtp", + "rt", + "--max-context", + "32768", + "--kv", + "int8", + "--mmap-experts", + "--pcie-frac", + "0.55", + "--spec-min-p", + "0.70" + ], + "model_name": "qwen3.8-flash-next-iq2_xs", + "lib_dirs": [ + "\\Lib\\site-packages\\nvidia\\cu13\\bin\\x86_64" + ], + "port": 8080, + "gpu": 0, + "gpus_asked": true, + "env": { + "STRATA_FETCH_THREADS": "16" + } + }, + "launch": "strata.exe --pack iq2_xs --native Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf --ple-gguf Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf --expert-profile expert-profile.bin --expert-cache auto --prefill auto --spec 4 --mtp rt --max-context 32768 --kv int8 --mmap-experts --pcie-frac 0.55 --spec-min-p 0.70", + "engine_info": { + "model": "qwen3.8-flash-next-iq2_xs", + "max_context": 32768, + "images": false, + "version": "0.1.48.1", + "logit_bias": 1, + "context": 32768, + "kv": "int8", + "kv_resident": 0, + "expert_slots": 3780, + "expert_cache_mib": 5202, + "expert_slots_primary": 3780, + "expert_cache_primary_mib": 5202, + "spec": 6, + "mtp_max": 4, + "lookup": 3, + "vram_free_mib": 337, + "cvec": 0, + "arena_mib": 0, + "pool_workers": 9, + "pcie_frac": "0.55", + "spec_min_p": "0.70", + "conversation_cache_mib": 3584, + "conversation_cache_slots": 4, + "conversation_cache_min_free_mib": 2560, + "conversation_cache_min_tokens": 0, + "tail_role_token": -1, + "vram_elastic": 0, + "prefill_cap": 8192, + "prefill_stages": 1, + "stream_all_min": 1024, + "prefill_pipe": 0, + "engine": "0.1.48.1", + "drafting": "mtp", + "drafting_note": "" + }, + "vram_after": { + "elastic": false + } + }, + "results": { + "chat": { + "requests": [ + { + "ok": true, + "err": null, + "ttft_s": 0.922, + "total_s": 7.6, + "prompt_tokens": 51, + "reused_tokens": 0, + "read_tokens": 51, + "generated_tokens": 100, + "prompt_tps": 57.3, + "prompt_tps_source": "engine", + "decode_tps": 14.9, + "decode_tps_source": "engine", + "prompt_ms": 889.9, + "decode_ms": 6690.3, + "drafts_offered": 75, + "drafts_accepted": 66, + "run": 0, + "prompt": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 1.857, + "total_s": 6.615, + "prompt_tokens": 43, + "reused_tokens": 0, + "read_tokens": 43, + "generated_tokens": 100, + "prompt_tps": 23.3, + "prompt_tps_source": "engine", + "decode_tps": 21.0, + "decode_tps_source": "engine", + "prompt_ms": 1842.4, + "decode_ms": 4763.7, + "drafts_offered": 64, + "drafts_accepted": 43, + "run": 0, + "prompt": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 2.119, + "total_s": 8.933, + "prompt_tokens": 46, + "reused_tokens": 0, + "read_tokens": 46, + "generated_tokens": 100, + "prompt_tps": 21.9, + "prompt_tps_source": "engine", + "decode_tps": 14.6, + "decode_tps_source": "engine", + "prompt_ms": 2096.5, + "decode_ms": 6826.5, + "drafts_offered": 69, + "drafts_accepted": 50, + "run": 0, + "prompt": 2 + }, + { + "ok": true, + "err": null, + "ttft_s": 0.83, + "total_s": 2.604, + "prompt_tokens": 51, + "reused_tokens": 0, + "read_tokens": 51, + "generated_tokens": 100, + "prompt_tps": 64.7, + "prompt_tps_source": "engine", + "decode_tps": 55.8, + "decode_tps_source": "engine", + "prompt_ms": 788.8, + "decode_ms": 1790.7, + "drafts_offered": 73, + "drafts_accepted": 67, + "run": 1, + "prompt": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 0.588, + "total_s": 2.334, + "prompt_tokens": 43, + "reused_tokens": 0, + "read_tokens": 43, + "generated_tokens": 100, + "prompt_tps": 75.3, + "prompt_tps_source": "engine", + "decode_tps": 56.8, + "decode_tps_source": "engine", + "prompt_ms": 570.8, + "decode_ms": 1760.1, + "drafts_offered": 66, + "drafts_accepted": 47, + "run": 1, + "prompt": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 0.73, + "total_s": 2.705, + "prompt_tokens": 46, + "reused_tokens": 0, + "read_tokens": 46, + "generated_tokens": 100, + "prompt_tps": 65.8, + "prompt_tps_source": "engine", + "decode_tps": 50.3, + "decode_tps_source": "engine", + "prompt_ms": 699.0, + "decode_ms": 1989.3, + "drafts_offered": 60, + "drafts_accepted": 43, + "run": 1, + "prompt": 2 + }, + { + "ok": true, + "err": null, + "ttft_s": 0.711, + "total_s": 2.425, + "prompt_tokens": 51, + "reused_tokens": 0, + "read_tokens": 51, + "generated_tokens": 100, + "prompt_tps": 75.7, + "prompt_tps_source": "engine", + "decode_tps": 57.8, + "decode_tps_source": "engine", + "prompt_ms": 673.6, + "decode_ms": 1730.8, + "drafts_offered": 75, + "drafts_accepted": 68, + "run": 2, + "prompt": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 0.528, + "total_s": 2.012, + "prompt_tokens": 43, + "reused_tokens": 0, + "read_tokens": 43, + "generated_tokens": 100, + "prompt_tps": 84.0, + "prompt_tps_source": "engine", + "decode_tps": 66.8, + "decode_tps_source": "engine", + "prompt_ms": 512.0, + "decode_ms": 1497.6, + "drafts_offered": 68, + "drafts_accepted": 45, + "run": 2, + "prompt": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 0.53, + "total_s": 2.157, + "prompt_tokens": 46, + "reused_tokens": 0, + "read_tokens": 46, + "generated_tokens": 100, + "prompt_tps": 89.3, + "prompt_tps_source": "engine", + "decode_tps": 61.0, + "decode_tps_source": "engine", + "prompt_ms": 515.0, + "decode_ms": 1639.2, + "drafts_offered": 74, + "drafts_accepted": 50, + "run": 2, + "prompt": 2 + } + ], + "decode_tps": { + "median": 55.8, + "min": 14.6, + "max": 66.8, + "n": 9 + }, + "ttft_s": { + "median": 0.73, + "min": 0.528, + "max": 2.119, + "n": 9 + }, + "generated_tokens": { + "median": 100, + "min": 100, + "max": 100, + "n": 9 + }, + "prompt_tokens": { + "median": 46, + "min": 43, + "max": 51, + "n": 9 + } + }, + "prompts": { + "4K": { + "target_tokens": 4096, + "requests": [ + { + "ok": true, + "err": null, + "ttft_s": 41.626, + "total_s": 45.653, + "prompt_tokens": 4019, + "reused_tokens": 0, + "read_tokens": 4019, + "generated_tokens": 24, + "prompt_tps": 96.7, + "prompt_tps_source": "engine", + "decode_tps": 5.9, + "decode_tps_source": "engine", + "prompt_ms": 41560.0, + "decode_ms": 4066.5, + "drafts_offered": 17, + "drafts_accepted": 6, + "run": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 46.27, + "total_s": 49.226, + "prompt_tokens": 4019, + "reused_tokens": 0, + "read_tokens": 4019, + "generated_tokens": 24, + "prompt_tps": 87.0, + "prompt_tps_source": "engine", + "decode_tps": 7.9, + "decode_tps_source": "engine", + "prompt_ms": 46169.7, + "decode_ms": 3020.1, + "drafts_offered": 16, + "drafts_accepted": 11, + "run": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 70.809, + "total_s": 73.234, + "prompt_tokens": 4019, + "reused_tokens": 0, + "read_tokens": 4019, + "generated_tokens": 24, + "prompt_tps": 56.9, + "prompt_tps_source": "engine", + "decode_tps": 9.6, + "decode_tps_source": "engine", + "prompt_ms": 70693.3, + "decode_ms": 2502.9, + "drafts_offered": 16, + "drafts_accepted": 11, + "run": 2 + } + ], + "prompt_tps": { + "median": 87.0, + "min": 56.9, + "max": 96.7, + "n": 3 + }, + "ttft_s": { + "median": 46.27, + "min": 41.626, + "max": 70.809, + "n": 3 + }, + "prompt_tokens": { + "median": 4019, + "min": 4019, + "max": 4019, + "n": 3 + }, + "reused_tokens": { + "median": 0, + "min": 0, + "max": 0, + "n": 3 + } + }, + "16K": { + "target_tokens": 16384, + "requests": [ + { + "ok": true, + "err": null, + "ttft_s": 240.989, + "total_s": 243.199, + "prompt_tokens": 16058, + "reused_tokens": 0, + "read_tokens": 16058, + "generated_tokens": 24, + "prompt_tps": 66.7, + "prompt_tps_source": "engine", + "decode_tps": 10.7, + "decode_tps_source": "engine", + "prompt_ms": 240897.1, + "decode_ms": 2238.4, + "drafts_offered": 19, + "drafts_accepted": 8, + "run": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 158.668, + "total_s": 160.377, + "prompt_tokens": 16058, + "reused_tokens": 0, + "read_tokens": 16058, + "generated_tokens": 24, + "prompt_tps": 101.3, + "prompt_tps_source": "engine", + "decode_tps": 13.8, + "decode_tps_source": "engine", + "prompt_ms": 158573.2, + "decode_ms": 1737.3, + "drafts_offered": 13, + "drafts_accepted": 10, + "run": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 152.524, + "total_s": 154.604, + "prompt_tokens": 16058, + "reused_tokens": 0, + "read_tokens": 16058, + "generated_tokens": 24, + "prompt_tps": 105.3, + "prompt_tps_source": "engine", + "decode_tps": 11.4, + "decode_tps_source": "engine", + "prompt_ms": 152442.8, + "decode_ms": 2111.2, + "drafts_offered": 19, + "drafts_accepted": 8, + "run": 2 + } + ], + "prompt_tps": { + "median": 101.3, + "min": 66.7, + "max": 105.3, + "n": 3 + }, + "ttft_s": { + "median": 158.668, + "min": 152.524, + "max": 240.989, + "n": 3 + }, + "prompt_tokens": { + "median": 16058, + "min": 16058, + "max": 16058, + "n": 3 + }, + "reused_tokens": { + "median": 0, + "min": 0, + "max": 0, + "n": 3 + } + } + }, + "agent": { + "sessions": [ + [ + { + "ok": true, + "err": null, + "ttft_s": 29.207, + "total_s": 32.246, + "prompt_tokens": 609, + "reused_tokens": 0, + "read_tokens": 609, + "generated_tokens": 30, + "prompt_tps": 20.9, + "prompt_tps_source": "engine", + "decode_tps": 9.8, + "decode_tps_source": "engine", + "prompt_ms": 29096.9, + "decode_ms": 3067.7, + "drafts_offered": 33, + "drafts_accepted": 17, + "run": 0, + "turn": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 62.821, + "total_s": 69.608, + "prompt_tokens": 4109, + "reused_tokens": 641, + "read_tokens": 3468, + "generated_tokens": 64, + "prompt_tps": 55.3, + "prompt_tps_source": "engine", + "decode_tps": 9.4, + "decode_tps_source": "engine", + "prompt_ms": 62679.5, + "decode_ms": 6806.7, + "drafts_offered": 58, + "drafts_accepted": 38, + "run": 0, + "turn": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 46.985, + "total_s": 78.305, + "prompt_tokens": 5642, + "reused_tokens": 4173, + "read_tokens": 1469, + "generated_tokens": 64, + "prompt_tps": 31.4, + "prompt_tps_source": "engine", + "decode_tps": 2.0, + "decode_tps_source": "engine", + "prompt_ms": 46802.3, + "decode_ms": 31413.5, + "drafts_offered": 51, + "drafts_accepted": 42, + "run": 0, + "turn": 2 + }, + { + "ok": true, + "err": null, + "ttft_s": 53.442, + "total_s": 123.043, + "prompt_tokens": 9075, + "reused_tokens": 5635, + "read_tokens": 3440, + "generated_tokens": 46, + "prompt_tps": 64.5, + "prompt_tps_source": "engine", + "decode_tps": 0.7, + "decode_tps_source": "engine", + "prompt_ms": 53298.4, + "decode_ms": 69691.1, + "drafts_offered": 39, + "drafts_accepted": 18, + "run": 0, + "turn": 3 + }, + { + "ok": true, + "err": null, + "ttft_s": 52.172, + "total_s": 100.882, + "prompt_tokens": 11828, + "reused_tokens": 9120, + "read_tokens": 2708, + "generated_tokens": 57, + "prompt_tps": 52.0, + "prompt_tps_source": "engine", + "decode_tps": 1.2, + "decode_tps_source": "engine", + "prompt_ms": 52048.2, + "decode_ms": 48761.5, + "drafts_offered": 61, + "drafts_accepted": 32, + "run": 0, + "turn": 4 + }, + { + "ok": true, + "err": null, + "ttft_s": 80.738, + "total_s": 105.696, + "prompt_tokens": 14601, + "reused_tokens": 11884, + "read_tokens": 2717, + "generated_tokens": 64, + "prompt_tps": 34.1, + "prompt_tps_source": "engine", + "decode_tps": 2.5, + "decode_tps_source": "engine", + "prompt_ms": 79757.1, + "decode_ms": 25820.2, + "drafts_offered": 51, + "drafts_accepted": 30, + "run": 0, + "turn": 5 + } + ], + [ + { + "ok": true, + "err": null, + "ttft_s": 24.909, + "total_s": 27.129, + "prompt_tokens": 609, + "reused_tokens": 0, + "read_tokens": 609, + "generated_tokens": 29, + "prompt_tps": 24.5, + "prompt_tps_source": "engine", + "decode_tps": 12.9, + "decode_tps_source": "engine", + "prompt_ms": 24843.4, + "decode_ms": 2256.6, + "drafts_offered": 24, + "drafts_accepted": 19, + "run": 1, + "turn": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 49.236, + "total_s": 54.806, + "prompt_tokens": 4108, + "reused_tokens": 640, + "read_tokens": 3468, + "generated_tokens": 64, + "prompt_tps": 70.5, + "prompt_tps_source": "engine", + "decode_tps": 11.4, + "decode_tps_source": "engine", + "prompt_ms": 49157.2, + "decode_ms": 5595.2, + "drafts_offered": 54, + "drafts_accepted": 38, + "run": 1, + "turn": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 55.615, + "total_s": 141.52, + "prompt_tokens": 5641, + "reused_tokens": 4101, + "read_tokens": 1540, + "generated_tokens": 64, + "prompt_tps": 27.7, + "prompt_tps_source": "engine", + "decode_tps": 0.7, + "decode_tps_source": "engine", + "prompt_ms": 55532.3, + "decode_ms": 85940.1, + "drafts_offered": 48, + "drafts_accepted": 36, + "run": 1, + "turn": 2 + }, + { + "ok": true, + "err": null, + "ttft_s": 77.664, + "total_s": 91.416, + "prompt_tokens": 9074, + "reused_tokens": 5634, + "read_tokens": 3440, + "generated_tokens": 64, + "prompt_tps": 45.6, + "prompt_tps_source": "engine", + "decode_tps": 4.3, + "decode_tps_source": "engine", + "prompt_ms": 75405.1, + "decode_ms": 14872.1, + "drafts_offered": 41, + "drafts_accepted": 27, + "run": 1, + "turn": 3 + }, + { + "ok": true, + "err": null, + "ttft_s": 65.818, + "total_s": 69.869, + "prompt_tokens": 11846, + "reused_tokens": 9137, + "read_tokens": 2709, + "generated_tokens": 59, + "prompt_tps": 41.9, + "prompt_tps_source": "engine", + "decode_tps": 14.4, + "decode_tps_source": "engine", + "prompt_ms": 64702.1, + "decode_ms": 4083.7, + "drafts_offered": 54, + "drafts_accepted": 30, + "run": 1, + "turn": 4 + }, + { + "ok": true, + "err": null, + "ttft_s": 77.776, + "total_s": 89.132, + "prompt_tokens": 14621, + "reused_tokens": 11904, + "read_tokens": 2717, + "generated_tokens": 64, + "prompt_tps": 35.0, + "prompt_tps_source": "engine", + "decode_tps": 5.6, + "decode_tps_source": "engine", + "prompt_ms": 77689.0, + "decode_ms": 11413.5, + "drafts_offered": 48, + "drafts_accepted": 30, + "run": 1, + "turn": 5 + } + ], + [ + { + "ok": true, + "err": null, + "ttft_s": 24.119, + "total_s": 25.949, + "prompt_tokens": 609, + "reused_tokens": 0, + "read_tokens": 609, + "generated_tokens": 19, + "prompt_tps": 25.3, + "prompt_tps_source": "engine", + "decode_tps": 10.3, + "decode_tps_source": "engine", + "prompt_ms": 24058.0, + "decode_ms": 1843.6, + "drafts_offered": 18, + "drafts_accepted": 11, + "run": 2, + "turn": 0 + }, + { + "ok": true, + "err": null, + "ttft_s": 50.742, + "total_s": 56.062, + "prompt_tokens": 4098, + "reused_tokens": 628, + "read_tokens": 3470, + "generated_tokens": 64, + "prompt_tps": 68.5, + "prompt_tps_source": "engine", + "decode_tps": 12.0, + "decode_tps_source": "engine", + "prompt_ms": 50672.2, + "decode_ms": 5341.1, + "drafts_offered": 55, + "drafts_accepted": 39, + "run": 2, + "turn": 1 + }, + { + "ok": true, + "err": null, + "ttft_s": 41.852, + "total_s": 52.966, + "prompt_tokens": 5631, + "reused_tokens": 4091, + "read_tokens": 1540, + "generated_tokens": 64, + "prompt_tps": 36.9, + "prompt_tps_source": "engine", + "decode_tps": 5.7, + "decode_tps_source": "engine", + "prompt_ms": 41764.9, + "decode_ms": 11161.1, + "drafts_offered": 56, + "drafts_accepted": 37, + "run": 2, + "turn": 2 + }, + { + "ok": true, + "err": null, + "ttft_s": 76.891, + "total_s": 82.941, + "prompt_tokens": 9063, + "reused_tokens": 5694, + "read_tokens": 3369, + "generated_tokens": 40, + "prompt_tps": 43.9, + "prompt_tps_source": "engine", + "decode_tps": 6.5, + "decode_tps_source": "engine", + "prompt_ms": 76780.9, + "decode_ms": 6109.0, + "drafts_offered": 31, + "drafts_accepted": 18, + "run": 2, + "turn": 3 + }, + { + "ok": true, + "err": null, + "ttft_s": 77.969, + "total_s": 82.274, + "prompt_tokens": 11810, + "reused_tokens": 9102, + "read_tokens": 2708, + "generated_tokens": 55, + "prompt_tps": 34.8, + "prompt_tps_source": "engine", + "decode_tps": 12.6, + "decode_tps_source": "engine", + "prompt_ms": 77873.5, + "decode_ms": 4357.4, + "drafts_offered": 53, + "drafts_accepted": 35, + "run": 2, + "turn": 4 + }, + { + "ok": true, + "err": null, + "ttft_s": 76.536, + "total_s": 81.291, + "prompt_tokens": 14581, + "reused_tokens": 11866, + "read_tokens": 2715, + "generated_tokens": 50, + "prompt_tps": 35.5, + "prompt_tps_source": "engine", + "decode_tps": 10.4, + "decode_tps_source": "engine", + "prompt_ms": 76443.7, + "decode_ms": 4796.3, + "drafts_offered": 41, + "drafts_accepted": 24, + "run": 2, + "turn": 5 + } + ] + ], + "hit_pct": { + "median": 69.453, + "min": 69.366, + "max": 69.502, + "n": 3 + }, + "reread_tokens": { + "median": 73, + "min": 73, + "max": 144, + "n": 3 + }, + "new_tokens": { + "median": 13729, + "min": 13729, + "max": 13730, + "n": 3 + }, + "turn_ttft_s": { + "median": 62.821, + "min": 41.852, + "max": 80.738, + "n": 15 + }, + "last_turn_prompt_tokens": { + "median": 14601, + "min": 14581, + "max": 14621, + "n": 3 + }, + "turn_decode_tps": { + "median": 5.7, + "min": 0.7, + "max": 14.4, + "n": 15 + }, + "first_turn_ttft_s": { + "median": 24.909, + "min": 24.119, + "max": 29.207, + "n": 3 + } + }, + "clients": {}, + "skipped": {}, + "failures": [] + } +}