From 8e6f55c655ea9369addca20a5bbb0f18ea0277e7 Mon Sep 17 00:00:00 2001 From: Unbug Lee Date: Sun, 27 Sep 2026 00:54:10 +0800 Subject: [PATCH] server : fix usage.completion_tokens undercount with n > 1 Assisted-by: Copilot --- tools/server/server-context.cpp | 10 ++++++++++ tools/server/tests/unit/test_chat_completion.py | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index e95fb63ab3c..b4f1c7e5764 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -4367,10 +4367,20 @@ std::unique_ptr server_routes::handle_completions_impl( res->ok(arr[0]); } else if (res_type == TASK_RESPONSE_TYPE_OAI_CHAT || res_type == TASK_RESPONSE_TYPE_OAI_CMPL) { // if multiple results in OAI format, we need to re-format them + json & usage = arr[0]["usage"]; json & choices = arr[0]["choices"]; for (size_t i = 1; i < arr.size(); i++) { choices.push_back(std::move(arr[i]["choices"][0])); } + + // OAI counts generated tokens across all choices + int64_t n_completion_tokens = 0; + for (auto & item : arr) { + n_completion_tokens += item["usage"]["completion_tokens"].get(); + } + usage["completion_tokens"] = n_completion_tokens; + usage["total_tokens"] = usage["prompt_tokens"].get() + n_completion_tokens; + res->ok(arr[0]); } else { // multi-results, non-OAI compat diff --git a/tools/server/tests/unit/test_chat_completion.py b/tools/server/tests/unit/test_chat_completion.py index 0258b539ed8..afbcef03066 100644 --- a/tools/server/tests/unit/test_chat_completion.py +++ b/tools/server/tests/unit/test_chat_completion.py @@ -587,6 +587,10 @@ def test_chat_completions_multiple_choices(): for choice in res.body["choices"]: assert "assistant" == choice["message"]["role"] assert choice["finish_reason"] == "length" + # each choice generates max_tokens tokens, usage counts them all + # ref: https://github.com/ggml-org/llama.cpp/issues/29451 + assert res.body["usage"]["completion_tokens"] == 16 + assert res.body["usage"]["total_tokens"] == res.body["usage"]["prompt_tokens"] + 16 def test_chat_completions_token_count():