From 0ce5908aac568eb316c54f8393d1c705aeee944a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:27:46 +0000 Subject: [PATCH] server: support Responses assistant message phases --- common/chat-peg-parser.cpp | 5 + common/chat-peg-parser.h | 3 + common/chat.cpp | 42 ++- common/chat.h | 7 +- .../templates/llama-cpp-qwen3.8-codex.jinja | 37 ++- tests/test-qwen38-codex-template.cpp | 248 +++++++++++++++++- tools/server/server-chat.cpp | 22 +- tools/server/server-task.cpp | 30 ++- tools/server/server-task.h | 1 + 9 files changed, 371 insertions(+), 24 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 06737b165c05..669c8961d043 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -318,6 +318,7 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) { // Handle reasoning/content tags bool is_reasoning = node.tag == common_chat_peg_builder::REASONING; bool is_content = node.tag == common_chat_peg_builder::CONTENT; + bool is_phase = node.tag == common_chat_peg_builder::PHASE; if (is_reasoning) { // GPT OSS can have more than 1 reasoning block, so concatenate here result.reasoning_content += std::string(node.text); @@ -329,6 +330,10 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) { result.content += std::string(node.text); } + if (is_phase) { + result.phase = std::string(node.text); + } + // Handle tool-related tags (supporting both JSON and tagged formats) bool is_tool_open = node.tag == common_chat_peg_builder::TOOL_OPEN; bool is_tool_close = node.tag == common_chat_peg_builder::TOOL_CLOSE; diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 5d764dbaa0ec..3ebd51a32ff5 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -63,6 +63,7 @@ class common_chat_peg_builder : public common_peg_parser_builder { static constexpr const char * REASONING_BLOCK = "reasoning-block"; static constexpr const char * REASONING = "reasoning"; static constexpr const char * CONTENT = "content"; + static constexpr const char * PHASE = "phase"; // Tag constants static constexpr const char * TOOL = "tool"; @@ -85,6 +86,8 @@ class common_chat_peg_builder : public common_peg_parser_builder { common_peg_parser content(const common_peg_parser & p) { return tag(CONTENT, p); } + common_peg_parser phase(const common_peg_parser & p) { return atomic(tag(PHASE, p)); } + common_peg_parser tag_with_safe_content(const std::string & tag_name, const std::string & marker, const common_peg_parser & p); diff --git a/common/chat.cpp b/common/chat.cpp index a1001ec351e2..57e758620d47 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -452,6 +452,12 @@ std::vector common_chat_msgs_parse_oaicompat(const json & messa if (message.contains("reasoning_content")) { msg.reasoning_content = message.at("reasoning_content"); } + if (message.contains("phase") && !message.at("phase").is_null()) { + if (!message.at("phase").is_string()) { + throw std::invalid_argument("Invalid 'phase' type: expected string"); + } + msg.phase = message.at("phase"); + } if (message.contains("name")) { msg.tool_name = message.at("name"); } @@ -534,14 +540,21 @@ struct messages_inp_normalizer { } }; -static json render_message_to_json(const std::vector & msgs, const jinja::caps & c) { +static json render_message_to_json( + const std::vector & msgs, + const jinja::caps & c, + bool include_phase = false) { if (!c.supports_string_content && !c.supports_typed_content) { LOG_WRN("%s: Neither string content nor typed content is supported by the template. This is unexpected and may lead to issues.\n", __func__); } json messages = json::array(); for (const auto & msg : msgs) { - messages.push_back(msg.to_json_oaicompat(/* concat_typed_text= */ false)); + json rendered = msg.to_json_oaicompat(/* concat_typed_text= */ false); + if (include_phase && !msg.phase.empty()) { + rendered["phase"] = msg.phase; + } + messages.push_back(std::move(rendered)); } return messages_inp_normalizer(c).normalize(messages); } @@ -1198,6 +1211,10 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_ auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); const bool json_string_args = tmpl.source().find("llama.cpp:xml-string-args=json") != std::string::npos; + const bool responses_phase_protocol = + tmpl.source().find("llama.cpp:responses-phase=marker-v1") != std::string::npos && + inputs.extra_context.is_object() && + inputs.extra_context.value("responses_phase_protocol", false); if (inputs.has_continuation()) { const auto & msg = inputs.continue_msg; @@ -1236,9 +1253,18 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_ (p.literal("") | p.peek(p.literal("")))); } + auto phase = p.eps(); + if (responses_phase_protocol) { + auto phase_block = p.atomic( + p.literal("") + + p.phase(p.literal("commentary") | p.literal("final_answer")) + + p.literal("")); + phase = p.optional(phase_block + p.space()); + } + // Response format parser if (has_response_format) { - return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema))); + return generation_prompt + (reasoning << phase << p.content(p.schema(p.json(), "response-format", inputs.json_schema))); } // Tool call parser @@ -1302,11 +1328,11 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_ auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1)); return generation_prompt + - (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls); + (reasoning << phase << p.content(p.until_one_of(tool_call_starts)) << tool_calls); } // Content only parser - return generation_prompt + (reasoning << p.content(p.rest())); + return generation_prompt + (reasoning << phase << p.content(p.rest())); }); data.parser = parser.save(); @@ -3619,7 +3645,11 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ workaround::trim_all_content(trimmed_messages); messages_to_render = &trimmed_messages; } - params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps()); + const auto phase_protocol_it = inputs.chat_template_kwargs.find("responses_phase_protocol"); + const bool responses_phase_protocol = + phase_protocol_it != inputs.chat_template_kwargs.end() && phase_protocol_it->second == "true"; + params.messages = render_message_to_json( + *messages_to_render, tmpl.original_caps(), responses_phase_protocol); params.tool_choice = inputs.tool_choice; params.reasoning_format = inputs.reasoning_format; params.enable_thinking = inputs.enable_thinking; diff --git a/common/chat.h b/common/chat.h index 6d5b220aebb5..5bdc760e5d6a 100644 --- a/common/chat.h +++ b/common/chat.h @@ -86,6 +86,9 @@ struct common_chat_msg { std::string reasoning_content; std::string tool_name; std::string tool_call_id; + // Optional assistant lifecycle metadata used by Responses-capable parsers. + // Keep this last so existing aggregate initializers retain their field mapping. + std::string phase; nlohmann::ordered_json to_json_oaicompat(bool concat_typed_text = false) const; @@ -93,7 +96,7 @@ struct common_chat_msg { bool empty() const { return content.empty() && content_parts.empty() && tool_calls.empty() && reasoning_content.empty() && - tool_name.empty() && tool_call_id.empty(); + phase.empty() && tool_name.empty() && tool_call_id.empty(); } bool contains_media() const { @@ -122,7 +125,7 @@ struct common_chat_msg { bool operator==(const common_chat_msg & other) const { return role == other.role && content == other.content && content_parts == other.content_parts && tool_calls == other.tool_calls && reasoning_content == other.reasoning_content && - tool_name == other.tool_name && tool_call_id == other.tool_call_id; + phase == other.phase && tool_name == other.tool_name && tool_call_id == other.tool_call_id; } bool operator!=(const common_chat_msg & other) const { return !(*this == other); } diff --git a/models/templates/llama-cpp-qwen3.8-codex.jinja b/models/templates/llama-cpp-qwen3.8-codex.jinja index ba7decc0e172..8d326c950823 100644 --- a/models/templates/llama-cpp-qwen3.8-codex.jinja +++ b/models/templates/llama-cpp-qwen3.8-codex.jinja @@ -1,5 +1,6 @@ {%- set template_version = "qwen3.8-codex-safe-v1" %} {#- llama.cpp:xml-string-args=json -#} +{#- llama.cpp:responses-phase=marker-v1 -#} {#- ------------------------------------------------------------------------- Whitespace contract: every tag opener in this file carries a leading minus, so it strips the whitespace and newlines that precede it. All emitted text @@ -26,6 +27,9 @@ {%- endif %} {%- set add_vision_id = add_vision_id if add_vision_id is defined else false %} {%- set enable_thinking = enable_thinking if enable_thinking is defined else true %} +{%- set responses_phase_protocol = + responses_phase_protocol if responses_phase_protocol is defined else false +%} {%- set auto_disable_thinking_with_tools = auto_disable_thinking_with_tools if auto_disable_thinking_with_tools is defined @@ -159,6 +163,20 @@ %} {%- endif %} {%- endif %} +{%- if responses_phase_protocol %} + {%- set phase_instructions = + 'Responses message phases are transport metadata and are separate from reasoning. Emit exactly one internal phase block immediately after and before any visible assistant text or tool call: commentary when the visible assistant message is an intermediate update before tool use or further work, or final_answer when the visible assistant message is the terminal answer of the turn. If the template has already supplied an empty block, begin your generation with the phase block. Never repeat or mention the phase block in visible text.' + %} + {%- if reasoning_instructions %} + {%- set reasoning_instructions = + reasoning_instructions ~ ' + +' ~ phase_instructions + %} + {%- else %} + {%- set reasoning_instructions = phase_instructions %} + {%- endif %} +{%- endif %} {#- ------------------------------------------------------------------------- Content rendering ------------------------------------------------------------------------- -#} @@ -378,17 +396,32 @@ {%- endif %} {%- endif %} {%- set reasoning_content = reasoning_content | trim %} + {%- set phase_prefix = '' %} + {%- set after_think = '\n\n' %} + {%- if responses_phase_protocol + and message.phase is defined + and message.phase is not none + and message.phase %} + {%- if message.phase not in ('commentary', 'final_answer') %} + {{- raise_exception('Unexpected Responses message phase ' ~ message.phase ~ '.') }} + {%- endif %} + {%- set phase_prefix = + '' ~ message.phase ~ '\n' + %} + {%- set after_think = '\n' ~ phase_prefix %} + {%- endif %} {#- Preserve the original invariant: when thinking is preserved, always reconstruct the thinking block, even when it is empty. -#} {%- if _preserve_thinking or loop.index0 > ns.last_query_index %} {{- '<|im_start|>assistant\n\n' ~ reasoning_content - ~ '\n\n\n' + ~ '\n' + ~ after_think ~ content }} {%- else %} - {{- '<|im_start|>assistant\n' ~ content }} + {{- '<|im_start|>assistant\n' ~ phase_prefix ~ content }} {%- endif %} {%- if message.tool_calls is defined and message.tool_calls is not none diff --git a/tests/test-qwen38-codex-template.cpp b/tests/test-qwen38-codex-template.cpp index 55e81a0f926b..d8c1b410bec2 100644 --- a/tests/test-qwen38-codex-template.cpp +++ b/tests/test-qwen38-codex-template.cpp @@ -39,6 +39,12 @@ int main() { const std::string template_path = "models/templates/llama-cpp-qwen3.8-codex.jinja"; auto tmpls = common_chat_templates_ptr(common_chat_templates_init(nullptr, read_file(template_path))); + const std::string template_source = read_file(template_path); + if (template_source.find("llama.cpp:responses-phase=marker-v1") == std::string::npos) { + std::cerr << "Codex template did not opt into the Responses phase protocol\n"; + return 1; + } + const auto caps = common_chat_templates_get_caps(tmpls.get()); const auto supports_object_arguments = caps.find("supports_object_arguments"); if (supports_object_arguments == caps.end() || !supports_object_arguments->second) { @@ -548,6 +554,246 @@ int main() { return 1; } - std::cout << "Qwen3.8 Codex user text, generic string framing, custom framing, and compaction tests passed\n"; + // Responses phase is internal metadata: it must survive Responses history -> + // template rendering, but never leak through normal Chat Completions JSON. + common_chat_msg chat_only_phase = message("assistant", "Internal metadata test."); + chat_only_phase.phase = "commentary"; + if (chat_only_phase.to_json_oaicompat().contains("phase")) { + std::cerr << "Responses phase leaked through Chat Completions serialization\n"; + return 1; + } + + const nlohmann::ordered_json phase_history_request = { + {"model", "test-model"}, + {"input", nlohmann::ordered_json::array({ + { + {"type", "message"}, + {"role", "user"}, + {"content", nlohmann::ordered_json::array({ + {{"type", "input_text"}, {"text", "Inspect the repository."}}, + })}, + }, + { + {"type", "message"}, + {"role", "assistant"}, + {"status", "completed"}, + {"phase", "commentary"}, + {"content", nlohmann::ordered_json::array({ + {{"type", "output_text"}, {"text", "Checking the repository."}}, + })}, + }, + })}, + }; + const auto converted_phase_history = server_chat_convert_responses_to_chatcmpl(phase_history_request); + if (!converted_phase_history.contains("chat_template_kwargs") || + !converted_phase_history.at("chat_template_kwargs").value("responses_phase_protocol", false) || + !converted_phase_history.contains("messages") || + converted_phase_history.at("messages").size() != 2 || + converted_phase_history.at("messages")[1].value("phase", std::string()) != "commentary") { + std::cerr << "Responses phase metadata was not preserved for the internal template bridge\n"; + return 1; + } + + common_chat_templates_inputs phase_inputs; + phase_inputs.messages = common_chat_msgs_parse_oaicompat(converted_phase_history.at("messages")); + phase_inputs.tools = { shell_tool }; + phase_inputs.add_generation_prompt = true; + phase_inputs.enable_thinking = true; + phase_inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; + phase_inputs.chat_template_kwargs["responses_phase_protocol"] = "true"; + const auto phase_params = common_chat_templates_apply(tmpls.get(), phase_inputs); + + const std::string historical_phase_marker = + "\ncommentary\nChecking the repository."; + if (phase_params.prompt.find(historical_phase_marker) == std::string::npos || + phase_params.prompt.find("Responses message phases are transport metadata") == std::string::npos) { + std::cerr << "Qwen phase protocol was not taught/replayed through the Codex template\n"; + return 1; + } + + common_chat_parser_params phase_parser(phase_params); + phase_parser.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; + phase_parser.parser.load(phase_params.parser); + + const std::string commentary_marker = "commentary"; + const std::string generated_commentary = + std::string("I should inspect the workspace.\n\n") + commentary_marker + "\n" + "I'll inspect the repository.\n\n" + "\n" + "\n" + "\n" + + nlohmann::ordered_json("pwd").dump() + + "\n\n" + "\n" + ""; + const auto parsed_commentary = common_chat_parse(generated_commentary, false, phase_parser); + if (parsed_commentary.phase != "commentary" || + parsed_commentary.content.find("response_phase") != std::string::npos || + parsed_commentary.reasoning_content.find("response_phase") != std::string::npos || + parsed_commentary.tool_calls.size() != 1) { + std::cerr << "Qwen commentary phase marker was not consumed as metadata\n"; + return 1; + } + + // Exercise partial parsing of the internal marker. No marker prefix may + // escape into a streamed reasoning/content delta while it is incomplete. + for (size_t n = 1; n < commentary_marker.size(); ++n) { + const auto partial_phase = common_chat_parse( + "I should inspect the workspace.\n\n" + commentary_marker.substr(0, n), true, phase_parser); + if (partial_phase.content.find("response_phase") != std::string::npos || + partial_phase.reasoning_content.find("response_phase") != std::string::npos) { + std::cerr << "Partial Qwen phase marker leaked into streamed model text\n"; + return 1; + } + } + + const std::string generated_final = + "The task is complete.\n\nfinal_answer\nDone."; + const auto parsed_final = common_chat_parse(generated_final, false, phase_parser); + if (parsed_final.phase != "final_answer" || parsed_final.content != "Done." || + parsed_final.reasoning_content.find("response_phase") != std::string::npos) { + std::cerr << "Qwen final_answer phase marker was not consumed as metadata\n"; + return 1; + } + + // The model signal is preferred, but omission must remain backward + // compatible: the old Qwen parse still succeeds and leaves phase empty so + // Responses egress can apply its structural fallback. + const auto parsed_unmarked = common_chat_parse( + "Legacy reasoning.\n\n\nLegacy final text.", false, phase_parser); + if (!parsed_unmarked.phase.empty() || parsed_unmarked.content != "Legacy final text.") { + std::cerr << "Unmarked Qwen output did not preserve Responses phase fallback\n"; + return 1; + } + + auto no_think_inputs = phase_inputs; + no_think_inputs.messages = { system, user }; + no_think_inputs.enable_thinking = false; + const auto no_think_params = common_chat_templates_apply(tmpls.get(), no_think_inputs); + if (no_think_params.generation_prompt.find("\n\n\n\n") == std::string::npos) { + std::cerr << "Qwen thinking-disabled generation prompt lost its empty think framing\n"; + return 1; + } + common_chat_parser_params no_think_parser(no_think_params); + no_think_parser.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; + no_think_parser.parser.load(no_think_params.parser); + const auto parsed_no_think = common_chat_parse( + "final_answer\nDone without thinking.", false, no_think_parser); + if (parsed_no_think.phase != "final_answer" || parsed_no_think.content != "Done without thinking.") { + std::cerr << "Qwen phase protocol depended on model-generated reasoning\n"; + return 1; + } + + task_result_state final_phase_stream_state(phase_parser, {}, false); + server_task_result_cmpl_partial final_phase_partial; + final_phase_partial.res_type = TASK_RESPONSE_TYPE_OAI_RESP; + final_phase_partial.content = generated_final; + final_phase_partial.n_decoded = 1; + final_phase_partial.update(final_phase_stream_state); + bool saw_final_phase_added = false; + for (const auto & event : final_phase_partial.to_json_oaicompat_resp()) { + if (event.value("event", std::string()) == "response.output_item.added" && + event.contains("data") && event.at("data").contains("item") && + event.at("data").at("item").value("type", std::string()) == "message") { + saw_final_phase_added = + event.at("data").at("item").value("phase", std::string()) == "final_answer"; + } + } + if (!saw_final_phase_added) { + std::cerr << "Streaming Responses output_item.added lost final_answer phase\n"; + return 1; + } + + server_task_result_cmpl_final final_phase_result; + final_phase_result.oaicompat_model = "test-model"; + final_phase_result.oai_resp_id = "resp_final_phase_test"; + final_phase_result.oai_resp_message_id = "msg_final_phase_test"; + final_phase_result.n_prompt_tokens = 0; + final_phase_result.n_prompt_tokens_cache = 0; + final_phase_result.n_decoded = 0; + final_phase_result.oaicompat_msg = parsed_final; + final_phase_result.oaicompat_msg.role = "assistant"; + + const auto final_phase_response = final_phase_result.to_json_oaicompat_resp(); + bool saw_final_phase_nonstream = false; + if (final_phase_response.contains("output") && final_phase_response.at("output").is_array()) { + for (const auto & item : final_phase_response.at("output")) { + if (item.value("type", std::string()) == "message") { + saw_final_phase_nonstream = item.value("phase", std::string()) == "final_answer"; + break; + } + } + } + if (!saw_final_phase_nonstream) { + std::cerr << "Responses egress lost the model-selected final_answer phase\n"; + return 1; + } + + bool saw_final_phase_done = false; + bool saw_final_phase_completed = false; + for (const auto & event : final_phase_result.to_json_oaicompat_resp_stream()) { + if (event.value("event", std::string()) == "response.output_item.done" && + event.contains("data") && event.at("data").contains("item") && + event.at("data").at("item").value("type", std::string()) == "message") { + saw_final_phase_done = + event.at("data").at("item").value("phase", std::string()) == "final_answer"; + } + if (event.value("event", std::string()) == "response.completed" && + event.contains("data") && event.at("data").contains("response") && + event.at("data").at("response").contains("output") && + event.at("data").at("response").at("output").is_array()) { + for (const auto & item : event.at("data").at("response").at("output")) { + if (item.value("type", std::string()) == "message") { + saw_final_phase_completed = item.value("phase", std::string()) == "final_answer"; + break; + } + } + } + } + if (!saw_final_phase_done || !saw_final_phase_completed) { + std::cerr << "Streaming Responses egress lost final_answer phase\n"; + return 1; + } + + server_task_result_cmpl_final commentary_phase_result; + commentary_phase_result.oaicompat_model = "test-model"; + commentary_phase_result.oaicompat_msg = parsed_commentary; + commentary_phase_result.oaicompat_msg.role = "assistant"; + const auto commentary_phase_response = commentary_phase_result.to_json_oaicompat_resp(); + bool saw_commentary_message = false; + bool saw_commentary_tool_call = false; + if (commentary_phase_response.contains("output") && commentary_phase_response.at("output").is_array()) { + for (const auto & item : commentary_phase_response.at("output")) { + const auto type = item.value("type", std::string()); + if (type == "message") { + saw_commentary_message = item.value("phase", std::string()) == "commentary"; + } else if (type == "function_call") { + saw_commentary_tool_call = true; + } + } + } + if (!saw_commentary_message || !saw_commentary_tool_call) { + std::cerr << "Responses egress lost the model-selected commentary phase\n"; + return 1; + } + + // Backward compatibility: an unmarked parser result still gets the old + // structural fallback, so non-participating templates/models keep working. + common_chat_msg fallback_msg = message("assistant", "Legacy preamble."); + fallback_msg.tool_calls.push_back({ + "shell_command", + nlohmann::ordered_json({{"command", "pwd"}}).dump(), + "fallback_tool", + }); + server_task_result_cmpl_final fallback_result; + fallback_result.oaicompat_model = "test-model"; + fallback_result.oaicompat_msg = fallback_msg; + const auto fallback_response = fallback_result.to_json_oaicompat_resp(); + if (fallback_response.at("output")[0].value("phase", std::string()) != "commentary") { + std::cerr << "Unmarked Responses message lost the tool-call phase fallback\n"; + return 1; + } + + std::cout << "Qwen3.8 Codex user text, generic string framing, custom framing, compaction, tool search, and Responses phase tests passed\n"; return 0; } diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index 4bf1d390e6b3..2edd09800e02 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -319,6 +319,9 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) { if (merge_prev) { auto & prev_msg = chatcmpl_messages.back(); + if (exists_and_is_string(item, "phase")) { + prev_msg["phase"] = item.at("phase"); + } if (!exists_and_is_array(prev_msg, "content")) { prev_msg["content"] = json::array(); } @@ -549,17 +552,26 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) { throw std::invalid_argument("'input' must be a string or array of objects"); } - if (is_compaction_request) { + { json chat_template_kwargs = json_value(chatcmpl_body, "chat_template_kwargs", json::object()); if (!chat_template_kwargs.is_object()) { throw std::invalid_argument("'chat_template_kwargs' must be an object"); } - // Keep model-specific compaction prompting in the chat template. This - // flag does not alter durable history; a template can append its own - // compaction tail after rendering the normal message/tool prefix. - chat_template_kwargs["is_compaction"] = true; + // Responses phase metadata is model-assisted by templates that know how + // to signal it; templates that do not recognize the flag simply ignore it. + chat_template_kwargs["responses_phase_protocol"] = true; + + if (is_compaction_request) { + // Keep model-specific compaction prompting in the chat template. This + // flag does not alter durable history; a template can append its own + // compaction tail after rendering the normal message/tool prefix. + chat_template_kwargs["is_compaction"] = true; + } chatcmpl_body["chat_template_kwargs"] = std::move(chat_template_kwargs); + } + + if (is_compaction_request) { chatcmpl_body["__llamacpp_responses_compaction"] = true; } diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 2609c8c22425..2422a677944b 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -145,6 +145,13 @@ static json server_task_build_response_tool_call( : server_task_build_response_function_call(tool_call, status); } +static std::string server_task_response_message_phase(const common_chat_msg & msg) { + if (msg.phase == "commentary" || msg.phase == "final_answer") { + return msg.phase; + } + return msg.tool_calls.empty() ? "final_answer" : "commentary"; +} + // // task_params // @@ -698,6 +705,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp() { }})}, {"id", "msg_" + random_string()}, {"role", msg.role}, + {"phase", server_task_response_message_phase(msg)}, {"status", "completed"}, {"type", "message"}, }); @@ -789,7 +797,8 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() { {"status", "completed"}, {"id", oai_resp_message_id}, {"content", json::array({content_part})}, - {"role", "assistant"} + {"role", "assistant"}, + {"phase", server_task_response_message_phase(oaicompat_msg)} }; server_sent_events.push_back(json { @@ -1177,6 +1186,7 @@ void server_task_result_cmpl_partial::update(task_result_state & state) { oai_resp_id = state.oai_resp_id; oai_resp_reasoning_id = state.oai_resp_reasoning_id; oai_resp_message_id = state.oai_resp_message_id; + oai_resp_message_phase = state.chat_msg.phase; oai_resp_fc_id = state.oai_resp_fc_id; oai_resp_fc_is_custom = state.oai_resp_fc_is_custom; oai_resp_fc_is_tool_search = state.oai_resp_fc_is_tool_search; @@ -1423,17 +1433,21 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() { if (!diff.content_delta.empty()) { if (!text_block_started) { + json message_item = { + {"content", json::array()}, + {"id", oai_resp_message_id}, + {"role", "assistant"}, + {"status", "in_progress"}, + {"type", "message"}, + }; + if (!oai_resp_message_phase.empty()) { + message_item["phase"] = oai_resp_message_phase; + } events.push_back(json { {"event", "response.output_item.added"}, {"data", json { {"type", "response.output_item.added"}, - {"item", json { - {"content", json::array()}, - {"id", oai_resp_message_id}, - {"role", "assistant"}, - {"status", "in_progress"}, - {"type", "message"}, - }}, + {"item", std::move(message_item)}, }}, }); events.push_back(json { diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 000b331fa6bd..e4dc72d79124 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -460,6 +460,7 @@ struct server_task_result_cmpl_partial : server_task_result { std::string oai_resp_id; std::string oai_resp_reasoning_id; std::string oai_resp_message_id; + std::string oai_resp_message_phase; std::string oai_resp_fc_id; bool oai_resp_fc_is_custom = false; bool oai_resp_fc_is_tool_search = false;