Skip to content

server: support Responses assistant message phases #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
inf9144 merged 1 commit into from
Aug 18, 2026
Merged

server: support Responses assistant message phases #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions common/chat-peg-parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {
// Handle reasoning/content tags
bool is_reasoning = node.tag == common_chat_peg_builder::REASONING;
bool is_content = node.tag == common_chat_peg_builder::CONTENT;
bool is_phase = node.tag == common_chat_peg_builder::PHASE;

if (is_reasoning) { // GPT OSS can have more than 1 reasoning block, so concatenate here
result.reasoning_content += std::string(node.text);
Expand All @@ -329,6 +330,10 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {
result.content += std::string(node.text);
}

if (is_phase) {
result.phase = std::string(node.text);
}

// Handle tool-related tags (supporting both JSON and tagged formats)
bool is_tool_open = node.tag == common_chat_peg_builder::TOOL_OPEN;
bool is_tool_close = node.tag == common_chat_peg_builder::TOOL_CLOSE;
Expand Down
3 changes: 3 additions & 0 deletions common/chat-peg-parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ class common_chat_peg_builder : public common_peg_parser_builder {
static constexpr const char * REASONING_BLOCK = "reasoning-block";
static constexpr const char * REASONING = "reasoning";
static constexpr const char * CONTENT = "content";
static constexpr const char * PHASE = "phase";

// Tag constants
static constexpr const char * TOOL = "tool";
Expand All @@ -85,6 +86,8 @@ class common_chat_peg_builder : public common_peg_parser_builder {

common_peg_parser content(const common_peg_parser & p) { return tag(CONTENT, p); }

common_peg_parser phase(const common_peg_parser & p) { return atomic(tag(PHASE, p)); }

common_peg_parser tag_with_safe_content(const std::string & tag_name,
const std::string & marker,
const common_peg_parser & p);
Expand Down
42 changes: 36 additions & 6 deletions common/chat.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,12 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
if (message.contains("reasoning_content")) {
msg.reasoning_content = message.at("reasoning_content");
}
if (message.contains("phase") && !message.at("phase").is_null()) {
if (!message.at("phase").is_string()) {
throw std::invalid_argument("Invalid 'phase' type: expected string");
}
msg.phase = message.at("phase");
}
if (message.contains("name")) {
msg.tool_name = message.at("name");
}
Expand Down Expand Up @@ -534,14 +540,21 @@ struct messages_inp_normalizer {
}
};

static json render_message_to_json(const std::vector<common_chat_msg> & msgs, const jinja::caps & c) {
static json render_message_to_json(
const std::vector<common_chat_msg> & msgs,
const jinja::caps & c,
bool include_phase = false) {
if (!c.supports_string_content && !c.supports_typed_content) {
LOG_WRN("%s: Neither string content nor typed content is supported by the template. This is unexpected and may lead to issues.\n", __func__);
}

json messages = json::array();
for (const auto & msg : msgs) {
messages.push_back(msg.to_json_oaicompat(/* concat_typed_text= */ false));
json rendered = msg.to_json_oaicompat(/* concat_typed_text= */ false);
if (include_phase && !msg.phase.empty()) {
rendered["phase"] = msg.phase;
}
messages.push_back(std::move(rendered));
}
return messages_inp_normalizer(c).normalize(messages);
}
Expand Down Expand Up @@ -1198,6 +1211,10 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
const bool json_string_args = tmpl.source().find("llama.cpp:xml-string-args=json") != std::string::npos;
const bool responses_phase_protocol =
tmpl.source().find("llama.cpp:responses-phase=marker-v1") != std::string::npos &&
inputs.extra_context.is_object() &&
inputs.extra_context.value("responses_phase_protocol", false);

if (inputs.has_continuation()) {
const auto & msg = inputs.continue_msg;
Expand Down Expand Up @@ -1236,9 +1253,18 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_
(p.literal("</think>") | p.peek(p.literal("<tool_call>"))));
}

auto phase = p.eps();
if (responses_phase_protocol) {
auto phase_block = p.atomic(
p.literal("<response_phase>") +
p.phase(p.literal("commentary") | p.literal("final_answer")) +
p.literal("</response_phase>"));
phase = p.optional(phase_block + p.space());
}

// Response format parser
if (has_response_format) {
return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
return generation_prompt + (reasoning << phase << p.content(p.schema(p.json(), "response-format", inputs.json_schema)));
}

// Tool call parser
Expand Down Expand Up @@ -1302,11 +1328,11 @@ static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_
auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1));

return generation_prompt +
(reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls);
(reasoning << phase << p.content(p.until_one_of(tool_call_starts)) << tool_calls);
}

// Content only parser
return generation_prompt + (reasoning << p.content(p.rest()));
return generation_prompt + (reasoning << phase << p.content(p.rest()));
});

data.parser = parser.save();
Expand Down Expand Up @@ -3619,7 +3645,11 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
workaround::trim_all_content(trimmed_messages);
messages_to_render = &trimmed_messages;
}
params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps());
const auto phase_protocol_it = inputs.chat_template_kwargs.find("responses_phase_protocol");
const bool responses_phase_protocol =
phase_protocol_it != inputs.chat_template_kwargs.end() && phase_protocol_it->second == "true";
params.messages = render_message_to_json(
*messages_to_render, tmpl.original_caps(), responses_phase_protocol);
params.tool_choice = inputs.tool_choice;
params.reasoning_format = inputs.reasoning_format;
params.enable_thinking = inputs.enable_thinking;
Expand Down
7 changes: 5 additions & 2 deletions common/chat.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,14 +86,17 @@ struct common_chat_msg {
std::string reasoning_content;
std::string tool_name;
std::string tool_call_id;
// Optional assistant lifecycle metadata used by Responses-capable parsers.
// Keep this last so existing aggregate initializers retain their field mapping.
std::string phase;

nlohmann::ordered_json to_json_oaicompat(bool concat_typed_text = false) const;

std::string render_content(const std::string & delimiter = "\n\n") const;

bool empty() const {
return content.empty() && content_parts.empty() && tool_calls.empty() && reasoning_content.empty() &&
tool_name.empty() && tool_call_id.empty();
phase.empty() && tool_name.empty() && tool_call_id.empty();
}

bool contains_media() const {
Expand Down Expand Up @@ -122,7 +125,7 @@ struct common_chat_msg {
bool operator==(const common_chat_msg & other) const {
return role == other.role && content == other.content && content_parts == other.content_parts &&
tool_calls == other.tool_calls && reasoning_content == other.reasoning_content &&
tool_name == other.tool_name && tool_call_id == other.tool_call_id;
phase == other.phase && tool_name == other.tool_name && tool_call_id == other.tool_call_id;
}

bool operator!=(const common_chat_msg & other) const { return !(*this == other); }
Expand Down
37 changes: 35 additions & 2 deletions models/templates/llama-cpp-qwen3.8-codex.jinja
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
{%- set template_version = "qwen3.8-codex-safe-v1" %}
{#- llama.cpp:xml-string-args=json -#}
{#- llama.cpp:responses-phase=marker-v1 -#}
{#- -------------------------------------------------------------------------
Whitespace contract: every tag opener in this file carries a leading minus,
so it strips the whitespace and newlines that precede it. All emitted text
Expand All @@ -26,6 +27,9 @@
{%- endif %}
{%- set add_vision_id = add_vision_id if add_vision_id is defined else false %}
{%- set enable_thinking = enable_thinking if enable_thinking is defined else true %}
{%- set responses_phase_protocol =
responses_phase_protocol if responses_phase_protocol is defined else false
%}
{%- set auto_disable_thinking_with_tools =
auto_disable_thinking_with_tools
if auto_disable_thinking_with_tools is defined
Expand Down Expand Up @@ -159,6 +163,20 @@
%}
{%- endif %}
{%- endif %}
{%- if responses_phase_protocol %}
{%- set phase_instructions =
'Responses message phases are transport metadata and are separate from reasoning. Emit exactly one internal phase block immediately after </think> and before any visible assistant text or tool call: <response_phase>commentary</response_phase> when the visible assistant message is an intermediate update before tool use or further work, or <response_phase>final_answer</response_phase> when the visible assistant message is the terminal answer of the turn. If the template has already supplied an empty <think></think> block, begin your generation with the phase block. Never repeat or mention the phase block in visible text.'
%}
{%- if reasoning_instructions %}
{%- set reasoning_instructions =
reasoning_instructions ~ '

' ~ phase_instructions
%}
{%- else %}
{%- set reasoning_instructions = phase_instructions %}
{%- endif %}
{%- endif %}
{#- -------------------------------------------------------------------------
Content rendering
------------------------------------------------------------------------- -#}
Expand Down Expand Up @@ -378,17 +396,32 @@
{%- endif %}
{%- endif %}
{%- set reasoning_content = reasoning_content | trim %}
{%- set phase_prefix = '' %}
{%- set after_think = '\n\n' %}
{%- if responses_phase_protocol
and message.phase is defined
and message.phase is not none
and message.phase %}
{%- if message.phase not in ('commentary', 'final_answer') %}
{{- raise_exception('Unexpected Responses message phase ' ~ message.phase ~ '.') }}
{%- endif %}
{%- set phase_prefix =
'<response_phase>' ~ message.phase ~ '</response_phase>\n'
%}
{%- set after_think = '\n' ~ phase_prefix %}
{%- endif %}
{#- Preserve the original invariant: when thinking is preserved,
always reconstruct the thinking block, even when it is empty. -#}
{%- if _preserve_thinking
or loop.index0 > ns.last_query_index %}
{{- '<|im_start|>assistant\n<think>\n'
~ reasoning_content
~ '\n</think>\n\n'
~ '\n</think>'
~ after_think
~ content
}}
{%- else %}
{{- '<|im_start|>assistant\n' ~ content }}
{{- '<|im_start|>assistant\n' ~ phase_prefix ~ content }}
{%- endif %}
{%- if message.tool_calls is defined
and message.tool_calls is not none
Expand Down
Loading
Loading