From 4ffcc1037f565e8e9615c1eaa905b69a54df3c87 Mon Sep 17 00:00:00 2001 From: Xiaoze Fan Date: Fri, 14 Aug 2026 04:06:31 +0000 Subject: [PATCH 1/2] feat(server): enable reasoning_effort on /v1/chat/completions --- python/freetoken/server/api_models.py | 1 + python/freetoken/server/openai_api.py | 28 ++++++++++++++--- tests/server/test_openai_api.py | 44 +++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/python/freetoken/server/api_models.py b/python/freetoken/server/api_models.py index a2def1aea6..b147db4ecb 100644 --- a/python/freetoken/server/api_models.py +++ b/python/freetoken/server/api_models.py @@ -77,6 +77,7 @@ class ChatCompletionRequest(BaseModel): presence_penalty: float = 0.0 frequency_penalty: float = 0.0 chat_template_kwargs: dict[str, Any] = Field(default_factory=dict) + reasoning_effort: str | None = None ignore_eos: bool = False tools: list[Tool] | None = None tool_choice: Literal["none", "auto", "required"] | ToolChoiceObject | None = None diff --git a/python/freetoken/server/openai_api.py b/python/freetoken/server/openai_api.py index 5f3f81e080..5b2f121eb9 100644 --- a/python/freetoken/server/openai_api.py +++ b/python/freetoken/server/openai_api.py @@ -37,8 +37,24 @@ ) -def chat_request_to_genspec(req: ChatCompletionRequest, model_sampling: dict[str, Any]) -> GenSpec: +def chat_request_to_genspec( + req: ChatCompletionRequest, + model_sampling: dict[str, Any], + reasoning_parser: str | None = None, +) -> GenSpec: """OpenAI ChatCompletionRequest -> GenSpec (the OpenAI 'to_sampling_params').""" + from .model_meta import think_toggle_kwargs + + # Thinking toggle: an explicit chat_template_kwargs extra field wins; else + # top-level reasoning_effort drives the template through the per-family + # mapping in model_meta. Effort "none" disables thinking (vLLM semantics); + # other efforts enable it and are forwarded for templates that grade them + # (gpt-oss). + ctk = req.chat_template_kwargs + if req.reasoning_effort and not ctk: + ctk = dict(think_toggle_kwargs(reasoning_parser, req.reasoning_effort != "none")) + if req.reasoning_effort != "none": + ctk.setdefault("reasoning_effort", req.reasoning_effort) return GenSpec( messages=render_messages([m.model_dump(exclude_none=True) for m in req.messages]), sampling_params=resolve_sampling( @@ -50,7 +66,7 @@ def chat_request_to_genspec(req: ChatCompletionRequest, model_sampling: dict[str model_sampling=model_sampling, stop=req.stop, ), - chat_template_kwargs=req.chat_template_kwargs, + chat_template_kwargs=ctk, template_tools=_tools_for_template(req), parser_tools=(_all_tool_dicts(req.tools) if _should_parse_tools(req) else None), ) @@ -133,7 +149,9 @@ async def handle_chat_completion( return create_error_response("Only n=1 is supported", param="n") try: - spec = chat_request_to_genspec(req, model_sampling) + spec = chat_request_to_genspec( + req, model_sampling, reasoning_parser=getattr(state.config, "reasoning_parser", None) + ) except ValueError as exc: return create_error_response(str(exc)) uid = await submit_generation(spec, state) @@ -182,7 +200,9 @@ async def stream_chat_completion_chunks( ) -> AsyncIterator[bytes]: """Format generate_events() into the OpenAI chat.completion.chunk SSE stream.""" if spec is None: - spec = chat_request_to_genspec(req, {}) + spec = chat_request_to_genspec( + req, {}, reasoning_parser=getattr(state.config, "reasoning_parser", None) + ) yield _sse( _chat_chunk( req, diff --git a/tests/server/test_openai_api.py b/tests/server/test_openai_api.py index 69e6173f10..cc8fafbd3b 100644 --- a/tests/server/test_openai_api.py +++ b/tests/server/test_openai_api.py @@ -161,6 +161,50 @@ def test_chat_request_reasoning_replay_field_aliases(): assert asst["thinking"] == "prior thought", field +def _chat_req(**extra) -> ChatCompletionRequest: + return ChatCompletionRequest.model_validate( + {"model": "m", "messages": [{"role": "user", "content": "hi"}], **extra} + ) + + +def test_chat_reasoning_effort_enables_thinking(): + spec = chat_request_to_genspec(_chat_req(reasoning_effort="high"), {}) + assert spec.chat_template_kwargs == {"enable_thinking": True, "reasoning_effort": "high"} + + # explicit chat_template_kwargs extra field wins over the effort mapping + spec = chat_request_to_genspec( + _chat_req(reasoning_effort="none", chat_template_kwargs={"enable_thinking": True}), {} + ) + assert spec.chat_template_kwargs == {"enable_thinking": True} + + # absent effort -> kwargs pass through untouched + assert chat_request_to_genspec(_chat_req(), {}).chat_template_kwargs == {} + + +def test_chat_reasoning_effort_none_disables_thinking(): + # vLLM-compatible semantics: an explicit effort "none" DISABLES thinking. + spec = chat_request_to_genspec(_chat_req(reasoning_effort="none"), {}) + assert spec.chat_template_kwargs == {"enable_thinking": False} + + +def test_chat_reasoning_effort_routes_through_family_mapping(): + """The toggle goes through model_meta's per-family mapping -- for M3 that is + thinking_mode, not the (inert) enable_thinking key.""" + on = _chat_req(reasoning_effort="high") + spec = chat_request_to_genspec(on, {}, reasoning_parser="minimax_m3") + assert spec.chat_template_kwargs == {"thinking_mode": "enabled", "reasoning_effort": "high"} + + off = _chat_req(reasoning_effort="none") + spec = chat_request_to_genspec(off, {}, reasoning_parser="minimax_m3") + assert spec.chat_template_kwargs == {"thinking_mode": "disabled"} + + # gpt-oss: the template grades effort and has no off gear + spec = chat_request_to_genspec(on, {}, reasoning_parser="gpt_oss") + assert spec.chat_template_kwargs == {"reasoning_effort": "high"} + spec = chat_request_to_genspec(off, {}, reasoning_parser="gpt_oss") + assert spec.chat_template_kwargs == {} + + def test_non_stream_chat_completion_returns_openai_tool_calls_and_sends_tools(): output = '[TOOL_CALLS] [{"name":"get_weather","arguments":{"city":"Paris"}}]' state = FakeState( From 852136b1a1eb479dcee7c520d3ee02632946958f Mon Sep 17 00:00:00 2001 From: Xiaoze Fan Date: Fri, 14 Aug 2026 06:21:06 +0000 Subject: [PATCH 2/2] fix(server): glm thinking-off mislabeling and reasoning_effort mapping edge cases --- python/freetoken/server/generation.py | 4 +++ python/freetoken/server/model_meta.py | 26 +++++++++++++++ python/freetoken/server/openai_api.py | 13 ++------ python/freetoken/server/responses_api.py | 14 ++------ tests/server/test_openai_api.py | 41 ++++++++++++++++-------- tests/server/test_responses_api.py | 15 +++++++-- 6 files changed, 76 insertions(+), 37 deletions(-) diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.py index bd8741b8ee..5d2db7194f 100644 --- a/python/freetoken/server/generation.py +++ b/python/freetoken/server/generation.py @@ -317,6 +317,10 @@ def _make_reasoning_parser(spec: GenSpec, state: Any) -> ReasoningParser | None: # enable_thinking is explicitly false, so the model emits only the closing # . Mirror that default here, else the chain-of-thought leaks into content. force_reasoning = (spec.chat_template_kwargs or {}).get("enable_thinking") is not False + elif parser_name == "glm": + # GLM's template honors enable_thinking (default on) even with tools; the + # generic fallback would force thinking and mislabel disabled output as reasoning. + force_reasoning = (spec.chat_template_kwargs or {}).get("enable_thinking") is not False elif parser_name == "gemma4": # Gemma4 defaults thinking off even when tools are present: its template injects an # empty thought channel before generation. Do not let Codex tool definitions make all diff --git a/python/freetoken/server/model_meta.py b/python/freetoken/server/model_meta.py index d976c9247f..c8b0f8f0c1 100644 --- a/python/freetoken/server/model_meta.py +++ b/python/freetoken/server/model_meta.py @@ -67,6 +67,32 @@ def think_toggle_kwargs(reasoning_parser: str | None, enabled: bool) -> dict: return think_chat_template_kwargs(reasoning_parser, gear) +_THINKING_KWARG_KEYS = ("enable_thinking", "thinking", "thinking_mode", "reasoning_effort") +_DISABLE_EFFORTS = ("none", "off") + + +def effort_toggle_kwargs( + reasoning_parser: str | None, + effort: str | None, + chat_template_kwargs: dict | None, +) -> dict: + """Fold a protocol-level reasoning-effort request into the template kwargs. + An explicit thinking-related key wins wholesale; unrelated extras ride along. + Effort "none"/"off" (case-insensitive) disables thinking; any other or absent + effort enables it, forwarded for templates that grade it (gpt-oss).""" + ctk = dict(chat_template_kwargs or {}) + if any(key in ctk for key in _THINKING_KWARG_KEYS): + return ctk + if isinstance(effort, str): + effort = effort.strip().lower() + disabled = effort in _DISABLE_EFFORTS + mapped = dict(think_toggle_kwargs(reasoning_parser, not disabled)) + if effort and not disabled: + mapped.setdefault("reasoning_effort", effort) + mapped.update(ctk) + return mapped + + def moe_total_experts(config: Any) -> int: """Total routed-expert slots the model has: experts per layer x MoE layers. Matches the engine's own basis (``Engine._resolve_auto_moe_cache_size``), so a residency rate derived diff --git a/python/freetoken/server/openai_api.py b/python/freetoken/server/openai_api.py index 5b2f121eb9..4b79350211 100644 --- a/python/freetoken/server/openai_api.py +++ b/python/freetoken/server/openai_api.py @@ -43,18 +43,11 @@ def chat_request_to_genspec( reasoning_parser: str | None = None, ) -> GenSpec: """OpenAI ChatCompletionRequest -> GenSpec (the OpenAI 'to_sampling_params').""" - from .model_meta import think_toggle_kwargs + from .model_meta import effort_toggle_kwargs - # Thinking toggle: an explicit chat_template_kwargs extra field wins; else - # top-level reasoning_effort drives the template through the per-family - # mapping in model_meta. Effort "none" disables thinking (vLLM semantics); - # other efforts enable it and are forwarded for templates that grade them - # (gpt-oss). ctk = req.chat_template_kwargs - if req.reasoning_effort and not ctk: - ctk = dict(think_toggle_kwargs(reasoning_parser, req.reasoning_effort != "none")) - if req.reasoning_effort != "none": - ctk.setdefault("reasoning_effort", req.reasoning_effort) + if req.reasoning_effort: + ctk = effort_toggle_kwargs(reasoning_parser, req.reasoning_effort, ctk) return GenSpec( messages=render_messages([m.model_dump(exclude_none=True) for m in req.messages]), sampling_params=resolve_sampling( diff --git a/python/freetoken/server/responses_api.py b/python/freetoken/server/responses_api.py index 31a316f806..9ed1f1ac4c 100644 --- a/python/freetoken/server/responses_api.py +++ b/python/freetoken/server/responses_api.py @@ -230,19 +230,11 @@ def convert_responses_to_genspec( else: template_tools, parser_tools = split_tool_lists(raw_tools, selected) - # Thinking toggle: an explicit chat_template_kwargs extra field wins; else - # the protocol-native `reasoning` object drives the template through the - # per-family mapping in model_meta. effort "none" disables thinking (vLLM - # semantics); other efforts enable it and are forwarded for templates that - # grade them (gpt-oss). - from .model_meta import think_toggle_kwargs + from .model_meta import effort_toggle_kwargs ctk = dict(getattr(req, "chat_template_kwargs", None) or {}) - if req.reasoning and not ctk: - effort = req.reasoning.get("effort") - ctk = dict(think_toggle_kwargs(reasoning_parser, effort != "none")) - if effort and effort != "none": - ctk.setdefault("reasoning_effort", effort) + if req.reasoning: + ctk = effort_toggle_kwargs(reasoning_parser, req.reasoning.get("effort"), ctk) return GenSpec( messages=render_messages(messages), diff --git a/tests/server/test_openai_api.py b/tests/server/test_openai_api.py index cc8fafbd3b..709203c56e 100644 --- a/tests/server/test_openai_api.py +++ b/tests/server/test_openai_api.py @@ -161,40 +161,40 @@ def test_chat_request_reasoning_replay_field_aliases(): assert asst["thinking"] == "prior thought", field -def _chat_req(**extra) -> ChatCompletionRequest: - return ChatCompletionRequest.model_validate( - {"model": "m", "messages": [{"role": "user", "content": "hi"}], **extra} - ) - - def test_chat_reasoning_effort_enables_thinking(): - spec = chat_request_to_genspec(_chat_req(reasoning_effort="high"), {}) + spec = chat_request_to_genspec(chat_request(reasoning_effort="high"), {}) assert spec.chat_template_kwargs == {"enable_thinking": True, "reasoning_effort": "high"} - # explicit chat_template_kwargs extra field wins over the effort mapping + # an explicit thinking-related chat_template_kwargs key wins over the mapping spec = chat_request_to_genspec( - _chat_req(reasoning_effort="none", chat_template_kwargs={"enable_thinking": True}), {} + chat_request(reasoning_effort="none", chat_template_kwargs={"enable_thinking": True}), {} ) assert spec.chat_template_kwargs == {"enable_thinking": True} + # unrelated extra kwargs ride along without discarding the effort mapping + spec = chat_request_to_genspec( + chat_request(reasoning_effort="none", chat_template_kwargs={"custom_var": 1}), {} + ) + assert spec.chat_template_kwargs == {"enable_thinking": False, "custom_var": 1} + # absent effort -> kwargs pass through untouched - assert chat_request_to_genspec(_chat_req(), {}).chat_template_kwargs == {} + assert chat_request_to_genspec(chat_request(), {}).chat_template_kwargs == {} def test_chat_reasoning_effort_none_disables_thinking(): # vLLM-compatible semantics: an explicit effort "none" DISABLES thinking. - spec = chat_request_to_genspec(_chat_req(reasoning_effort="none"), {}) + spec = chat_request_to_genspec(chat_request(reasoning_effort="none"), {}) assert spec.chat_template_kwargs == {"enable_thinking": False} def test_chat_reasoning_effort_routes_through_family_mapping(): """The toggle goes through model_meta's per-family mapping -- for M3 that is thinking_mode, not the (inert) enable_thinking key.""" - on = _chat_req(reasoning_effort="high") + on = chat_request(reasoning_effort="high") spec = chat_request_to_genspec(on, {}, reasoning_parser="minimax_m3") assert spec.chat_template_kwargs == {"thinking_mode": "enabled", "reasoning_effort": "high"} - off = _chat_req(reasoning_effort="none") + off = chat_request(reasoning_effort="none") spec = chat_request_to_genspec(off, {}, reasoning_parser="minimax_m3") assert spec.chat_template_kwargs == {"thinking_mode": "disabled"} @@ -205,6 +205,21 @@ def test_chat_reasoning_effort_routes_through_family_mapping(): assert spec.chat_template_kwargs == {} +def test_glm_reasoning_parser_honors_disabled_thinking_with_tools(): + # The parse side must match the encode side: thinking off + tools present + # must not start the parser inside a think block. + from freetoken.server.generation import _make_reasoning_parser + + state = FakeState([], reasoning_parser="glm") + off = chat_request_to_genspec(chat_request(reasoning_effort="none"), {}, reasoning_parser="glm") + parser = _make_reasoning_parser(off, state) + assert parser is not None and parser.detector.force_reasoning is False + + on = chat_request_to_genspec(chat_request(), {}, reasoning_parser="glm") + parser = _make_reasoning_parser(on, state) + assert parser is not None and parser.detector.force_reasoning is True + + def test_non_stream_chat_completion_returns_openai_tool_calls_and_sends_tools(): output = '[TOOL_CALLS] [{"name":"get_weather","arguments":{"city":"Paris"}}]' state = FakeState( diff --git a/tests/server/test_responses_api.py b/tests/server/test_responses_api.py index ef9e602daa..9950c665d3 100644 --- a/tests/server/test_responses_api.py +++ b/tests/server/test_responses_api.py @@ -867,7 +867,7 @@ def test_convert_reasoning_field_enables_thinking(): spec = RP.convert_responses_to_genspec(req, {}) assert spec.chat_template_kwargs == {"enable_thinking": True, "reasoning_effort": "high"} - # explicit chat_template_kwargs extra field wins over the reasoning mapping + # an explicit thinking-related chat_template_kwargs key wins over the mapping req2 = ResponsesRequest.model_validate( {"model": "m", "input": "hi", "reasoning": {"effort": "low"}, "chat_template_kwargs": {"thinking_mode": "chat"}} @@ -875,9 +875,18 @@ def test_convert_reasoning_field_enables_thinking(): spec2 = RP.convert_responses_to_genspec(req2, {}) assert spec2.chat_template_kwargs == {"thinking_mode": "chat"} + # unrelated extra kwargs ride along without discarding the reasoning mapping + req3 = ResponsesRequest.model_validate( + {"model": "m", "input": "hi", "reasoning": {"effort": "none"}, + "chat_template_kwargs": {"custom_var": 1}} + ) + assert RP.convert_responses_to_genspec(req3, {}).chat_template_kwargs == { + "enable_thinking": False, "custom_var": 1, + } + # absent reasoning -> no kwargs - req3 = ResponsesRequest.model_validate({"model": "m", "input": "hi"}) - assert RP.convert_responses_to_genspec(req3, {}).chat_template_kwargs == {} + req4 = ResponsesRequest.model_validate({"model": "m", "input": "hi"}) + assert RP.convert_responses_to_genspec(req4, {}).chat_template_kwargs == {} def test_convert_reasoning_effort_none_disables_thinking():