diff --git a/python/freetoken/server/api_models.py b/python/freetoken/server/api_models.py index a2def1aea6..b147db4ecb 100644 --- a/python/freetoken/server/api_models.py +++ b/python/freetoken/server/api_models.py @@ -77,6 +77,7 @@ class ChatCompletionRequest(BaseModel): presence_penalty: float = 0.0 frequency_penalty: float = 0.0 chat_template_kwargs: dict[str, Any] = Field(default_factory=dict) + reasoning_effort: str | None = None ignore_eos: bool = False tools: list[Tool] | None = None tool_choice: Literal["none", "auto", "required"] | ToolChoiceObject | None = None diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.py index bd8741b8ee..5d2db7194f 100644 --- a/python/freetoken/server/generation.py +++ b/python/freetoken/server/generation.py @@ -317,6 +317,10 @@ def _make_reasoning_parser(spec: GenSpec, state: Any) -> ReasoningParser | None: # enable_thinking is explicitly false, so the model emits only the closing # . Mirror that default here, else the chain-of-thought leaks into content. force_reasoning = (spec.chat_template_kwargs or {}).get("enable_thinking") is not False + elif parser_name == "glm": + # GLM's template honors enable_thinking (default on) even with tools; the + # generic fallback would force thinking and mislabel disabled output as reasoning. + force_reasoning = (spec.chat_template_kwargs or {}).get("enable_thinking") is not False elif parser_name == "gemma4": # Gemma4 defaults thinking off even when tools are present: its template injects an # empty thought channel before generation. Do not let Codex tool definitions make all diff --git a/python/freetoken/server/model_meta.py b/python/freetoken/server/model_meta.py index d976c9247f..c8b0f8f0c1 100644 --- a/python/freetoken/server/model_meta.py +++ b/python/freetoken/server/model_meta.py @@ -67,6 +67,32 @@ def think_toggle_kwargs(reasoning_parser: str | None, enabled: bool) -> dict: return think_chat_template_kwargs(reasoning_parser, gear) +_THINKING_KWARG_KEYS = ("enable_thinking", "thinking", "thinking_mode", "reasoning_effort") +_DISABLE_EFFORTS = ("none", "off") + + +def effort_toggle_kwargs( + reasoning_parser: str | None, + effort: str | None, + chat_template_kwargs: dict | None, +) -> dict: + """Fold a protocol-level reasoning-effort request into the template kwargs. + An explicit thinking-related key wins wholesale; unrelated extras ride along. + Effort "none"/"off" (case-insensitive) disables thinking; any other or absent + effort enables it, forwarded for templates that grade it (gpt-oss).""" + ctk = dict(chat_template_kwargs or {}) + if any(key in ctk for key in _THINKING_KWARG_KEYS): + return ctk + if isinstance(effort, str): + effort = effort.strip().lower() + disabled = effort in _DISABLE_EFFORTS + mapped = dict(think_toggle_kwargs(reasoning_parser, not disabled)) + if effort and not disabled: + mapped.setdefault("reasoning_effort", effort) + mapped.update(ctk) + return mapped + + def moe_total_experts(config: Any) -> int: """Total routed-expert slots the model has: experts per layer x MoE layers. Matches the engine's own basis (``Engine._resolve_auto_moe_cache_size``), so a residency rate derived diff --git a/python/freetoken/server/openai_api.py b/python/freetoken/server/openai_api.py index 5f3f81e080..4b79350211 100644 --- a/python/freetoken/server/openai_api.py +++ b/python/freetoken/server/openai_api.py @@ -37,8 +37,17 @@ ) -def chat_request_to_genspec(req: ChatCompletionRequest, model_sampling: dict[str, Any]) -> GenSpec: +def chat_request_to_genspec( + req: ChatCompletionRequest, + model_sampling: dict[str, Any], + reasoning_parser: str | None = None, +) -> GenSpec: """OpenAI ChatCompletionRequest -> GenSpec (the OpenAI 'to_sampling_params').""" + from .model_meta import effort_toggle_kwargs + + ctk = req.chat_template_kwargs + if req.reasoning_effort: + ctk = effort_toggle_kwargs(reasoning_parser, req.reasoning_effort, ctk) return GenSpec( messages=render_messages([m.model_dump(exclude_none=True) for m in req.messages]), sampling_params=resolve_sampling( @@ -50,7 +59,7 @@ def chat_request_to_genspec(req: ChatCompletionRequest, model_sampling: dict[str model_sampling=model_sampling, stop=req.stop, ), - chat_template_kwargs=req.chat_template_kwargs, + chat_template_kwargs=ctk, template_tools=_tools_for_template(req), parser_tools=(_all_tool_dicts(req.tools) if _should_parse_tools(req) else None), ) @@ -133,7 +142,9 @@ async def handle_chat_completion( return create_error_response("Only n=1 is supported", param="n") try: - spec = chat_request_to_genspec(req, model_sampling) + spec = chat_request_to_genspec( + req, model_sampling, reasoning_parser=getattr(state.config, "reasoning_parser", None) + ) except ValueError as exc: return create_error_response(str(exc)) uid = await submit_generation(spec, state) @@ -182,7 +193,9 @@ async def stream_chat_completion_chunks( ) -> AsyncIterator[bytes]: """Format generate_events() into the OpenAI chat.completion.chunk SSE stream.""" if spec is None: - spec = chat_request_to_genspec(req, {}) + spec = chat_request_to_genspec( + req, {}, reasoning_parser=getattr(state.config, "reasoning_parser", None) + ) yield _sse( _chat_chunk( req, diff --git a/python/freetoken/server/responses_api.py b/python/freetoken/server/responses_api.py index 31a316f806..9ed1f1ac4c 100644 --- a/python/freetoken/server/responses_api.py +++ b/python/freetoken/server/responses_api.py @@ -230,19 +230,11 @@ def convert_responses_to_genspec( else: template_tools, parser_tools = split_tool_lists(raw_tools, selected) - # Thinking toggle: an explicit chat_template_kwargs extra field wins; else - # the protocol-native `reasoning` object drives the template through the - # per-family mapping in model_meta. effort "none" disables thinking (vLLM - # semantics); other efforts enable it and are forwarded for templates that - # grade them (gpt-oss). - from .model_meta import think_toggle_kwargs + from .model_meta import effort_toggle_kwargs ctk = dict(getattr(req, "chat_template_kwargs", None) or {}) - if req.reasoning and not ctk: - effort = req.reasoning.get("effort") - ctk = dict(think_toggle_kwargs(reasoning_parser, effort != "none")) - if effort and effort != "none": - ctk.setdefault("reasoning_effort", effort) + if req.reasoning: + ctk = effort_toggle_kwargs(reasoning_parser, req.reasoning.get("effort"), ctk) return GenSpec( messages=render_messages(messages), diff --git a/tests/server/test_openai_api.py b/tests/server/test_openai_api.py index 69e6173f10..709203c56e 100644 --- a/tests/server/test_openai_api.py +++ b/tests/server/test_openai_api.py @@ -161,6 +161,65 @@ def test_chat_request_reasoning_replay_field_aliases(): assert asst["thinking"] == "prior thought", field +def test_chat_reasoning_effort_enables_thinking(): + spec = chat_request_to_genspec(chat_request(reasoning_effort="high"), {}) + assert spec.chat_template_kwargs == {"enable_thinking": True, "reasoning_effort": "high"} + + # an explicit thinking-related chat_template_kwargs key wins over the mapping + spec = chat_request_to_genspec( + chat_request(reasoning_effort="none", chat_template_kwargs={"enable_thinking": True}), {} + ) + assert spec.chat_template_kwargs == {"enable_thinking": True} + + # unrelated extra kwargs ride along without discarding the effort mapping + spec = chat_request_to_genspec( + chat_request(reasoning_effort="none", chat_template_kwargs={"custom_var": 1}), {} + ) + assert spec.chat_template_kwargs == {"enable_thinking": False, "custom_var": 1} + + # absent effort -> kwargs pass through untouched + assert chat_request_to_genspec(chat_request(), {}).chat_template_kwargs == {} + + +def test_chat_reasoning_effort_none_disables_thinking(): + # vLLM-compatible semantics: an explicit effort "none" DISABLES thinking. + spec = chat_request_to_genspec(chat_request(reasoning_effort="none"), {}) + assert spec.chat_template_kwargs == {"enable_thinking": False} + + +def test_chat_reasoning_effort_routes_through_family_mapping(): + """The toggle goes through model_meta's per-family mapping -- for M3 that is + thinking_mode, not the (inert) enable_thinking key.""" + on = chat_request(reasoning_effort="high") + spec = chat_request_to_genspec(on, {}, reasoning_parser="minimax_m3") + assert spec.chat_template_kwargs == {"thinking_mode": "enabled", "reasoning_effort": "high"} + + off = chat_request(reasoning_effort="none") + spec = chat_request_to_genspec(off, {}, reasoning_parser="minimax_m3") + assert spec.chat_template_kwargs == {"thinking_mode": "disabled"} + + # gpt-oss: the template grades effort and has no off gear + spec = chat_request_to_genspec(on, {}, reasoning_parser="gpt_oss") + assert spec.chat_template_kwargs == {"reasoning_effort": "high"} + spec = chat_request_to_genspec(off, {}, reasoning_parser="gpt_oss") + assert spec.chat_template_kwargs == {} + + +def test_glm_reasoning_parser_honors_disabled_thinking_with_tools(): + # The parse side must match the encode side: thinking off + tools present + # must not start the parser inside a think block. + from freetoken.server.generation import _make_reasoning_parser + + state = FakeState([], reasoning_parser="glm") + off = chat_request_to_genspec(chat_request(reasoning_effort="none"), {}, reasoning_parser="glm") + parser = _make_reasoning_parser(off, state) + assert parser is not None and parser.detector.force_reasoning is False + + on = chat_request_to_genspec(chat_request(), {}, reasoning_parser="glm") + parser = _make_reasoning_parser(on, state) + assert parser is not None and parser.detector.force_reasoning is True + + def test_non_stream_chat_completion_returns_openai_tool_calls_and_sends_tools(): output = '[TOOL_CALLS] [{"name":"get_weather","arguments":{"city":"Paris"}}]' state = FakeState( diff --git a/tests/server/test_responses_api.py b/tests/server/test_responses_api.py index ef9e602daa..9950c665d3 100644 --- a/tests/server/test_responses_api.py +++ b/tests/server/test_responses_api.py @@ -867,7 +867,7 @@ def test_convert_reasoning_field_enables_thinking(): spec = RP.convert_responses_to_genspec(req, {}) assert spec.chat_template_kwargs == {"enable_thinking": True, "reasoning_effort": "high"} - # explicit chat_template_kwargs extra field wins over the reasoning mapping + # an explicit thinking-related chat_template_kwargs key wins over the mapping req2 = ResponsesRequest.model_validate( {"model": "m", "input": "hi", "reasoning": {"effort": "low"}, "chat_template_kwargs": {"thinking_mode": "chat"}} @@ -875,9 +875,18 @@ def test_convert_reasoning_field_enables_thinking(): spec2 = RP.convert_responses_to_genspec(req2, {}) assert spec2.chat_template_kwargs == {"thinking_mode": "chat"} + # unrelated extra kwargs ride along without discarding the reasoning mapping + req3 = ResponsesRequest.model_validate( + {"model": "m", "input": "hi", "reasoning": {"effort": "none"}, + "chat_template_kwargs": {"custom_var": 1}} + ) + assert RP.convert_responses_to_genspec(req3, {}).chat_template_kwargs == { + "enable_thinking": False, "custom_var": 1, + } + # absent reasoning -> no kwargs - req3 = ResponsesRequest.model_validate({"model": "m", "input": "hi"}) - assert RP.convert_responses_to_genspec(req3, {}).chat_template_kwargs == {} + req4 = ResponsesRequest.model_validate({"model": "m", "input": "hi"}) + assert RP.convert_responses_to_genspec(req4, {}).chat_template_kwargs == {} def test_convert_reasoning_effort_none_disables_thinking():