serve: opt-in guard for a repeated phrase (repeat_stop_period), and name repetition stops (stop_cause)

docs/DETAILS.md

@@ -857,6 +857,13 @@ print(r.choices[0].message.content)
   server ends it there with `finish_reason` `"length"` and says so in its window: a model in a loop, or a broken
   state that answers one token forever (#606 saw 36,689 tokens of `!`). `"repeat_stop_tokens": N` in
   `strata-<model>.json` sets the run length; `0` turns it off (for a request that really wants one token many times).
+  A short phrase repeated back to back can be ended the same way (opt-in): with `"repeat_stop_period": N` (2 to
+  64; 16 is a good choice), a run of `repeat_stop_tokens` tokens that repeats with a period of 2 to N tokens ends the
+  reply (seen on a scanned page: `[unclear] ` written until `max_tokens` ran out, 12,000 tokens and 229 s). The
+  default, 1, keeps only the one-token rule. A reply either rule ends keeps `finish_reason` `"length"` (what OpenAI
+  clients expect). With `repeat_stop_period` set, its choice also says why with Strata's own field
+  `"stop_cause": "repetition"` (`"reasoning_loop"` for the thinking guard below): a larger `max_tokens` would not have
+  helped. With the default the field is left out, so replies stay as they were.
 - **Non-finite logits end the request and drop the cached state (#879, on by default).** Rarely, a window's logits
   become NaN and the reply degenerates into token 0 (`!`) repeated (#606); the poison sits in the conversation's K/V
   and recurrent state, so with the prompt cache every later turn of that conversation stays poisoned. The q8_1 clamps
@@ -2173,6 +2180,9 @@ tools/MCP are refused explicitly. Without `response_format`, ordinary text and t
 
 Structured SSE buffers the answer while sending keep-alive comments. It emits content only after validation,
 then usage/timings and `[DONE]`; failures emit an SSE error and `[DONE]` without invalid content deltas.
+With `repeat_stop_period` set, an answer the server ended because it repeated itself (`stop_cause` `"repetition"`,
+above) fails with a message that says so, instead of "increase the output budget", which only fits an answer that ran
+out of `max_tokens`.
 `/v1/status.structured_output` advertises the formats, validation method and buffered streaming behavior.
 
 ### API request monitor

serve/server.py

@@ -147,6 +147,10 @@ def write_temporary(path: Path, parts: list) -> None:
 # broken state that answers one token forever (an issue saw 36,689 tokens of "!"). The config's "repeat_stop_tokens"
 # sets it; 0 turns it off.
 REPEAT_STOP_TOKENS = 256
+# The same guard for a short phrase repeated back to back ("[unclear] " forever, 3 tokens a time), opt-in: with the
+# config's "repeat_stop_period": N (2-64; 16 is a good choice), a run of repeat_stop_tokens tokens that repeats with
+# a period of 2 to N tokens ends the reply the same way.  1 (the default) keeps only the one-token guard.
+REPEAT_STOP_PERIOD = 1
 ANSWER_ROOM_MIN_CAP = 1024    # 0.1.45: from this max_tokens on, thinking without a budget is closed early enough to leave room to answer
 ANSWER_RESERVE_MIN = 512      # #984: the tokens kept for the answer when a thinking budget is cut down to fit max_tokens
 # #123: what closes the thinking when it reaches reasoning_budget_tokens (the model's own end-of-thinking tag after it)
@@ -3182,6 +3186,7 @@ def __init__(self, engine: Engine, tokenizer, template: ChatTemplate, model_name
         self.reasoning_loop_recovery = False
         self.reasoning_budget_tokens = 0                 # #123: the config's default thinking budget (0: none)
         self.repeat_stop_tokens = REPEAT_STOP_TOKENS     # #606: one token this many times in a row ends a reply (0: off)
+        self.repeat_stop_period = REPEAT_STOP_PERIOD     # ... and a phrase of up to this many tokens repeated (1: off)
         self.anthropic_think_unasked = True               # #278: "anthropic_thinking": "on_request" -> False
         self.tool_call_recovery = False                   # opt-in: tool calls in forms next to the template's
         self.images = None                                # serve/images.py's ImageGen when the config has "images"
@@ -4216,6 +4221,11 @@ def _run(self, ids, thinking, tools, max_new, sampling, cancel, force=None) -> I
         parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True, recover=self.tool_call_recovery)
         detok, n, finish = Detokenizer(self.tok), 0, "length"
         run_tok, run_len, repeated = None, 0, False     # #606: the current run of one repeated token
+        stop_cause = None                               # why the generator ended a reply itself ("done": stop_cause)
+        # a short phrase repeated back to back ("[unclear] " forever): for each period p in 2..repeat_stop_period, how
+        # many tokens in a row equal the one p back; the run is p tokens longer than that count
+        period_max = max(1, min(int(self.repeat_stop_period or 1), 64))
+        period_tail, period_run, period_hit = [], [0] * (period_max + 1), 0
         thinking_n = 0                                  # tokens written while thinking (Responses' reasoning_tokens)
         stop_list = stop_strings(sampling)
         stops = StopMatcher(stop_list) if stop_list else None
@@ -4373,6 +4383,17 @@ def cut(evs):
                                 if self.repeat_stop_tokens and run_len >= self.repeat_stop_tokens:
                                     repeated = True     # #606: a degenerate output, not an answer: end it here
                                     break
+                                if self.repeat_stop_tokens and period_max > 1:
+                                    for p in range(2, period_max + 1):
+                                        same = len(period_tail) >= p and period_tail[-p] == t
+                                        period_run[p] = period_run[p] + 1 if same else 0
+                                        if same and period_run[p] + p >= self.repeat_stop_tokens:
+                                            period_hit = p
+                                    period_tail.append(t)
+                                    del period_tail[:-period_max]
+                                    if period_hit:
+                                        repeated = True  # the same short phrase over and over: end it as above
+                                        break
                                 piece = detok.push(t)
                                 tail = (tail + piece)[-2:]
                                 evs = cut(parser.feed(piece))
@@ -4541,10 +4562,19 @@ def cut(evs):
                     if cancel.is_set():
                         finish = "cancel"
                     elif looped:
+                        stop_cause = "reasoning_loop"
                         print(f"[strata] the thinking repeated the same passages (coverage={repeat_coverage:.3f}) at "
                               f"{n} tokens: ended as \"length\" (reasoning_loop_recovery: \"stop\" in "
                               "strata-<model>.json; remove it to turn this off)", flush=True)
+                    elif repeated and period_hit:
+                        stop_cause = "repetition"
+                        phrase = self.tok.decode(period_tail[-period_hit:])
+                        print(f"[strata] the reply repeated the same {period_hit} tokens ({phrase!r}) "
+                              f"{(period_run[period_hit] + period_hit) // period_hit} times in a row: ended as "
+                              "\"length\" with stop_cause \"repetition\" (repeat_stop_tokens / repeat_stop_period in "
+                              "strata-<model>.json; 0 / 1 turn this off)", flush=True)
                     elif repeated:
+                        stop_cause = "repetition"
                         print(f"[strata] the reply repeated one token ({self.tok.decode([run_tok])!r}) "
                               f"{run_len} times in a row: ended as \"length\" (repeat_stop_tokens in "
                               "strata-<model>.json; 0 turns this off). If a new request with a short prompt does the "
@@ -4645,6 +4675,10 @@ def cut(evs):
                 "reasoning_recoveries": recovery_count}
         if stops is not None and stops.hit is not None:
             done["stop_sequence"] = stops.hit
+        # "length" that a larger max_tokens would not have changed; named only with the opt-in repeat_stop_period, so a
+        # default config's replies stay as they were (the one-token guard and reasoning_loop_recovery set it too)
+        if stop_cause is not None and finish == "length" and period_max > 1:
+            done["stop_cause"] = stop_cause
         yield "done", done
 
 
@@ -4915,6 +4949,10 @@ def chunk(delta, finish=None):
                              "prompt_tokens_details": {"cached_tokens": x.get("reused") or 0}}
             if x.get("timings"):
                 last["timings"] = x["timings"]          # llama.cpp's field: the speed its clients show
+            if x.get("stop_cause"):
+                # Strata's own field beside finish_reason: "length" because the reply repeated itself ("repetition")
+                # or the thinking looped ("reasoning_loop"), so a larger max_tokens would not have helped
+                last["choices"][0]["stop_cause"] = x["stop_cause"]
             yield last
 
 
@@ -4957,6 +4995,8 @@ def openai_collect(chunks) -> dict:
     out = {"id": last["id"], "object": "chat.completion", "created": last["created"], "model": last["model"],
            "choices": [{"index": 0, "message": msg, "finish_reason": last["choices"][0]["finish_reason"]}],
            "usage": last["usage"]}
+    if last["choices"][0].get("stop_cause"):
+        out["choices"][0]["stop_cause"] = last["choices"][0]["stop_cause"]
     if last.get("timings"):
         out["timings"] = last["timings"]
     return out
@@ -4976,7 +5016,8 @@ def structured_chunks(chunks, validator):
                 yield chunk if progress else None
         result = openai_collect(buffered)
         choice = result["choices"][0]
-        content = validated_json(choice["message"]["content"], validator, choice["finish_reason"])
+        content = validated_json(choice["message"]["content"], validator, choice["finish_reason"],
+                                 choice.get("stop_cause"))
         yield buffered[0]
         delta = {"content": content}
         if choice["message"].get("reasoning_content"):
@@ -6030,8 +6071,10 @@ def _responses(self, req):
             except (GpuBusy, EngineStarting, EngineStuck, EngineDied) as e:
                 return self._json(503, responses_error_body(str(e), "server_error", code="server_error"))
 
+            ended = {}                                       # the generator's "done": its stop_cause for check()
+
             def check(text, finish):
-                return validated_json(text, validator, finish)
+                return validated_json(text, validator, finish, ended.get("stop_cause"))
 
             def events():
                 yield from asm.start()
@@ -6043,6 +6086,7 @@ def events():
                         yield from asm.feed(x)
                     elif kind == "done":
                         done = x
+                        ended.update(x)
                 if done is not None and done["finish"] != "cancel" and not cancel.is_set():
                     yield from asm.finish(done, check)
             items = self._capture(events(), "responses")
@@ -7057,6 +7101,11 @@ def main() -> int:
     if isinstance(rs, bool) or not isinstance(rs, int) or rs < 0:
         raise SystemExit(f"[strata] config \"repeat_stop_tokens\" must be a whole number >= 0 (0 = off), not {rs!r}")
     svc.repeat_stop_tokens = rs
+    rp = cfg.get("repeat_stop_period", REPEAT_STOP_PERIOD)   # a short phrase repeated: up to this many tokens long
+    if isinstance(rp, bool) or not isinstance(rp, int) or not 1 <= rp <= 64:
+        raise SystemExit(f"[strata] config \"repeat_stop_period\" must be a whole number from 1 to 64 (1 = only one "
+                         f"repeated token, as #606), not {rp!r}")
+    svc.repeat_stop_period = rp
     svc.effort_end = bool(effort_end)                   # #458: "effort_position": "end" with an engine that has it
     if cfg.get("reasoning_budget_tokens") is not None:  # #123: a default thinking budget for every request
         try:

serve/structured.py

@@ -189,7 +189,24 @@ def _extract_json(text: str) -> str:
     return s[start:]
 
 
-def validated_json(text, validator, finish):
+INCOMPLETE = {
+    # the generator's "stop_cause" for a reply it ended before its end (see Service.run): a bigger budget cannot help
+    "repetition": ("structured output was incomplete: the reply repeated the same text over and over and Strata stopped "
+                   "it (repeat_stop_tokens / repeat_stop_period), so a larger output budget will not help. Long free "
+                   "text inside a JSON string can make the model loop"),
+    "reasoning_loop": ("structured output was incomplete: the thinking repeated the same passages and Strata stopped "
+                       "it (reasoning_loop_recovery), so a larger output budget will not help"),
+}
+
+
+def incomplete_message(finish, cause=None):
+    """The error for a structured answer that did not end on its own."""
+    return INCOMPLETE.get(cause) or f"structured output was incomplete (finish_reason={finish}); increase the output budget"
+
+
+def validated_json(text, validator, finish, cause=None):
+    """The answer as canonical JSON, or StructuredOutputError.  `cause`: the generator's stop_cause, when it ended
+    the reply itself (a repetition) rather than at the output budget."""
     def pairs(items):
         obj = {}
         for key, value in items:
@@ -202,7 +219,7 @@ def constant(value):
         raise ValueError(f"invalid JSON constant: {value}")
 
     if finish != "stop":
-        raise StructuredOutputError(f"structured output was incomplete (finish_reason={finish}); increase the output budget")
+        raise StructuredOutputError(incomplete_message(finish, cause))
     try:
         value = json.loads(_extract_json(text), object_pairs_hook=pairs, parse_constant=constant)
         canonical = json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":"))

serve/test_server.py

@@ -2264,11 +2264,13 @@ def test_no_profile_in_the_args(self):
 class RepeatStop(unittest.TestCase):
     """#606: one token repeated repeat_stop_tokens times in a row ends the reply as "length"; 0 turns it off."""
 
-    def run_reply(self, script, limit=None):
+    def run_reply(self, script, limit=None, period=None):
         tok = ByteTokenizer()
         svc = Service(MockEngine(tok, script, max_context=CTX), tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
         if limit is not None:
             svc.repeat_stop_tokens = limit
+        if period is not None:
+            svc.repeat_stop_period = period
         ids = tok.encode("hi")
         with contextlib.redirect_stdout(io.StringIO()) as out:
             done = [x for kind, x in svc.run(ids, False, None, 3000, {}, threading.Event()) if kind == "done"][0]
@@ -2279,15 +2281,55 @@ def test_a_long_run_is_ended(self):
         self.assertEqual(done["finish"], "length")
         self.assertEqual(done["completion_tokens"], 3 + 256)
         self.assertIn("repeated one token ('!') 256 times", log)
+        self.assertNotIn("stop_cause", done)                # the default config: the reply as before
+        done, _ = self.run_reply("ok " + "!" * 1000 + " never", period=16)
+        self.assertEqual((done["finish"], done["stop_cause"]), ("length", "repetition"))   # named with the opt-in
 
     def test_short_runs_and_off(self):
         done, _ = self.run_reply("=" * 255 + " fine")
         self.assertEqual(done["finish"], "stop")
         done, log = self.run_reply("!" * 1000, limit=0)
         self.assertEqual((done["finish"], done["completion_tokens"]), ("stop", 1001))
         self.assertNotIn("repeated one token", log)
-        done, _ = self.run_reply("ab" * 400, limit=8)       # alternating tokens are not one run
+        done, _ = self.run_reply("ab" * 400, limit=8)       # alternating tokens: not one run (period 1)
         self.assertEqual(done["finish"], "stop")
+        done, _ = self.run_reply("abcd" * 3, limit=16, period=16)   # a phrase repeated, but shorter than the limit
+        self.assertEqual(done["finish"], "stop")
+
+    def test_a_repeated_phrase_is_ended(self):
+        # "[unclear] " forever: a phrase of several tokens repeated, which the one-token run never sees
+        done, _ = self.run_reply("ok " + "[unclear] " * 100 + " never")      # off by default (period 1)
+        self.assertEqual(done["finish"], "stop")
+        done, log = self.run_reply("ok " + "[unclear] " * 100 + " never", period=16)
+        self.assertEqual((done["finish"], done["stop_cause"]), ("length", "repetition"))
+        # the run starts at the space that ends "ok " (" [unclear]" repeats from there): 256 tokens after "ok"
+        self.assertEqual(done["completion_tokens"], 2 + 256)
+        self.assertIn("repeated the same 10 tokens", log)
+        done, _ = self.run_reply("ab" * 400, limit=8, period=16)   # alternating tokens: a phrase of two
+        self.assertEqual((done["finish"], done["stop_cause"]), ("length", "repetition"))
+        self.assertEqual(done["completion_tokens"], 8)
+
+    def test_period_one_keeps_only_the_one_token_rule(self):
+        tok = ByteTokenizer()
+        svc = Service(MockEngine(tok, "ab" * 400, max_context=CTX), tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        svc.repeat_stop_tokens, svc.repeat_stop_period = 8, 1
+        with contextlib.redirect_stdout(io.StringIO()):
+            done = [x for kind, x in svc.run(tok.encode("hi"), False, None, 3000, {}, threading.Event())
+                    if kind == "done"][0]
+        self.assertEqual(done["finish"], "stop")
+        self.assertNotIn("stop_cause", done)
+        svc.engine = MockEngine(tok, "ok " + "!" * 1000, max_context=CTX)   # the one-token guard fires: still unnamed
+        with contextlib.redirect_stdout(io.StringIO()):
+            done = [x for kind, x in svc.run(tok.encode("hi"), False, None, 3000, {}, threading.Event())
+                    if kind == "done"][0]
+        self.assertEqual(done["finish"], "length")
+        self.assertNotIn("stop_cause", done)
+
+    def test_a_long_reply_that_does_not_repeat_runs_to_the_end(self):
+        varied = "".join("abcdefghijklmnopqrstuvwxyz "[(i * i * 7 + i * 3) % 27] for i in range(2000))
+        done, _ = self.run_reply(varied, period=16)
+        self.assertEqual(done["finish"], "stop")
+        self.assertNotIn("stop_cause", done)
 
 
 class NonFiniteEngine(MockEngine):