diff --git a/packages/agent/src/local_agent/agent_loop.rs b/packages/agent/src/local_agent/agent_loop.rs index 151f11505..32ea8746b 100644 --- a/packages/agent/src/local_agent/agent_loop.rs +++ b/packages/agent/src/local_agent/agent_loop.rs @@ -1307,46 +1307,6 @@ impl LocalAgentL serde_json::to_string(&messages_json).unwrap_or_default(), )); - // Dev-only file dump (NODESPACE_PROMPT_DUMP): record the EXACT prompt - // sent to the model on THIS iteration — full system prompt + the - // complete (untruncated) message list, which on later iterations - // includes the accumulated tool results fed back in. Tools are dumped - // once (iteration 0). Reliable local-disk view of exactly what - // reaches the model at every step of the loop. - if crate::local_agent::prompt_dump::enabled() { - let full_messages: Vec = messages - .iter() - .map(|m| { - serde_json::json!({ - "role": format!("{:?}", m.role).to_lowercase(), - "content": m.content, - }) - }) - .collect(); - let tools_full: Vec = if iteration == 0 { - tools - .iter() - .map(|t| { - serde_json::json!({ - "name": t.name, - "description": t.description, - "parameters": t.parameters_schema, - }) - }) - .collect() - } else { - Vec::new() - }; - crate::local_agent::prompt_dump::dump_turn_iteration( - &session.id, - iteration, - user_message, - &system_content, - &full_messages, - &tools_full, - ); - } - // Status: Thinking on_status(LocalAgentStatus::Thinking); session.status = LocalAgentStatus::Thinking; @@ -1426,17 +1386,6 @@ impl LocalAgentL serde_json::to_string(&tool_calls_json).unwrap_or_default(), )); - // Dev-only file dump (NODESPACE_PROMPT_DUMP): the RAW model response - // (pre-normalization) and parsed tool calls for THIS iteration. One - // record per ReAct iteration, so multi-iteration tool loops are fully - // captured in order. - crate::local_agent::prompt_dump::dump_response( - &session.id, - iteration, - &response_text, - &tool_calls_json, - ); - iter_span.set_attribute(KeyValue::new("prompt_tokens", usage.prompt_tokens as i64)); iter_span.set_attribute(KeyValue::new( "completion_tokens", diff --git a/packages/agent/src/local_agent/mod.rs b/packages/agent/src/local_agent/mod.rs index 649ce3019..8643a3efd 100644 --- a/packages/agent/src/local_agent/mod.rs +++ b/packages/agent/src/local_agent/mod.rs @@ -4,8 +4,8 @@ pub mod model_manager; pub mod ndjson; pub mod openai_compat_discovery; pub mod openai_compat_inference; +pub mod openai_compat_prompt_dump; pub mod otlp_tracer; -pub mod prompt_dump; pub mod prompt_templates; pub mod response_processing; pub mod routing; diff --git a/packages/agent/src/local_agent/model_manager.rs b/packages/agent/src/local_agent/model_manager.rs index 795f9dc95..3b1d4754e 100644 --- a/packages/agent/src/local_agent/model_manager.rs +++ b/packages/agent/src/local_agent/model_manager.rs @@ -190,10 +190,48 @@ const GEMMA_4_31B: CatalogEntry = CatalogEntry { min_memory_gb: 24, }; -/// Gemma 4 12B -- Google's mid-tier dense model; stronger reasoning than E4B, -/// viable on 16GB Apple Silicon with Q8_0 KV cache quantization. -/// Q4_K_M quantization; Q8_0 KV cache cuts 32K KV from ~5GB (F16) to ~2.5GB, -/// making it fit alongside ~7.4GB weights on a 16GB device. +/// Gemma 4 26B-A4B -- Google's MoE tier (25.2B total / 3.8B active experts). +/// Optional high-RAM tier, not the default: the agent-matrix eval (issue +/// #1956) scored it 12.3/16 mean across 3 reps, below Gemma 4 E4B's 13.7/16 +/// -- it under-calls `search_nodes` on query-style follow-ups more often +/// than E4B. Its failure mode is qualitatively different from dense Gemma 4 +/// 12B's, though: across all 3 reps and every tool call attempted, argument +/// JSON was never malformed (no corrupted field names, no truncated nested +/// structures) -- MoE's ~3.8B active-parameter footprint per token appears +/// to avoid the JSON-generation failures dense 12B/Q4_K_M exhibits on +/// complex nested payloads like `create_schema`. Q8_0 (not Q4) to keep +/// quantization precision loss out of that comparison. Exposed as an +/// additional selectable tier for users with RAM to spare, not a +/// replacement for the E4B default (`recommended_model_id()` is unchanged). +const GEMMA_4_26B_A4B: CatalogEntry = CatalogEntry { + id: "gemma-4-26b-a4b-q8", + family: ModelFamily::Gemma4, + name: "Gemma 4 26B-A4B Instruct Q8_0", + filename: "gemma-4-26B-A4B-it-Q8_0.gguf", + size_bytes: 26_859_860_992, + quantization: "Q8_0", + url: "https://huggingface.co/ggml-org/gemma-4-26B-A4B-it-GGUF/resolve/bb4531cda34d1ea09d9814959ed4d5833cf2a4c8/gemma-4-26B-A4B-it-Q8_0.gguf", + sha256: "b5108fd13147d1c866bb595295bc9d56f5fe744d7209f18421031d0cc47009c6", + context_window: 32_768, + default_temperature: 0.3, + type_k: Some(nodespace_nlp_engine::KvCacheQuantType::Q8_0), + type_v: Some(nodespace_nlp_engine::KvCacheQuantType::Q8_0), + min_memory_gb: 32, +}; + +/// Gemma 4 12B -- Google's mid-tier dense model. Parked, not exposed: the +/// agent-matrix eval (issue #1956) found it repeatedly emits malformed +/// tool-call JSON (escaped underscores, garbled/truncated nested structures) +/// on complex payloads like `create_schema`, across both this ggml-org GGUF +/// and the Unsloth re-upload below -- confirmed identical template/tokenizer +/// metadata between the two sources (byte-for-byte matching embedded Jinja +/// template, `eos_token_id`, and `` token-type flag), so the failure +/// is a model-capability property of dense 12B at this quantization, not a +/// GGUF-source defect. See `GEMMA_4_26B_A4B` for a Gemma 4 tier that avoids +/// this failure mode. `min_memory_gb: 24` (not the 16 an earlier revision +/// claimed) reflects repeated hard GPU OOM on a 16GB machine documented in +/// issue #1348 -- the Q8_0 KV-cache quantization here reduces but does not +/// eliminate that headroom problem. const GEMMA_4_12B: CatalogEntry = CatalogEntry { id: "gemma-4-12b-q4km", family: ModelFamily::Gemma4, @@ -205,16 +243,22 @@ const GEMMA_4_12B: CatalogEntry = CatalogEntry { sha256: "1278394b693672ac2799eadc9a83fd98259a6a88a40acfb1dcaa6c6fc895a606", context_window: 32_768, default_temperature: 0.3, - // Q8_0 cuts the 32K KV cache from ~5GB (F16) to ~2.5GB, leaving enough - // headroom alongside the ~7.4GB weights on a 16GB device. + // Q8_0 cuts the 32K KV cache from ~5GB (F16) to ~2.5GB, but this alone + // does not make 16GB viable -- see the doc comment above. type_k: Some(nodespace_nlp_engine::KvCacheQuantType::Q8_0), type_v: Some(nodespace_nlp_engine::KvCacheQuantType::Q8_0), - min_memory_gb: 16, + min_memory_gb: 24, }; -/// Gemma 4 12B (Unsloth) -- April 11 updated GGUF with corrected chat template. -/// The ggml-org GGUF has `` not marked as EOG, causing generation to -/// never stop after tool calls. Unsloth's re-upload fixes this. Trial only. +/// Gemma 4 12B (Unsloth) -- April 11 re-upload. Parked, not exposed: fails +/// identically to `GEMMA_4_12B` above (same malformed-JSON failure on +/// `create_schema`, confirmed via the agent-matrix eval for issue #1956). +/// The two GGUF sources have byte-for-byte identical embedded chat template +/// and tokenizer EOG metadata (verified against both files directly), so +/// there is no template-level difference between them -- whatever Unsloth's +/// re-upload changes, if anything, is not visible at this metadata layer. +/// `min_memory_gb: 24` for the same reason as `GEMMA_4_12B` (issue #1348's +/// repeated 16GB OOM reproductions apply equally to this file). const GEMMA_4_12B_UNSLOTH: CatalogEntry = CatalogEntry { id: "gemma-4-12b-unsloth-q4km", family: ModelFamily::Gemma4, @@ -228,7 +272,7 @@ const GEMMA_4_12B_UNSLOTH: CatalogEntry = CatalogEntry { default_temperature: 0.3, type_k: Some(nodespace_nlp_engine::KvCacheQuantType::Q8_0), type_v: Some(nodespace_nlp_engine::KvCacheQuantType::Q8_0), - min_memory_gb: 16, + min_memory_gb: 24, }; /// All catalog entries, in preference order. @@ -242,14 +286,30 @@ const CATALOG: &[&CatalogEntry] = &[ &GEMMA_4_12B, &GEMMA_4_12B_UNSLOTH, &GEMMA_4_31B, + &GEMMA_4_26B_A4B, ]; -/// RAM threshold (in bytes) at or above which the mid-tier model (Gemma 4 12B) -/// is selected instead of the small one (Gemma 4 E4B). +/// RAM threshold (in bytes) at or above which the mid-tier Ministral model +/// (Ministral 8B) is selected instead of the small one (Ministral 3B). +/// Matches `MINISTRAL_8B.min_memory_gb`. NOT used for Gemma4's medium tier -- +/// see [`RAM_THRESHOLD_GEMMA4_MEDIUM`], which tracks a different value +/// (`GEMMA_4_12B.min_memory_gb`, corrected to 24GB per issue #1348's OOM +/// evidence) and would silently drift from that catalog value again if the +/// two families shared one constant. const RAM_THRESHOLD_MEDIUM: u64 = 16 * 1024 * 1024 * 1024; // 16 GB +/// RAM threshold (in bytes) at or above which Gemma4's mid-tier model +/// (Gemma 4 12B) is selected instead of the small one (Gemma 4 E4B). Matches +/// `GEMMA_4_12B.min_memory_gb` -- kept as its own constant (not shared with +/// [`RAM_THRESHOLD_MEDIUM`]) because 12B's real memory floor is 24GB, not +/// 16GB like Ministral 8B's. +const RAM_THRESHOLD_GEMMA4_MEDIUM: u64 = 24 * 1024 * 1024 * 1024; // 24 GB + /// RAM threshold (in bytes) at or above which the large recommended model -/// (Gemma 4 31B) is selected instead of the mid-tier one (Gemma 4 12B). +/// (Gemma 4 26B-A4B) is selected instead of the mid-tier one (Gemma 4 12B). +/// Matches `GEMMA_4_26B_A4B.min_memory_gb`. Not Gemma 4 31B -- see +/// `recommended_model_id_for`'s doc comment for why the large tier +/// recommends 26B-A4B instead. const RAM_THRESHOLD_LARGE: u64 = 32 * 1024 * 1024 * 1024; // 32 GB // --------------------------------------------------------------------------- @@ -350,16 +410,21 @@ impl GgufModelManager { /// /// Returns Gemma 4 E4B as the primary llama.cpp default, per ADR-056. /// Unlike [`Self::recommended_model_id_for`]'s general three-tier Gemma4 - /// behavior, this always recommends E4B regardless of RAM: the 12B and - /// 31B tiers remain parked per ADR-046 (unresolved tool-call defects) - /// and must not become the default recommendation - /// on higher-RAM machines. Callers that want the full RAM-tiered - /// within-family recommendation should use + /// behavior, this always recommends E4B regardless of RAM: the mid tier + /// of that ladder is 12B, which remains parked per ADR-046 (unresolved + /// tool-call defects, reconfirmed via issue #1956's re-evaluation) and + /// must not become the default recommendation on higher-RAM machines. + /// (The large tier, 26B-A4B, is not parked -- it was validated and + /// exposed by issue #1956 -- but the ladder is still bypassed here + /// wholesale rather than partially, since this function's contract is a + /// single fixed default, not a RAM-tiered one.) Callers that want the + /// full RAM-tiered within-family recommendation should use /// [`Self::recommended_model_id_for`] directly. fn recommended_model_id() -> &'static str { // NOT recommended_model_id_for(ModelFamily::Gemma4) -- that RAM-tiers - // into 12B/31B, which are parked per ADR-046/ADR-056. Do not "simplify" - // this to the _for() call without re-checking that parking status. + // into 12B/26B-A4B, and 12B is parked per ADR-046/ADR-056. Do not + // "simplify" this to the _for() call without re-checking 12B's + // parking status. GEMMA_4_E4B.id } @@ -367,14 +432,21 @@ impl GgufModelManager { /// current system's RAM. /// /// - `Ministral`: 8B at or above [`RAM_THRESHOLD_MEDIUM`] (16 GB), otherwise 3B. - /// - `Gemma4`: three-tier — 31B at or above [`RAM_THRESHOLD_LARGE`], - /// 12B at or above [`RAM_THRESHOLD_MEDIUM`], otherwise E4B. + /// - `Gemma4`: three-tier — 26B-A4B at or above [`RAM_THRESHOLD_LARGE`], + /// 12B at or above [`RAM_THRESHOLD_GEMMA4_MEDIUM`] (24 GB), otherwise E4B. + /// Recommends 26B-A4B rather than 31B at the large tier: 31B is parked + /// (unevaluated by issue #1956) while 26B-A4B is the tier this project + /// actually validated and exposed. Neither 12B tier is recommended by + /// this function's Gemma4 branch reaching production, though -- see + /// [`Self::recommended_model_id`], which pins the real default to E4B + /// regardless of RAM for exactly that parked-model reason. /// - `OpenAiCompat`: has no GGUF catalog entries; falls back to the default /// Ministral recommendation. pub fn recommended_model_id_for(family: ModelFamily) -> &'static str { let total_ram = detect_system_ram(); let large = total_ram >= RAM_THRESHOLD_LARGE; let medium = total_ram >= RAM_THRESHOLD_MEDIUM; + let gemma4_medium = total_ram >= RAM_THRESHOLD_GEMMA4_MEDIUM; match family { ModelFamily::Ministral => { if medium { @@ -385,8 +457,8 @@ impl GgufModelManager { } ModelFamily::Gemma4 => { if large { - GEMMA_4_31B.id - } else if medium { + GEMMA_4_26B_A4B.id + } else if gemma4_medium { GEMMA_4_12B.id } else { GEMMA_4_E4B.id @@ -1352,12 +1424,13 @@ mod tests { async fn list_returns_all_catalog_models() { let (mgr, _tmp) = test_manager(); let models = mgr.list().await.unwrap(); - assert_eq!(models.len(), 9); + assert_eq!(models.len(), 10); assert!(models.iter().any(|m| m.id == "ministral-3b-q4km")); assert!(models.iter().any(|m| m.id == "ministral-8b-q4km")); assert!(models.iter().any(|m| m.id == "gemma-4-e4b-q4km")); assert!(models.iter().any(|m| m.id == "gemma-4-12b-q4km")); assert!(models.iter().any(|m| m.id == "gemma-4-31b-q4km")); + assert!(models.iter().any(|m| m.id == "gemma-4-26b-a4b-q8")); } #[tokio::test] @@ -1384,6 +1457,28 @@ mod tests { .as_ref() .is_some_and(|u| u.contains("ggml-org/gemma-4-31B-it-GGUF"))); assert_eq!(g31.min_memory_gb, 24); + + let g12 = models.iter().find(|m| m.id == "gemma-4-12b-q4km").unwrap(); + assert_eq!(g12.min_memory_gb, 24); + + let g12_unsloth = models + .iter() + .find(|m| m.id == "gemma-4-12b-unsloth-q4km") + .unwrap(); + assert_eq!(g12_unsloth.min_memory_gb, 24); + + let g26 = models + .iter() + .find(|m| m.id == "gemma-4-26b-a4b-q8") + .unwrap(); + assert_eq!(g26.family, ModelFamily::Gemma4); + assert_eq!(g26.quantization, "Q8_0"); + assert!(g26.size_bytes > 26_000_000_000); // ~26.9 GB + assert!(g26 + .url + .as_ref() + .is_some_and(|u| u.contains("ggml-org/gemma-4-26B-A4B-it-GGUF"))); + assert_eq!(g26.min_memory_gb, 32); } #[tokio::test] @@ -1586,7 +1681,8 @@ mod tests { assert_eq!(spec.context_window, 32_768); assert!(spec.type_k.is_none()); - // 12B uses Q8_0 KV compression to fit alongside the ~7.4GB weights on 16GB RAM. + // 12B uses Q8_0 KV compression to reduce (not eliminate) memory pressure + // from the ~7.4GB weights -- min_memory_gb is 24, not 16 (issue #1348). let spec = mgr.model_spec_for("gemma-4-12b-q4km").unwrap(); assert_eq!(spec.model_id, "gemma-4-12b-q4km"); assert_eq!(spec.family, ModelFamily::Gemma4); @@ -1630,6 +1726,34 @@ mod tests { assert_ne!(ministral_rec, gemma_rec); } + #[test] + fn ram_thresholds_agree_with_the_catalog_entries_they_gate() { + // recommended_model_id_for's RAM tiering constants must match the + // min_memory_gb of the catalog entry each threshold gates -- a + // catalog correction (e.g. issue #1348's 12B min_memory_gb: 16 -> 24) + // that isn't mirrored here silently recommends a model to machines + // that don't meet its own declared memory floor. Asserted directly + // against the constants rather than by RAM-mocking + // recommended_model_id_for, since detect_system_ram() reads the real + // host and cannot be faked from a unit test. + assert_eq!( + RAM_THRESHOLD_MEDIUM, + MINISTRAL_8B.min_memory_gb as u64 * 1024 * 1024 * 1024, + "RAM_THRESHOLD_MEDIUM must match MINISTRAL_8B.min_memory_gb" + ); + assert_eq!( + RAM_THRESHOLD_GEMMA4_MEDIUM, + GEMMA_4_12B.min_memory_gb as u64 * 1024 * 1024 * 1024, + "RAM_THRESHOLD_GEMMA4_MEDIUM must match GEMMA_4_12B.min_memory_gb" + ); + assert_eq!( + RAM_THRESHOLD_LARGE, + GEMMA_4_26B_A4B.min_memory_gb as u64 * 1024 * 1024 * 1024, + "RAM_THRESHOLD_LARGE must match GEMMA_4_26B_A4B.min_memory_gb -- \ + the large Gemma4 tier recommends 26B-A4B, not the parked 31B" + ); + } + #[test] fn detect_system_ram_returns_nonzero() { let ram = detect_system_ram(); diff --git a/packages/agent/src/local_agent/openai_compat_inference.rs b/packages/agent/src/local_agent/openai_compat_inference.rs index 02b304727..5f1483595 100644 --- a/packages/agent/src/local_agent/openai_compat_inference.rs +++ b/packages/agent/src/local_agent/openai_compat_inference.rs @@ -262,6 +262,19 @@ impl ChatInferenceEngine for OpenAiCompatInferenceEngine { tracing::debug!(base_url = %self.base_url, payload = %json, "OpenAI-compat request"); } } + // Dev-only full-fidelity dump (NODESPACE_PROMPT_DUMP) — see + // openai_compat_prompt_dump's module doc. Complements the debug-log + // line above: that line requires RUST_LOG=debug and is not persisted + // beyond the log; this is a durable, structured record correlated + // with its response via `dump_seq`. + let dump_seq = crate::local_agent::openai_compat_prompt_dump::enabled().then(|| { + let request_json = + serde_json::to_value(&openai_request).unwrap_or(serde_json::Value::Null); + crate::local_agent::openai_compat_prompt_dump::dump_request( + &self.base_url, + &request_json, + ) + }); tracing::info!( base_url = %self.base_url, message_count = openai_request.messages.len(), @@ -303,6 +316,9 @@ impl ChatInferenceEngine for OpenAiCompatInferenceEngine { // Accumulates streamed tool-call fragments by index, since the // SSE delta format sends id/name once and arguments incrementally. let mut tool_call_ids: Vec> = Vec::new(); + // Raw SSE payload for NODESPACE_PROMPT_DUMP -- see the request-side + // dump above for why this exists alongside the debug-log line. + let mut raw_response_accum = String::new(); while let Some(chunk) = stream.next().await { let chunk = chunk.map_err(|e| InferenceError::Engine(e.to_string()))?; @@ -316,9 +332,21 @@ impl ChatInferenceEngine for OpenAiCompatInferenceEngine { }; let data = data.trim(); if data == "[DONE]" { + if let Some(seq) = dump_seq { + crate::local_agent::openai_compat_prompt_dump::dump_response( + seq, + &raw_response_accum, + ); + } on_chunk(StreamingChunk::Done { usage: final_usage }); return Ok(final_usage); } + if dump_seq.is_some() { + if !raw_response_accum.is_empty() { + raw_response_accum.push('\n'); + } + raw_response_accum.push_str(data); + } match serde_json::from_str::(data) { Ok(resp) => { @@ -372,6 +400,12 @@ impl ChatInferenceEngine for OpenAiCompatInferenceEngine { } } // Stream ended without an explicit [DONE] sentinel. + if let Some(seq) = dump_seq { + crate::local_agent::openai_compat_prompt_dump::dump_response( + seq, + &raw_response_accum, + ); + } on_chunk(StreamingChunk::Done { usage: final_usage }); } else { let body = response @@ -379,6 +413,10 @@ impl ChatInferenceEngine for OpenAiCompatInferenceEngine { .await .map_err(|e| InferenceError::Engine(e.to_string()))?; + if let Some(seq) = dump_seq { + crate::local_agent::openai_compat_prompt_dump::dump_response(seq, &body); + } + match serde_json::from_str::(&body) { Ok(resp) => { if let Some(usage) = &resp.usage { diff --git a/packages/agent/src/local_agent/openai_compat_prompt_dump.rs b/packages/agent/src/local_agent/openai_compat_prompt_dump.rs new file mode 100644 index 000000000..31cda9ec8 --- /dev/null +++ b/packages/agent/src/local_agent/openai_compat_prompt_dump.rs @@ -0,0 +1,88 @@ +//! Dev-only prompt/response dump for the OpenAI-compatible HTTP path. +//! +//! `OpenAiCompatInferenceEngine` never touches `nlp-engine::ChatEngine` — it +//! is a pure HTTP client — so it cannot use `nlp_engine::chat::prompt_dump`, +//! which covers the native (GGUF) path's single chokepoint instead. This is +//! the equivalent capture point for the OpenAI-compat path: same env var, +//! same file, same record shape, so a session mixing both paths (or a +//! provider swap mid-conversation) produces one continuous timeline. +//! +//! When `NODESPACE_PROMPT_DUMP` is set to a file path, every call appends the +//! exact outgoing request JSON and the raw HTTP response body to that file as +//! line-delimited JSON. When the env var is unset this is a zero-cost no-op. + +use std::io::Write; +use std::sync::atomic::{AtomicU64, Ordering}; + +const ENV_DUMP_PATH: &str = "NODESPACE_PROMPT_DUMP"; + +/// A parallel, independently-numbered counter from +/// `nlp_engine::chat::prompt_dump`'s own `SEQ` -- each engine implementation +/// has its own process-wide sequence, so a `seq` alone does not disambiguate +/// which engine produced a record. Every record from this module also +/// carries `"engine": "openai_compat"` for that reason. +static SEQ: AtomicU64 = AtomicU64::new(0); + +fn dump_path() -> Option { + match std::env::var(ENV_DUMP_PATH) { + Ok(p) if !p.trim().is_empty() => Some(p), + _ => None, + } +} + +pub fn enabled() -> bool { + dump_path().is_some() +} + +fn append(value: &serde_json::Value) { + let Some(path) = dump_path() else { return }; + let line = match serde_json::to_string(value) { + Ok(s) => s, + Err(e) => { + tracing::debug!(error = %e, "openai_compat prompt_dump: serialize failed"); + return; + } + }; + match std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + { + Ok(mut f) => { + if let Err(e) = writeln!(f, "{line}") { + tracing::debug!(error = %e, path = %path, "openai_compat prompt_dump: write failed"); + } + } + Err(e) => tracing::debug!(error = %e, path = %path, "openai_compat prompt_dump: open failed"), + } +} + +/// Dump the exact outgoing request (already-serialized JSON value). Returns +/// the sequence number to pass to [`dump_response`]. +pub fn dump_request(base_url: &str, request_json: &serde_json::Value) -> u64 { + let seq = SEQ.fetch_add(1, Ordering::Relaxed); + if enabled() { + append(&serde_json::json!({ + "kind": "prompt", + "engine": "openai_compat", + "seq": seq, + "base_url": base_url, + "request": request_json, + })); + } + seq +} + +/// Dump the raw HTTP response body for the call identified by `seq`. +pub fn dump_response(seq: u64, raw_body: &str) { + if !enabled() { + return; + } + append(&serde_json::json!({ + "kind": "response", + "engine": "openai_compat", + "seq": seq, + "raw_response": raw_body, + "raw_response_len": raw_body.len(), + })); +} diff --git a/packages/agent/src/local_agent/prompt_dump.rs b/packages/agent/src/local_agent/prompt_dump.rs deleted file mode 100644 index 10e3f8e38..000000000 --- a/packages/agent/src/local_agent/prompt_dump.rs +++ /dev/null @@ -1,111 +0,0 @@ -//! Dev-only prompt/response dump. -//! -//! When `NODESPACE_PROMPT_DUMP` is set to a file path, every agent turn appends -//! the *exact* assembled system prompt, the full message list sent to the model, -//! the tools offered, and the raw (pre-normalization) model response to that -//! file as line-delimited JSON. When the env var is unset this is a zero-cost -//! no-op. -//! -//! This gives a 100%-reliable view of what actually reached the model — the -//! daemon log only records the system-prompt *length* and a short preview, which -//! is insufficient for diagnosing prompt/tool-call issues. Reach for this when -//! you need to see the verbatim prompt and the raw model output. -//! -//! Usage: -//! ```sh -//! NODESPACE_PROMPT_DUMP=/tmp/dump.jsonl -//! # then inspect /tmp/dump.jsonl — one JSON object per line: -//! # {"kind":"turn","iteration":0,"system_prompt":"","messages":[...],"tools":[...]} -//! # {"kind":"response","iteration":0,"raw_response":"","tool_calls":[...]} -//! ``` - -use std::io::Write; - -/// Env var holding the dump file path. Unset → no-op. -const ENV_DUMP_PATH: &str = "NODESPACE_PROMPT_DUMP"; - -/// Returns the configured dump path, or `None` when dumping is disabled. -fn dump_path() -> Option { - match std::env::var(ENV_DUMP_PATH) { - Ok(p) if !p.trim().is_empty() => Some(p), - _ => None, - } -} - -/// Append one JSON value as a line to the dump file. Best-effort: any IO error -/// is logged at debug and otherwise ignored — dumping must never affect a turn. -fn append(value: &serde_json::Value) { - let Some(path) = dump_path() else { return }; - let line = match serde_json::to_string(value) { - Ok(s) => s, - Err(e) => { - tracing::debug!(error = %e, "prompt_dump: serialize failed"); - return; - } - }; - match std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&path) - { - Ok(mut f) => { - if let Err(e) = writeln!(f, "{line}") { - tracing::debug!(error = %e, path = %path, "prompt_dump: write failed"); - } - } - Err(e) => tracing::debug!(error = %e, path = %path, "prompt_dump: open failed"), - } -} - -/// True when dumping is enabled — lets callers skip building the JSON payload. -pub fn enabled() -> bool { - dump_path().is_some() -} - -/// Dump the exact prompt sent to the model on one ReAct iteration: the system -/// prompt, the complete (untruncated) message list — which on later iterations -/// includes accumulated tool results — and the tools offered (iteration 0 only, -/// empty thereafter to avoid repeating the schema each step). -pub fn dump_turn_iteration( - session_id: &str, - iteration: usize, - user_message: &str, - system_prompt: &str, - messages: &[serde_json::Value], - tools: &[serde_json::Value], -) { - if !enabled() { - return; - } - append(&serde_json::json!({ - "kind": "turn", - "session_id": session_id, - "iteration": iteration, - "user_message": user_message, - "system_prompt": system_prompt, - "system_prompt_len": system_prompt.len(), - "messages": messages, - "tools": tools, - })); -} - -/// Dump the raw (pre-normalization) model response and parsed tool calls for one -/// ReAct iteration. -pub fn dump_response( - session_id: &str, - iteration: usize, - raw_response: &str, - tool_calls: &[serde_json::Value], -) { - if !enabled() { - return; - } - append(&serde_json::json!({ - "kind": "response", - "session_id": session_id, - "iteration": iteration, - "raw_response": raw_response, - "raw_response_len": raw_response.len(), - "tool_calls": tool_calls, - })); -} diff --git a/packages/desktop-app/src-tauri/src/commands/chat_models.rs b/packages/desktop-app/src-tauri/src/commands/chat_models.rs index cbab63413..55f116f07 100644 --- a/packages/desktop-app/src-tauri/src/commands/chat_models.rs +++ b/packages/desktop-app/src-tauri/src/commands/chat_models.rs @@ -38,13 +38,20 @@ fn grpc_err(msg: impl std::fmt::Display) -> CommandError { /// /// Deliberately curated, not the full Rust catalog — models are added here /// only after evaluation. Per ADR-056, Gemma 4 E4B is the locked native -/// model (superseding the prior Ministral 8B lock); Gemma 4 12B/31B remain -/// parked (unresolved tool-call defects — see the `GEMMA_4_12B` catalog -/// entry's doc comment in `model_manager.rs`) and are intentionally not -/// exposed here. The Qwen-family catalog entries (Qwen3, Qwen3.5, Qwen3.6, -/// and the Qwen3.5-based Ornith 1.0 9B) have been removed from the catalog +/// default model (superseding the prior Ministral 8B lock) — unchanged by +/// this list. Gemma 4 12B (both the `ggml-org` and Unsloth GGUF sources) +/// remains parked: re-evaluated against the agent-matrix eval for issue +/// #1956, it still produces malformed tool-call JSON on complex nested +/// payloads (`create_schema`) — see the `GEMMA_4_12B`/`GEMMA_4_12B_UNSLOTH` +/// catalog entries' doc comments in `model_manager.rs`. Gemma 4 26B-A4B +/// (MoE) is exposed as an additional optional tier for high-RAM machines — +/// it scored below E4B overall but never exhibited 12B's JSON-malformation +/// failure mode (see `GEMMA_4_26B_A4B`'s doc comment); it is not the +/// default. Gemma 4 31B (dense) remains parked, unevaluated by this issue. +/// The Qwen-family catalog entries (Qwen3, Qwen3.5, Qwen3.6, and the +/// Qwen3.5-based Ornith 1.0 9B) have been removed from the catalog /// entirely, not just withheld here. -const EXPOSED_GGUF_MODEL_IDS: &[&str] = &["gemma-4-e4b-q4km"]; +const EXPOSED_GGUF_MODEL_IDS: &[&str] = &["gemma-4-e4b-q4km", "gemma-4-26b-a4b-q8"]; /// List models in the catalog with their current status. /// diff --git a/packages/desktop-app/src/lib/components/settings/model-manager.svelte b/packages/desktop-app/src/lib/components/settings/model-manager.svelte index e7da2f234..eba1b070c 100644 --- a/packages/desktop-app/src/lib/components/settings/model-manager.svelte +++ b/packages/desktop-app/src/lib/components/settings/model-manager.svelte @@ -24,8 +24,6 @@ const downloadProgress = $derived(modelStore.downloadProgress); const isLoading = $derived(modelStore.isLoading); const systemRamGb = $derived(modelStore.systemRamGb); - const MIN_RAM_GB = 16; - const ramTooLow = $derived(systemRamGb > 0 && systemRamGb < MIN_RAM_GB); // Settings shows the curated set already filtered at the Tauri layer // (see EXPOSED_GGUF_MODEL_IDS in chat_models.rs); families listed here must @@ -33,6 +31,16 @@ const LOCAL_FAMILIES: ModelFamily[] = ['gemma4']; const localModels = $derived(models.filter((m) => LOCAL_FAMILIES.includes(m.family))); + // The global notice fires when NOTHING fits (the smallest exposed model's + // own requirement), not a flat constant -- otherwise a machine that clears + // one model's floor but not another's (e.g. E4B's 16GB but not 26B-A4B's + // 32GB) would see every card dimmed by a bar that doesn't apply to all of + // them. Per-card dimming below uses each model's own min_memory_gb. + const minRequiredGb = $derived( + localModels.length > 0 ? Math.min(...localModels.map((m) => m.min_memory_gb)) : 0 + ); + const ramTooLow = $derived(systemRamGb > 0 && systemRamGb < minRequiredGb); + // --- Models discovered at configured OpenAI-compatible endpoints --- // The daemon queries each endpoint's /models and returns one row per model, // so "did it answer" is simply whether any rows came back. @@ -266,7 +274,7 @@ {#if ramTooLow}

- Your machine has {systemRamGb} GB RAM. Local models require at least {MIN_RAM_GB} GB. + Your machine has {systemRamGb} GB RAM. Local models require at least {minRequiredGb} GB. Use a remote endpoint instead.

{/if} @@ -276,7 +284,8 @@ {:else} {#each localModels as m (m.id)} {@const progress = downloadProgress[m.id]} -
+ {@const modelRamTooLow = systemRamGb > 0 && systemRamGb < m.min_memory_gb} +
{m.name} @@ -299,7 +308,7 @@
{#if m.status.status === 'not_downloaded'} - + {:else if m.status.status === 'downloading'} {:else if m.status.status === 'ready'} diff --git a/packages/desktop-app/src/lib/components/viewers/ai-chat-model-selector.svelte b/packages/desktop-app/src/lib/components/viewers/ai-chat-model-selector.svelte index fce43fce7..927463069 100644 --- a/packages/desktop-app/src/lib/components/viewers/ai-chat-model-selector.svelte +++ b/packages/desktop-app/src/lib/components/viewers/ai-chat-model-selector.svelte @@ -40,8 +40,6 @@ const log = createLogger('AiChatModelSelector'); - const MIN_RAM_GB = 16; - // Sentinel values used in the