From 3891dd3566fd08b7c05ea878d786031da19a5ad5 Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Tue, 14 Jul 2026 11:30:08 -0600 Subject: [PATCH] Fix chat hang: stop sending num_ctx to the LLM gateway MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit num_ctx makes Ollama reload the model at a different context size (an unload+load). Against a resident qwen3:32b at ctx 32768, our request asked for 8192 — forcing a 29GB reload on every call, which stalls indefinitely whenever the backend's eviction path is unhealthy. A/B against the gateway, identical requests otherwise: stream, no num_ctx -> 200 in 0.31s stream, num_ctx=8192 -> http 000, hangs 60s+ It was also counterproductive: it shrank the context the model already served. --- backend/src/spicebook/chat/llm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/src/spicebook/chat/llm.py b/backend/src/spicebook/chat/llm.py index 12463c4..322e481 100644 --- a/backend/src/spicebook/chat/llm.py +++ b/backend/src/spicebook/chat/llm.py @@ -97,7 +97,10 @@ async def chat_completion_stream( "messages": messages, "temperature": 0.3, "max_tokens": settings.chat_max_tokens, - "num_ctx": settings.chat_max_tokens, + # No num_ctx: it makes Ollama reload the model at a different context + # size, which is an unload+load. That stalls whenever the backend's + # eviction path is unhealthy, and it would *shrink* the context the + # resident model already serves. Let the gateway pick the context. "stream": True, }, ) as resp: