diff --git a/backend/src/spicebook/chat/llm.py b/backend/src/spicebook/chat/llm.py index 12463c4..322e481 100644 --- a/backend/src/spicebook/chat/llm.py +++ b/backend/src/spicebook/chat/llm.py @@ -97,7 +97,10 @@ async def chat_completion_stream( "messages": messages, "temperature": 0.3, "max_tokens": settings.chat_max_tokens, - "num_ctx": settings.chat_max_tokens, + # No num_ctx: it makes Ollama reload the model at a different context + # size, which is an unload+load. That stalls whenever the backend's + # eviction path is unhealthy, and it would *shrink* the context the + # resident model already serves. Let the gateway pick the context. "stream": True, }, ) as resp: