fix: truncate context logic

Plugins/ZShell/Llm/llmclient.cpp

@@ -12,15 +12,16 @@
 #include <QJsonObject>
 #include <QNetworkReply>
 #include <QNetworkRequest>
+#include <QRegularExpression>
 #include <QSet>
 #include <QUrl>
 
+#include <memory>
+
 namespace ZShell::llm {
 
 namespace {
 
-// Minimum prompt budget kept even when the transcript alone nearly fills
-// the window, so the model always sees the triggering message.
 constexpr int kMinContextTokens = 512;
 
 qsizetype jsonValueChars(const QJsonValue& value) {
@@ -49,16 +50,11 @@ int estimateMessageTokens(const QJsonObject& message) {
 	return int(jsonValueChars(message) / LlmTool::CharsPerToken) + 4;
 }
 
-// One message model row and the JSON messages it produces. Trimming works
-// in whole units so an assistant turn and its tool results always travel
-// together.
 struct ContextUnit {
 	QList<QJsonObject> messages;
 	int tokens = 0;
 };
 
-// Shrink the free-text fields of a unit until it fits the budget. Used
-// when a single unit (e.g. one huge user message) alone exceeds it.
 void shrinkUnitToBudget(QList<QJsonObject>& unit, int budgetTokens) {
 	static const QStringList kTextKeys = {
 		QStringLiteral("content"), QStringLiteral("reasoning_content")};
@@ -90,6 +86,41 @@ void shrinkUnitToBudget(QList<QJsonObject>& unit, int budgetTokens) {
 	}
 }
 
+int estimateArrayTokens(const QJsonArray& messages) {
+	int total = 0;
+	for (const QJsonValue& value : messages)
+		total += estimateMessageTokens(value.toObject());
+	return total;
+}
+
+constexpr int kToolResultStubChars = 512;
+
+constexpr double kPromptEstimateDiscount = 10.0 / 12.0;
+
+constexpr int kMinToolResultTokens = 128;
+
+int completionReserve(int contextSize) {
+	return qBound(2048, contextSize / 8, 8192);
+}
+
+int promptBudgetTokens(int contextSize) {
+	return int(
+		(contextSize - completionReserve(contextSize)) *
+		kPromptEstimateDiscount);
+}
+
+constexpr int kRefreshIntervalMs = 60'000;
+
+constexpr int kProbeTimeoutMs = 2000;
+
+int contextSizeFromErrorMessage(const QString& message) {
+	static const QRegularExpression re(
+		QLatin1String("available context size \\((\\d+)"));
+	const auto match = re.match(message);
+	if (match.hasMatch()) return match.captured(1).toInt();
+	return 0;
+}
+
 } // namespace
 
 QString LlmClient::completionsPath(
@@ -140,7 +171,12 @@ LlmClient::LlmClient(QObject* parent) : QObject(parent) {
 		&ToolRegistry::enabledChanged,
 		this,
 		&LlmClient::toolsEnabledChanged);
-	probeContextSize();
+
+	m_refreshTimer.setParent(this);
+	m_refreshTimer.setInterval(kRefreshIntervalMs);
+	connect(
+		&m_refreshTimer, &QTimer::timeout, this, &LlmClient::refreshFromServer);
+	m_refreshTimer.start();
 }
 
 LlmClient::~LlmClient() {
@@ -153,8 +189,8 @@ void LlmClient::setEndpoint(const QString& value) {
 	if (m_endpoint == value) return;
 	m_endpoint = value;
 	Q_EMIT endpointChanged();
-	probeContextSize();
-	if (m_model.isEmpty()) refreshModels();
+	m_refreshTimer.start();
+	refreshFromServer();
 }
 
 void LlmClient::setModel(const QString& value) {
@@ -170,6 +206,8 @@ void LlmClient::setTemperature(double value) {
 
 void LlmClient::startGeneration(ChatSession* session, ChatGeneration* target) {
 	if (m_busy || !session || !target) return;
+	refreshModels();
+	m_contextRetryUsed = false;
 	m_active = session;
 	m_streaming = target;
 	target->setStreaming(true);
@@ -202,11 +240,15 @@ void LlmClient::startGeneration(ChatSession* session, ChatGeneration* target) {
 	m_reasoningMark = 0;
 	setApprovalPending(false);
 
-	sendRound();
+	probeContextSize([this, session, target]() {
+		if (m_active != session || m_streaming != target) return;
+		sendRound();
+	});
 }
 
 void LlmClient::sendRound() {
 	if (!m_active || !m_streaming) return;
+	probeContextSize();
 	m_finishReason.clear();
 	m_callBuilders.clear();
 	m_callResults.clear();
@@ -234,8 +276,6 @@ void LlmClient::sendRound() {
 		return;
 	}
 
-	// The transcript (this generation so far) is always sent in full, so
-	// reserve its estimated size out of the context budget.
 	int transcriptTokens = 0;
 	for (const QJsonValue& value : m_transcript)
 		transcriptTokens += estimateMessageTokens(value.toObject());
@@ -244,6 +284,30 @@ void LlmClient::sendRound() {
 	for (const QJsonValue& value : m_transcript)
 		messages.append(value);
 
+	if (m_contextSize > 0) {
+		const int promptLimit = promptBudgetTokens(m_contextSize);
+		while (estimateArrayTokens(messages) > promptLimit) {
+			bool stubbed = false;
+			for (qsizetype i = 0; i < messages.size(); ++i) {
+				QJsonObject message = messages.at(i).toObject();
+				if (message.value("role").toString() != QLatin1String("tool"))
+					continue;
+				const QString content = message.value("content").toString();
+				if (content.size() <= kToolResultStubChars) continue;
+				QString stub = content.left(kToolResultStubChars);
+				stub += QStringLiteral(
+							"\n[truncated to fit the context window; was "
+							"%1 characters]")
+							.arg(content.size());
+				message["content"] = stub;
+				messages[i] = message;
+				stubbed = true;
+				break;
+			}
+			if (!stubbed) break;
+		}
+	}
+
 	QJsonObject body;
 	QJsonObject streamOptions;
 	streamOptions[QStringLiteral("include_usage")] = true;
@@ -283,8 +347,21 @@ void LlmClient::sendRound() {
 			roundFinished();
 		else if (error == QNetworkReply::OperationCanceledError)
 			finishTurn();
-		else
-			fail(serverErrorMessage(responseBody, errorString));
+		else {
+			const QString message =
+				serverErrorMessage(responseBody, errorString);
+			const int reported = contextSizeFromErrorMessage(message);
+			if (reported > 0 && reported != m_contextSize &&
+				!m_contextRetryUsed) {
+				qWarning() << "LlmClient: prompt overflow, server reports"
+						   << reported << "tokens; retrying round";
+				setContextSize(reported);
+				m_contextRetryUsed = true;
+				sendRound();
+				return;
+			}
+			fail(message);
+		}
 	});
 }
 
@@ -349,21 +426,17 @@ QJsonArray LlmClient::buildContextMessages(
 	}
 
 	if (m_contextSize > 0) {
-		// Keep headroom for the model's own reply so the prompt cannot
-		// fill the whole window.
-		const int completionReserve = qBound(1024, m_contextSize / 8, 8192);
-		int budget = m_contextSize - completionReserve - extraReserveTokens;
+		int budget = m_contextSize - completionReserve(m_contextSize) -
+					 extraReserveTokens;
 		budget = qMax(budget, kMinContextTokens);
 
 		int total = 0;
 		for (const ContextUnit& unit : units)
 			total += unit.tokens;
-		// Drop oldest units until the prompt fits the budget.
 		while (units.size() > 1 && total > budget) {
 			total -= units.first().tokens;
 			units.removeFirst();
 		}
-		// A conversation cannot start with an assistant turn.
 		while (
 			units.size() > 1 &&
 			units.first().messages.first()[QStringLiteral("role")].toString() ==
@@ -371,7 +444,6 @@ QJsonArray LlmClient::buildContextMessages(
 			total -= units.first().tokens;
 			units.removeFirst();
 		}
-		// A single oversized unit still gets sent, shrunk to fit.
 		if (units.size() == 1 && units.first().tokens > budget)
 			shrinkUnitToBudget(units.first().messages, budget);
 	}
@@ -417,7 +489,21 @@ void LlmClient::roundFinished() {
 	bool hasCalls = false;
 	for (const auto& builder : m_callBuilders)
 		if (builder.seen) hasCalls = true;
-	if (m_finishReason != QLatin1String("tool_calls") && !hasCalls) {
+	if (!hasCalls) {
+		finishTurn();
+		return;
+	}
+	if (m_finishReason != QLatin1String("tool_calls")) {
+		qWarning() << "LlmClient: stream ended" << m_finishReason
+				   << "with incomplete tool calls, ending turn";
+		for (const auto& builder : m_callBuilders) {
+			if (!builder.seen || !builder.segment) continue;
+			builder.segment->finishTool(
+				QStringLiteral(
+					"Stream truncated before the tool call "
+					"completed"),
+				false);
+		}
 		finishTurn();
 		return;
 	}
@@ -458,8 +544,6 @@ void LlmClient::roundFinished() {
 	m_transcript.append(assistant);
 	m_round++;
 
-	// The whole batch of this round waits for a single user decision;
-	// approveTools() runs all of it, denyTools() ends the turn.
 	setApprovalPending(true);
 }
 
@@ -468,6 +552,18 @@ void LlmClient::executeAllCalls() {
 	m_pendingCalls = 0;
 	m_callResults = QList<ToolCallResult>(m_callBuilders.size());
 
+	int batchSize = 0;
+	for (const auto& call : m_callBuilders)
+		if (call.seen) ++batchSize;
+	int perCallChars = 0;
+	if (batchSize > 0 && m_contextSize > 0) {
+		const int available = promptBudgetTokens(m_contextSize) -
+							  estimateArrayTokens(m_transcript);
+		const int perCallTokens =
+			qMax(available / batchSize, kMinToolResultTokens);
+		perCallChars = perCallTokens * LlmTool::CharsPerToken;
+	}
+
 	for (int i = 0; i < m_callBuilders.size(); ++i) {
 		const auto& call = m_callBuilders.at(i);
 		if (!call.seen) continue;
@@ -498,18 +594,22 @@ void LlmClient::executeAllCalls() {
 		}
 
 		++m_pendingCalls;
-		tool->execute(call.id, args, [this, i, call](const QJsonObject& result) {
-			if (!m_streaming) return;
-			const bool success = result.contains(QStringLiteral("output"));
-			const QString content =
-				success ? result[QStringLiteral("output")].toString()
-						: QStringLiteral("Error: ") +
-							  result[QStringLiteral("error")].toString();
-			m_callResults[i] = {content, success};
-			if (LlmSegment* segment = call.segment)
-				segment->finishTool(content, success);
-			if (--m_pendingCalls == 0) flushCallResults();
-		});
+		tool->execute(
+			call.id,
+			args,
+			perCallChars,
+			[this, i, call](const QJsonObject& result) {
+				if (!m_streaming) return;
+				const bool success = result.contains(QStringLiteral("output"));
+				const QString content =
+					success ? result[QStringLiteral("output")].toString()
+							: QStringLiteral("Error: ") +
+								  result[QStringLiteral("error")].toString();
+				m_callResults[i] = {content, success};
+				if (LlmSegment* segment = call.segment)
+					segment->finishTool(content, success);
+				if (--m_pendingCalls == 0) flushCallResults();
+			});
 	}
 	if (m_pendingCalls == 0) flushCallResults();
 }
@@ -533,8 +633,6 @@ void LlmClient::flushCallResults() {
 void LlmClient::setApprovalPending(bool value) {
 	if (m_approvalPending == value) return;
 	m_approvalPending = value;
-	// Mirror onto the streaming generation; QML reads and answers from
-	// there, so only the affected ProcessBlock reacts.
 	if (ChatGeneration* streaming = m_streaming)
 		streaming->setApprovalPending(value);
 }
@@ -548,8 +646,6 @@ void LlmClient::approveTools() {
 		if (segment->type() != LlmSegment::Type::ToolCall ||
 			segment->status() != LlmSegment::Status::Pending)
 			continue;
-		// Restart the segment timer so elapsed time covers execution, not
-		// the wait for approval.
 		segment->setStatus(LlmSegment::Status::Running);
 		segment->begin();
 	}
@@ -592,7 +688,10 @@ void LlmClient::stop() {
 		finishTurn();
 		return;
 	}
-	if (m_reply) m_reply->abort();
+	if (m_reply)
+		m_reply->abort();
+	else
+		endStream();
 }
 
 void LlmClient::endStream() {
@@ -604,8 +703,6 @@ void LlmClient::endStream() {
 	generation->setStreaming(false);
 	m_approvalPending = false;
 	generation->setApprovalPending(false);
-	// Finalize tool calls that never received an approval (turn cancelled,
-	// round limit reached, ...).
 	for (auto* segment : generation->segments())
 		if (segment->type() == LlmSegment::Type::ToolCall &&
 			segment->status() == LlmSegment::Status::Pending)
@@ -633,6 +730,7 @@ void LlmClient::finishTurn() {
 		m_pendingClear.clear();
 	}
 	if (session) session->persist();
+	refreshFromServer();
 }
 
 void LlmClient::clearOnFinish(ChatSession* session) {
@@ -707,6 +805,12 @@ void LlmClient::handleLine(const QByteArray& line) {
 	}
 }
 
+void LlmClient::refreshFromServer() {
+	if (m_endpoint.trimmed().isEmpty()) return;
+	probeContextSize();
+	refreshModels();
+}
+
 void LlmClient::refreshModels() {
 	const QUrl url =
 		QUrl::fromUserInput(completionsPath(m_endpoint, "/models"));
@@ -742,6 +846,9 @@ void LlmClient::refreshModels() {
 		if (m_model.isEmpty()) {
 			m_model = models.first();
 			Q_EMIT modelChanged();
+		} else if (!models.contains(m_model)) {
+			m_model = models.first();
+			Q_EMIT modelChanged();
 		}
 	});
 }
@@ -753,15 +860,22 @@ void LlmClient::setContextSize(int size) {
 	Q_EMIT contextSizeChanged();
 }
 
-void LlmClient::probeContextSize() {
+void LlmClient::probeContextSize(std::function<void()> done) {
 	QString base = m_endpoint.trimmed();
 	while (base.endsWith('/'))
 		base.chop(1);
 	const QUrl url = QUrl::fromUserInput(base + "/props");
-	if (!url.isValid() || url.host().isEmpty()) return;
+	if (!url.isValid() || url.host().isEmpty()) {
+		if (done) done();
+		return;
+	}
 
 	auto* reply = m_manager.get(QNetworkRequest(url));
-	connect(reply, &QNetworkReply::finished, this, [this, reply]() {
+	auto settled = std::make_shared<bool>(false);
+	const auto finish = [this, reply, settled, done]() {
+		if (*settled) return;
+		*settled = true;
+
 		const QNetworkReply::NetworkError error = reply->error();
 		const QByteArray data = reply->readAll();
 		reply->deleteLater();
@@ -786,7 +900,15 @@ void LlmClient::probeContextSize() {
 							   .toInt(0);
 			}
 		}
-		setContextSize(size > 0 ? size : 4096);
+		if (size > 0)
+			setContextSize(size);
+		else if (m_contextSize <= 0)
+			setContextSize(4096);
+		if (done) done();
+	};
+	connect(reply, &QNetworkReply::finished, this, finish);
+	QTimer::singleShot(kProbeTimeoutMs, this, [reply, settled]() {
+		if (!*settled) reply->abort();
 	});
 }