new chatlistview component, file attachments, performance & primitive test

2026-09-30 15:01:28 +02:00
parent 51ed270774
commit d8a3d5732e
36 changed files with 4368 additions and 310 deletions
@@ -1,5 +1,6 @@
 #include "llmclient.hpp"

+#include "attachment.hpp"
 #include "generation.hpp"
 #include "message.hpp"
 #include "messagemodel.hpp"
@@ -7,6 +8,7 @@
 #include "session.hpp"

 #include <QDebug>
+#include <QFile>
 #include <QJsonArray>
 #include <QJsonDocument>
 #include <QJsonObject>
@@ -132,26 +134,30 @@ QString LlmClient::completionsPath(
 	return base + subpath;
 }

+QString LlmClient::jsonErrorMessage(const QByteArray& body) {
+	const QJsonDocument doc = QJsonDocument::fromJson(body);
+	if (!doc.isObject()) return QString();
+	const QJsonValue errorValue = doc.object().value("error");
+	if (errorValue.isObject())
+		return errorValue.toObject().value("message").toString();
+	if (errorValue.isString()) return errorValue.toString();
+	return QString();
+}
+
 QString LlmClient::serverErrorMessage(
 	const QByteArray& body, const QString& fallback) {
-	const QJsonDocument doc = QJsonDocument::fromJson(body);
-	if (doc.isObject()) {
-		const QJsonObject obj = doc.object();
-		if (obj.contains("error")) {
-			const QJsonValue errorValue = obj["error"];
-			if (errorValue.isObject()) {
-				const QString message =
-					errorValue.toObject()["message"].toString();
-				if (!message.isEmpty()) return message;
-			} else if (!errorValue.toString().isEmpty()) {
-				return errorValue.toString();
-			}
-		}
-	}
+	const QString message = jsonErrorMessage(body);
+	if (!message.isEmpty()) return message;
 	return fallback.isEmpty() ? QStringLiteral("Request to LLM server failed")
 							  : fallback;
 }

+void LlmClient::setServerError(const QString& value) {
+	if (m_serverError == value) return;
+	m_serverError = value;
+	Q_EMIT serverErrorChanged();
+}
+
 void LlmClient::setBusy(bool value) {
 	if (m_busy == value) return;
 	m_busy = value;
@@ -175,14 +181,17 @@ LlmClient::LlmClient(QObject* parent) : QObject(parent) {
 	m_refreshTimer.setParent(this);
 	m_refreshTimer.setInterval(kRefreshIntervalMs);
 	connect(
-		&m_refreshTimer, &QTimer::timeout, this, &LlmClient::refreshFromServer);
+		&m_refreshTimer, &QTimer::timeout, this, &LlmClient::refreshAllStates);
 	m_refreshTimer.start();
 }

 LlmClient::~LlmClient() {
-	if (m_reply) m_reply->abort();
+	if (m_reply) {
+		QNetworkReply* reply = m_reply;
+		m_reply = nullptr;
+		reply->abort();
+	}
 	m_tools->cancelAll();
-	endStream();
 }

 void LlmClient::setEndpoint(const QString& value) {
@@ -190,7 +199,7 @@ void LlmClient::setEndpoint(const QString& value) {
 	m_endpoint = value;
 	Q_EMIT endpointChanged();
 	m_refreshTimer.start();
-	refreshFromServer();
+	refreshAllStates();
 }

 void LlmClient::setModel(const QString& value) {
@@ -204,9 +213,16 @@ void LlmClient::setTemperature(double value) {
 	m_temperature = value;
 }

+void LlmClient::setSandboxEnabled(bool value) {
+	m_sandboxEnabled = value;
+}
+
+QString LlmClient::activeWorkspace() const {
+	return m_active ? m_active->workspacePath() : QString();
+}
+
 void LlmClient::startGeneration(ChatSession* session, ChatGeneration* target) {
 	if (m_busy || !session || !target) return;
-	refreshModels();
 	m_contextRetryUsed = false;
 	m_active = session;
 	m_streaming = target;
@@ -242,6 +258,7 @@ void LlmClient::startGeneration(ChatSession* session, ChatGeneration* target) {

 	probeContextSize([this, session, target]() {
 		if (m_active != session || m_streaming != target) return;
+		refreshModels();
 		sendRound();
 	});
 }
@@ -253,6 +270,7 @@ void LlmClient::sendRound() {
 	m_callBuilders.clear();
 	m_callResults.clear();
 	m_roundDone = false;
+	m_streamDone = false;
 	m_toolPhase = false;
 	m_pendingCalls = 0;
 	m_buffer.clear();
@@ -308,6 +326,8 @@ void LlmClient::sendRound() {
 		}
 	}

+	appendSandboxPreamble(messages);
+
 	QJsonObject body;
 	QJsonObject streamOptions;
 	streamOptions[QStringLiteral("include_usage")] = true;
@@ -323,15 +343,24 @@ void LlmClient::sendRound() {
 	request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
 	request.setRawHeader("Accept", "text/event-stream");

-	m_reply = m_manager.post(request, QJsonDocument(body).toJson());
+	QNetworkReply* reply =
+		m_manager.post(request, QJsonDocument(body).toJson());
+	if (!reply) {
+		fail(QStringLiteral("Failed to start request to LLM server"));
+		return;
+	}
+	m_reply = reply;

-	connect(m_reply, &QNetworkReply::readyRead, this, [this]() {
-		if (m_reply) m_buffer.append(m_reply->readAll());
+	connect(reply, &QNetworkReply::readyRead, this, [this, reply]() {
+		if (m_reply != reply) return;
+		m_buffer.append(reply->readAll());
 		drainBuffer();
 	});
-	connect(m_reply, &QNetworkReply::finished, this, [this]() {
-		QNetworkReply* reply = m_reply;
-		if (!reply) return;
+	connect(reply, &QNetworkReply::finished, this, [this, reply]() {
+		if (m_reply != reply) {
+			reply->deleteLater();
+			return;
+		}
 		m_reply = nullptr;

 		const QNetworkReply::NetworkError error = reply->error();
@@ -343,25 +372,37 @@ void LlmClient::sendRound() {
 		drainBuffer();
 		if (!m_streaming) return;

-		if (error == QNetworkReply::NoError)
-			roundFinished();
-		else if (error == QNetworkReply::OperationCanceledError)
-			finishTurn();
-		else {
-			const QString message =
-				serverErrorMessage(responseBody, errorString);
-			const int reported = contextSizeFromErrorMessage(message);
-			if (reported > 0 && reported != m_contextSize &&
-				!m_contextRetryUsed) {
-				qWarning() << "LlmClient: prompt overflow, server reports"
-						   << reported << "tokens; retrying round";
-				setContextSize(reported);
-				m_contextRetryUsed = true;
-				sendRound();
+		if (error == QNetworkReply::NoError) {
+			if (m_streamDone) {
+				roundFinished();
 				return;
 			}
-			fail(message);
+			const QByteArray tail = m_buffer;
+			m_buffer.clear();
+			const QString serverMessage = jsonErrorMessage(tail.simplified());
+			fail(
+				serverMessage.isEmpty()
+					? QStringLiteral(
+						  "LLM server closed the stream before "
+						  "completing the response")
+					: serverMessage);
+			return;
 		}
+		if (error == QNetworkReply::OperationCanceledError) {
+			finishTurn();
+			return;
+		}
+		const QString message = serverErrorMessage(responseBody, errorString);
+		const int reported = contextSizeFromErrorMessage(message);
+		if (reported > 0 && reported != m_contextSize && !m_contextRetryUsed) {
+			qWarning() << "LlmClient: prompt overflow, server reports"
+					   << reported << "tokens; retrying round";
+			setContextSize(reported);
+			m_contextRetryUsed = true;
+			sendRound();
+			return;
+		}
+		fail(message);
 	});
 }

@@ -373,12 +414,51 @@ QJsonArray LlmClient::buildContextMessages(
 		const auto* message = model->at(row);
 		const auto* generation = message->activeGeneration();
 		if (!generation) continue;
+		const bool hasAnyContent = !generation->content().isEmpty() ||
+								   !generation->reasoning().isEmpty() ||
+								   generation->toolCallCount() > 0;
+		if (message->role() != ChatMessage::Role::User && !hasAnyContent)
+			continue;

 		ContextUnit unit;
 		if (message->role() == ChatMessage::Role::User) {
 			QJsonObject user;
 			user[QStringLiteral("role")] = QStringLiteral("user");
-			user[QStringLiteral("content")] = generation->content();
+			const QString text = generation->content();
+			const QList<Attachment*> attachments = message->attachments();
+			if (attachments.isEmpty()) {
+				user[QStringLiteral("content")] = text;
+			} else {
+				QJsonArray parts;
+				if (!text.isEmpty()) {
+					QJsonObject textPart;
+					textPart[QStringLiteral("type")] = QStringLiteral("text");
+					textPart[QStringLiteral("text")] = text;
+					parts.append(textPart);
+				}
+				for (const auto* attachment : attachments) {
+					if (attachment->kind() != Attachment::Kind::Image) continue;
+					QFile file(attachment->path());
+					if (!file.open(QIODevice::ReadOnly)) continue;
+					const QByteArray base64 = file.readAll().toBase64();
+					file.close();
+					if (base64.isEmpty()) continue;
+					QJsonObject imageUrl;
+					imageUrl[QStringLiteral("url")] =
+						QStringLiteral("data:%1;base64,%2")
+							.arg(attachment->mimeType())
+							.arg(QString::fromLatin1(base64));
+					QJsonObject imagePart;
+					imagePart[QStringLiteral("type")] =
+						QLatin1String("image_url");
+					imagePart[QStringLiteral("image_url")] = imageUrl;
+					parts.append(imagePart);
+				}
+				if (parts.isEmpty())
+					user[QStringLiteral("content")] = text;
+				else
+					user[QStringLiteral("content")] = parts;
+			}
 			unit.messages.append(user);
 		} else {
 			QList<const LlmSegment*> toolSegments;
@@ -455,6 +535,48 @@ QJsonArray LlmClient::buildContextMessages(
 	return messages;
 }

+void LlmClient::appendSandboxPreamble(QJsonArray& messages) {
+	if (!m_sandboxEnabled) return;
+	ChatSession* active = m_active;
+	if (!active) return;
+	const QString workspace = active->workspacePath();
+	if (workspace.isEmpty()) return;
+	const auto* model = active->messagesModel();
+	QStringList files;
+	QSet<QString> seenPaths;
+	for (int row = 0; row < model->rowCount(); ++row) {
+		const auto* message = model->at(row);
+		if (message->role() != ChatMessage::Role::User) continue;
+		for (const auto* attachment : message->attachments()) {
+			if (attachment->kind() != Attachment::Kind::File) continue;
+			const QString relativePath =
+				attachment->path().mid(workspace.size() + 1);
+			if (seenPaths.contains(relativePath)) continue;
+			seenPaths.insert(relativePath);
+			files.append(QStringLiteral("- %1 (available as %2)")
+							 .arg(attachment->name(), relativePath));
+		}
+	}
+	if (files.isEmpty()) return;
+	QJsonObject preamble;
+	preamble[QStringLiteral("role")] = QStringLiteral("system");
+	preamble[QStringLiteral("content")] =
+		QStringLiteral(
+			"The user attached the following files to this "
+			"conversation. They are available in your sandbox "
+			"directory at %1:\n%2\nruncommand executes shell "
+			"commands inside a bubblewrap sandbox: the rest of the "
+			"filesystem is read-only, there is no network access, "
+			"and only the sandbox directory is writable. Use it to "
+			"inspect, transform, or extract attached files (for "
+			"example to extract archives or search across many "
+			"files). Files you create stay inside the sandbox "
+			"directory. Use the webfetch tool for anything that "
+			"needs the network.")
+			.arg(workspace, files.join(QStringLiteral("\n")));
+	messages.prepend(preamble);
+}
+
 void LlmClient::applyToolCallDelta(const QJsonObject& call) {
 	ChatGeneration* streaming = m_streaming;
 	if (!streaming) return;
@@ -688,10 +810,12 @@ void LlmClient::stop() {
 		finishTurn();
 		return;
 	}
-	if (m_reply)
-		m_reply->abort();
-	else
-		endStream();
+	if (m_reply) {
+		QNetworkReply* reply = m_reply;
+		m_reply = nullptr;
+		reply->abort();
+	}
+	endStream();
 }

 void LlmClient::endStream() {
@@ -700,6 +824,11 @@ void LlmClient::endStream() {
 	ChatSession* session = m_active;
 	m_streaming = nullptr;
 	m_active = nullptr;
+	if (m_reply) {
+		QNetworkReply* reply = m_reply;
+		m_reply = nullptr;
+		reply->abort();
+	}
 	generation->setStreaming(false);
 	m_approvalPending = false;
 	generation->setApprovalPending(false);
@@ -708,7 +837,7 @@ void LlmClient::endStream() {
 			segment->status() == LlmSegment::Status::Pending)
 			segment->finishTool(QStringLiteral("Cancelled"), false);
 	if (generation->content().isEmpty() && generation->reasoning().isEmpty() &&
-		generation->toolCallCount() == 0) {
+		generation->toolCallCount() == 0 && !generation->error()) {
 		if (auto* message = qobject_cast<ChatMessage*>(generation->parent())) {
 			if (message->generationCount() <= 1) {
 				if (session) session->removeMessage(message);
@@ -730,7 +859,7 @@ void LlmClient::finishTurn() {
 		m_pendingClear.clear();
 	}
 	if (session) session->persist();
-	refreshFromServer();
+	refreshAllStates();
 }

 void LlmClient::clearOnFinish(ChatSession* session) {
@@ -747,6 +876,7 @@ void LlmClient::sessionRemoved(ChatSession* session) {

 void LlmClient::fail(const QString& message) {
 	qWarning() << "LlmClient:" << message;
+	if (ChatGeneration* streaming = m_streaming) streaming->setError(message);
 	finishTurn();
 	Q_EMIT errorOccurred(message);
 }
@@ -766,6 +896,7 @@ void LlmClient::handleLine(const QByteArray& line) {

 	const QByteArray data = line.mid(5).trimmed();
 	if (data == "[DONE]") {
+		m_streamDone = true;
 		roundFinished();
 		return;
 	}
@@ -776,8 +907,12 @@ void LlmClient::handleLine(const QByteArray& line) {
 	updateTokenUsage(obj);

 	if (obj.contains("error")) {
-		const QJsonObject error = obj["error"].toObject();
-		const QString message = error["message"].toString();
+		const QJsonValue errorValue = obj["error"];
+		QString message;
+		if (errorValue.isObject())
+			message = errorValue.toObject()["message"].toString();
+		else if (errorValue.isString())
+			message = errorValue.toString();
 		fail(
 			message.isEmpty() ? QStringLiteral("LLM server returned an error")
 							  : message);
@@ -805,25 +940,36 @@ void LlmClient::handleLine(const QByteArray& line) {
 	}
 }

-void LlmClient::refreshFromServer() {
-	if (m_endpoint.trimmed().isEmpty()) return;
-	probeContextSize();
-	refreshModels();
+void LlmClient::refreshAllStates() {
+	if (m_endpoint.trimmed().isEmpty()) {
+		setServerError(QStringLiteral("LLM endpoint is not configured"));
+		return;
+	}
+	probeContextSize([this]() { refreshModels(); });
 }

 void LlmClient::refreshModels() {
 	const QUrl url =
 		QUrl::fromUserInput(completionsPath(m_endpoint, "/models"));
-	if (!url.isValid() || url.host().isEmpty()) return;
+	if (!url.isValid() || url.host().isEmpty()) {
+		setServerError(QStringLiteral("LLM endpoint is not configured"));
+		return;
+	}

 	auto* reply = m_manager.get(QNetworkRequest(url));
 	connect(reply, &QNetworkReply::finished, this, [this, reply]() {
 		const QNetworkReply::NetworkError error = reply->error();
+		const QString errorString = reply->errorString();
 		const QByteArray data = reply->readAll();
 		reply->deleteLater();

 		if (error != QNetworkReply::NoError) {
-			qWarning() << "LlmClient: failed to fetch models:" << error;
+			const QString serverMessage = jsonErrorMessage(data);
+			setServerError(
+				serverMessage.isEmpty()
+					? QStringLiteral("Cannot reach LLM server: %1")
+						  .arg(errorString)
+					: serverMessage);
 			return;
 		}
 		const QJsonArray dataArr =
@@ -838,7 +984,12 @@ void LlmClient::refreshModels() {
 				models.append(id);
 			}
 		}
-		if (models.isEmpty()) return;
+		if (models.isEmpty()) {
+			setServerError(QStringLiteral(
+				"LLM server is running but has no models loaded"));
+			return;
+		}
+		setServerError(QString());

 		m_availableModels = models;
 		Q_EMIT availableModelsChanged();
@@ -866,6 +1017,8 @@ void LlmClient::probeContextSize(std::function<void()> done) {
 		base.chop(1);
 	const QUrl url = QUrl::fromUserInput(base + "/props");
 	if (!url.isValid() || url.host().isEmpty()) {
+		setServerError(
+			QStringLiteral("Invalid LLM endpoint: %1").arg(m_endpoint));
 		if (done) done();
 		return;
 	}
@@ -877,11 +1030,13 @@ void LlmClient::probeContextSize(std::function<void()> done) {
 		*settled = true;

 		const QNetworkReply::NetworkError error = reply->error();
+		const QString errorString = reply->errorString();
 		const QByteArray data = reply->readAll();
 		reply->deleteLater();

 		int size = 0;
 		if (error == QNetworkReply::NoError) {
+			setServerError(QString());
 			const QJsonDocument doc = QJsonDocument::fromJson(data);
 			if (doc.isArray()) {
 				for (const auto& value : doc.array()) {
@@ -899,6 +1054,13 @@ void LlmClient::probeContextSize(std::function<void()> done) {
 							   .toObject()["n_ctx"]
 							   .toInt(0);
 			}
+		} else {
+			const QString serverMessage = jsonErrorMessage(data);
+			setServerError(
+				serverMessage.isEmpty()
+					? QStringLiteral("Cannot reach LLM server: %1")
+						  .arg(errorString)
+					: serverMessage);
 		}
 		if (size > 0)
 			setContextSize(size);