add markdown parsing + latex + tree-sitter highlighting for codeblocks in llm responses

This commit is contained in:
2026-08-24 23:55:51 +02:00
parent aae3757daf
commit e18875a752
53 changed files with 5043 additions and 618 deletions
+540 -250
View File
@@ -1,7 +1,9 @@
#include "llmclient.hpp"
#include "generation.hpp"
#include "message.hpp"
#include "messagemodel.hpp"
#include "segment.hpp"
#include "session.hpp"
#include <QDebug>
@@ -62,12 +64,19 @@ void LlmClient::setStreamingChatId(const QString& id) {
}
LlmClient::LlmClient(QObject* parent) : QObject(parent) {
m_tools = new ToolRegistry(this);
connect(
m_tools,
&ToolRegistry::enabledChanged,
this,
&LlmClient::toolsEnabledChanged);
probeContextSize();
}
LlmClient::~LlmClient() {
if (m_reply)
m_reply->abort();
m_tools->cancelAll();
endStream();
}
@@ -111,10 +120,6 @@ void LlmClient::startGeneration(
return;
}
QNetworkRequest request(url);
request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
request.setRawHeader("Accept", "text/event-stream");
const auto* model = session->messagesModel();
const int targetRow =
model->rowOf(qobject_cast<ChatMessage*>(target->parent()));
@@ -124,25 +129,50 @@ void LlmClient::startGeneration(
return;
}
// Context: every message older than the target, oldest first.
QJsonArray messages;
for (int row = model->rowCount() - 1; row > targetRow; --row) {
const auto* message = model->at(row);
const auto* generation = message->activeGeneration();
if (!generation)
continue;
QJsonObject messageObj;
messageObj[QStringLiteral("role")] =
message->role() == ChatMessage::Role::User
? QStringLiteral("user")
: QStringLiteral("assistant");
messageObj[QStringLiteral("content")] = generation->content();
if (!generation->reasoning().isEmpty())
messageObj[QStringLiteral("reasoning_content")] =
generation->reasoning();
messages.append(messageObj);
m_transcript = QJsonArray();
m_callBuilders.clear();
m_callResults.clear();
m_finishReason.clear();
m_round = 0;
m_contentMark = 0;
m_reasoningMark = 0;
sendRound();
}
void LlmClient::sendRound() {
if (!m_active || !m_streaming)
return;
m_finishReason.clear();
m_callBuilders.clear();
m_callResults.clear();
m_roundDone = false;
m_toolPhase = false;
m_pendingCalls = 0;
m_buffer.clear();
const QUrl url =
QUrl::fromUserInput(completionsPath(m_endpoint, "/chat/completions"));
if (!url.isValid() || url.host().isEmpty()) {
fail(QStringLiteral("Invalid LLM endpoint: %1").arg(m_endpoint));
return;
}
const auto* model = m_active->messagesModel();
const int targetRow =
model->rowOf(qobject_cast<ChatMessage*>(m_streaming->parent()));
if (targetRow < 0) {
fail(QStringLiteral("Internal error: generation target is not in the "
"session"));
return;
}
// Context: every message older than the target, oldest first, plus
// the tool exchanges of the current turn so far.
QJsonArray messages = buildContextMessages(m_active, targetRow);
for (const QJsonValue& value : m_transcript)
messages.append(value);
QJsonObject body;
QJsonObject streamOptions;
streamOptions[QStringLiteral("include_usage")] = true;
@@ -152,8 +182,14 @@ void LlmClient::startGeneration(
body[QStringLiteral("temperature")] = m_temperature;
if (!m_model.isEmpty())
body[QStringLiteral("model")] = m_model;
const QJsonArray toolSpecs = m_tools->specifications();
if (!toolSpecs.isEmpty())
body[QStringLiteral("tools")] = toolSpecs;
QNetworkRequest request(url);
request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
request.setRawHeader("Accept", "text/event-stream");
m_buffer.clear();
m_reply = m_manager.post(request, QJsonDocument(body).toJson());
connect(m_reply, &QNetworkReply::readyRead, this, [this]() {
@@ -178,14 +214,493 @@ void LlmClient::startGeneration(
if (!m_streaming)
return;
if (error == QNetworkReply::NoError ||
error == QNetworkReply::OperationCanceledError)
finalize();
if (error == QNetworkReply::NoError)
roundFinished();
else if (error == QNetworkReply::OperationCanceledError)
finishTurn();
else
fail(serverErrorMessage(responseBody, errorString));
});
}
QJsonArray LlmClient::buildContextMessages(
ChatSession* session, int stopBeforeRow) const {
const auto* model = session->messagesModel();
QJsonArray messages;
for (int row = model->rowCount() - 1; row > stopBeforeRow; --row) {
const auto* message = model->at(row);
const auto* generation = message->activeGeneration();
if (!generation)
continue;
if (message->role() == ChatMessage::Role::User) {
QJsonObject user;
user[QStringLiteral("role")] = QStringLiteral("user");
user[QStringLiteral("content")] = generation->content();
messages.append(user);
continue;
}
// Assistant message: replay its tool calls (and their results)
// so the model keeps the full history of the turn.
QList<const LlmSegment*> toolSegments;
for (const auto* segment : generation->segments())
if (segment->type() == LlmSegment::Type::ToolCall)
toolSegments.append(segment);
QJsonObject assistant;
assistant[QStringLiteral("role")] = QStringLiteral("assistant");
if (toolSegments.isEmpty())
assistant[QStringLiteral("content")] = generation->content();
else if (!generation->content().isEmpty())
assistant[QStringLiteral("content")] = generation->content();
if (!generation->reasoning().isEmpty())
assistant[QStringLiteral("reasoning_content")] =
generation->reasoning();
if (toolSegments.isEmpty()) {
messages.append(assistant);
continue;
}
QJsonArray calls;
for (const auto* segment : toolSegments) {
QJsonObject function;
function[QStringLiteral("name")] = segment->name();
function[QStringLiteral("arguments")] = segment->arguments();
QJsonObject call;
call[QStringLiteral("id")] = segment->toolCallId();
call[QStringLiteral("type")] = QStringLiteral("function");
call[QStringLiteral("function")] = function;
calls.append(call);
}
assistant[QStringLiteral("tool_calls")] = calls;
messages.append(assistant);
for (const auto* segment : toolSegments) {
QJsonObject toolMessage;
toolMessage[QStringLiteral("role")] = QStringLiteral("tool");
toolMessage[QStringLiteral("tool_call_id")] =
segment->toolCallId();
toolMessage[QStringLiteral("content")] = segment->result();
messages.append(toolMessage);
}
}
return messages;
}
void LlmClient::applyToolCallDelta(const QJsonObject& call) {
if (!m_streaming)
return;
const int index = call[QStringLiteral("index")].toInt(-1);
if (index < 0)
return;
while (m_callBuilders.size() <= index)
m_callBuilders.append(ToolCallBuilder{});
ToolCallBuilder& builder = m_callBuilders[index];
builder.seen = true;
const QString id = call[QStringLiteral("id")].toString();
if (!id.isEmpty())
builder.id = id;
const QJsonObject function = call[QStringLiteral("function")].toObject();
const QString name = function[QStringLiteral("name")].toString();
if (!name.isEmpty())
builder.name = name;
const QString arguments =
function[QStringLiteral("arguments")].toString();
if (!arguments.isEmpty())
builder.arguments += arguments;
if (!builder.segment) {
// A new call: close the in-flight text segments and open a
// running tool-call segment so the UI can track it live.
m_streaming->closeOpenSegments();
builder.segment =
m_streaming->beginToolCall(builder.name, builder.id);
}
builder.segment->setName(builder.name);
builder.segment->setToolCallId(builder.id);
if (!arguments.isEmpty())
builder.segment->appendArguments(arguments);
}
void LlmClient::roundFinished() {
// [DONE] and the reply's finished signal both funnel here; only the
// first may act.
if (m_roundDone || !m_streaming)
return;
m_roundDone = true;
bool hasCalls = false;
for (const auto& builder : m_callBuilders)
if (builder.seen)
hasCalls = true;
if (m_finishReason != QLatin1String("tool_calls") && !hasCalls) {
finishTurn();
return;
}
if (m_round >= kMaxToolRounds) {
qWarning() << "LlmClient: tool round limit reached, ending turn";
finishTurn();
return;
}
m_streaming->closeOpenSegments();
// Record the assistant's tool-call message in the transcript so the
// next round (and the model) can see it.
const QString content = m_streaming->content();
const QString reasoning = m_streaming->reasoning();
QJsonObject assistant;
assistant[QStringLiteral("role")] = QStringLiteral("assistant");
if (content.size() > m_contentMark) {
assistant[QStringLiteral("content")] = content.mid(m_contentMark);
m_contentMark = content.size();
}
if (reasoning.size() > m_reasoningMark) {
assistant[QStringLiteral("reasoning_content")] =
reasoning.mid(m_reasoningMark);
m_reasoningMark = reasoning.size();
}
QJsonArray calls;
for (const auto& builder : m_callBuilders) {
if (!builder.seen)
continue;
QJsonObject function;
function[QStringLiteral("name")] = builder.name;
function[QStringLiteral("arguments")] = builder.arguments;
QJsonObject call;
call[QStringLiteral("id")] = builder.id;
call[QStringLiteral("type")] = QStringLiteral("function");
call[QStringLiteral("function")] = function;
calls.append(call);
}
assistant[QStringLiteral("tool_calls")] = calls;
m_transcript.append(assistant);
m_round++;
executeAllCalls();
}
void LlmClient::executeAllCalls() {
m_toolPhase = true;
m_pendingCalls = 0;
m_callResults = QList<ToolCallResult>(m_callBuilders.size());
for (int i = 0; i < m_callBuilders.size(); ++i) {
const auto& call = m_callBuilders.at(i);
if (!call.seen)
continue;
LlmTool* tool = m_tools->tool(call.name);
QJsonObject args;
QString errorText;
if (!tool) {
errorText = QStringLiteral("Error: unknown tool '%1'")
.arg(call.name);
} else if (!call.arguments.isEmpty()) {
const QJsonDocument doc =
QJsonDocument::fromJson(call.arguments.toUtf8());
if (!doc.isObject()) {
errorText =
QStringLiteral("Error: tool arguments are not valid "
"JSON: %1")
.arg(call.arguments);
} else {
args = doc.object();
}
}
if (!errorText.isEmpty()) {
m_callResults[i] = { errorText, false };
if (LlmSegment* segment = call.segment)
segment->finishTool(errorText, false);
continue;
}
++m_pendingCalls;
tool->execute(
args,
[this, i, call](const QJsonObject& result) {
if (!m_streaming)
return;
const bool success =
result.contains(QStringLiteral("output"));
const QString content = success
? result[QStringLiteral("output")].toString()
: QStringLiteral("Error: ") +
result[QStringLiteral("error")].toString();
m_callResults[i] = { content, success };
if (LlmSegment* segment = call.segment)
segment->finishTool(content, success);
if (--m_pendingCalls == 0)
flushCallResults();
});
}
if (m_pendingCalls == 0)
flushCallResults();
}
void LlmClient::flushCallResults() {
if (!m_toolPhase)
return;
m_toolPhase = false;
if (!m_streaming)
return;
for (int i = 0; i < m_callBuilders.size(); ++i) {
const auto& call = m_callBuilders.at(i);
if (!call.seen)
continue;
QJsonObject toolMessage;
toolMessage[QStringLiteral("role")] = QStringLiteral("tool");
toolMessage[QStringLiteral("tool_call_id")] = call.id;
toolMessage[QStringLiteral("content")] =
m_callResults.at(i).content;
m_transcript.append(toolMessage);
}
sendRound();
}
void LlmClient::stop() {
if (!m_busy)
return;
if (m_toolPhase) {
m_tools->cancelAll();
if (m_streaming) {
for (auto* segment : m_streaming->segments()) {
if (segment->type() == LlmSegment::Type::ToolCall &&
segment->running())
segment->finishTool(
QStringLiteral("Cancelled"), false);
}
}
finishTurn();
return;
}
if (m_reply)
m_reply->abort();
}
void LlmClient::endStream() {
if (!m_streaming)
return;
auto* generation = m_streaming;
auto* session = m_active;
m_streaming = nullptr;
m_active = nullptr;
generation->setStreaming(false);
if (generation->content().isEmpty() &&
generation->reasoning().isEmpty() &&
generation->toolCallCount() == 0) {
if (auto* message = qobject_cast<ChatMessage*>(generation->parent())) {
if (message->generationCount() <= 1) {
if (session)
session->removeMessage(message);
} else {
message->removeGeneration(generation);
}
}
}
setBusy(false);
setStreamingChatId(QString());
}
void LlmClient::finishTurn() {
if (!m_streaming)
return;
ChatSession* session = m_active;
endStream();
if (session && m_pendingClear == session) {
session->clearMessages();
m_pendingClear.clear();
}
if (session)
session->persist();
}
void LlmClient::clearOnFinish(ChatSession* session) {
m_pendingClear = session;
}
void LlmClient::sessionRemoved(ChatSession* session) {
if (m_pendingClear == session)
m_pendingClear.clear();
if (m_active == session) {
stop();
endStream();
}
}
void LlmClient::fail(const QString& message) {
qWarning() << "LlmClient:" << message;
finishTurn();
Q_EMIT errorOccurred(message);
}
void LlmClient::drainBuffer() {
while (true) {
const qsizetype newline = m_buffer.indexOf('\n');
if (newline < 0)
break;
const QByteArray line = m_buffer.left(newline).trimmed();
m_buffer.remove(0, newline + 1);
handleLine(line);
}
}
void LlmClient::handleLine(const QByteArray& line) {
if (!m_streaming || line.isEmpty() || !line.startsWith("data:"))
return;
const QByteArray data = line.mid(5).trimmed();
if (data == "[DONE]") {
roundFinished();
return;
}
const QJsonDocument doc = QJsonDocument::fromJson(data);
if (!doc.isObject())
return;
const QJsonObject obj = doc.object();
updateTokenUsage(obj);
if (obj.contains("error")) {
const QJsonObject error = obj["error"].toObject();
const QString message = error["message"].toString();
fail(message.isEmpty()
? QStringLiteral("LLM server returned an error")
: message);
return;
}
for (const QJsonValue& choiceValue : obj["choices"].toArray()) {
if (!m_streaming)
continue;
const QJsonObject choice = choiceValue.toObject();
const QJsonObject delta = choice["delta"].toObject();
const QString finishReason =
choice[QStringLiteral("finish_reason")].toString();
if (!finishReason.isEmpty())
m_finishReason = finishReason;
m_streaming->appendContent(delta["content"].toString());
QString reasoning =
delta["reasoning_content"].toString();
if (reasoning.isEmpty())
reasoning = delta["reasoning"].toString();
m_streaming->appendReasoning(reasoning);
for (const QJsonValue& callValue :
delta["tool_calls"].toArray()) {
if (!m_streaming)
break;
applyToolCallDelta(callValue.toObject());
}
}
}
void LlmClient::refreshModels() {
const QUrl url =
QUrl::fromUserInput(completionsPath(m_endpoint, "/models"));
if (!url.isValid() || url.host().isEmpty())
return;
auto* reply = m_manager.get(QNetworkRequest(url));
connect(reply, &QNetworkReply::finished, this, [this, reply]() {
const QNetworkReply::NetworkError error = reply->error();
const QByteArray data = reply->readAll();
reply->deleteLater();
if (error != QNetworkReply::NoError) {
qWarning() << "LlmClient: failed to fetch models:" << error;
return;
}
const QJsonArray dataArr =
QJsonDocument::fromJson(data).object()["data"].toArray();
QStringList models;
QSet<QString> seen;
for (const QJsonValue& value : dataArr) {
const QString id = value.toObject()["id"].toString();
if (!id.isEmpty() && !seen.contains(id)) {
seen.insert(id);
models.append(id);
}
}
if (models.isEmpty())
return;
m_availableModels = models;
Q_EMIT availableModelsChanged();
if (m_model.isEmpty()) {
m_model = models.first();
Q_EMIT modelChanged();
}
});
}
void LlmClient::setContextSize(int size) {
if (size <= 0 || m_contextSize == size)
return;
m_contextSize = size;
Q_EMIT contextSizeChanged();
}
void LlmClient::probeContextSize() {
// llama.cpp-specific endpoint; other servers fall back to 4096.
QString base = m_endpoint.trimmed();
while (base.endsWith('/'))
base.chop(1);
const QUrl url = QUrl::fromUserInput(base + "/props");
if (!url.isValid() || url.host().isEmpty())
return;
auto* reply = m_manager.get(QNetworkRequest(url));
connect(
reply,
&QNetworkReply::finished,
this,
[this, reply]() {
const QNetworkReply::NetworkError error = reply->error();
const QByteArray data = reply->readAll();
reply->deleteLater();
int size = 0;
if (error == QNetworkReply::NoError) {
const QJsonDocument doc = QJsonDocument::fromJson(data);
if (doc.isArray()) {
for (const auto& value : doc.array()) {
const QJsonObject slot = value.toObject();
if (slot.contains("n_ctx")) {
size = slot["n_ctx"].toInt(0);
if (size > 0)
break;
}
}
} else if (doc.isObject()) {
const QJsonObject obj = doc.object();
size = obj["n_ctx"].toInt(0);
if (size <= 0)
size = obj["default_generation_settings"].toObject()
["n_ctx"].toInt(0);
}
}
setContextSize(size > 0 ? size : 4096);
});
}
void LlmClient::updateTokenUsage(const QJsonObject& data) {
if (!m_active || m_contextSize <= 0)
return;
const QJsonObject usage = data["usage"].toObject();
if (usage.isEmpty())
return;
const double used =
usage.value("prompt_tokens").toDouble() +
usage.value("completion_tokens").toDouble();
if (used > 0)
m_active->setLastTokenCount(static_cast<int>(used));
}
void LlmClient::shortRequest(
const QString& tag,
const QString& systemPrompt,
@@ -340,229 +855,4 @@ void LlmClient::requestIcon(ChatSession* session, const QString& userText) {
});
}
void LlmClient::stop() {
if (!m_busy)
return;
if (m_reply)
m_reply->abort();
}
void LlmClient::endStream() {
if (!m_streaming)
return;
auto* generation = m_streaming;
auto* session = m_active;
m_streaming = nullptr;
m_active = nullptr;
generation->setStreaming(false);
if (generation->content().isEmpty() &&
generation->reasoning().isEmpty()) {
if (auto* message = qobject_cast<ChatMessage*>(generation->parent())) {
if (message->generationCount() <= 1) {
if (session)
session->removeMessage(message);
} else {
message->removeGeneration(generation);
}
}
}
setBusy(false);
setStreamingChatId(QString());
}
void LlmClient::finalize() {
if (!m_streaming)
return;
ChatSession* session = m_active;
endStream();
if (session && m_pendingClear == session) {
session->clearMessages();
m_pendingClear.clear();
}
if (session)
session->persist();
}
void LlmClient::clearOnFinish(ChatSession* session) {
m_pendingClear = session;
}
void LlmClient::sessionRemoved(ChatSession* session) {
if (m_pendingClear == session)
m_pendingClear.clear();
if (m_active == session) {
stop();
endStream();
}
}
void LlmClient::fail(const QString& message) {
qWarning() << "LlmClient:" << message;
if (m_streaming) {
ChatSession* session = m_active;
endStream();
if (session && m_pendingClear == session) {
session->clearMessages();
m_pendingClear.clear();
}
if (session)
session->persist();
}
Q_EMIT errorOccurred(message);
}
void LlmClient::drainBuffer() {
while (true) {
const qsizetype newline = m_buffer.indexOf('\n');
if (newline < 0)
break;
const QByteArray line = m_buffer.left(newline).trimmed();
m_buffer.remove(0, newline + 1);
handleLine(line);
}
}
void LlmClient::handleLine(const QByteArray& line) {
if (!m_streaming || line.isEmpty() || !line.startsWith("data:"))
return;
const QByteArray data = line.mid(5).trimmed();
if (data == "[DONE]") {
finalize();
return;
}
const QJsonDocument doc = QJsonDocument::fromJson(data);
if (!doc.isObject())
return;
const QJsonObject obj = doc.object();
updateTokenUsage(obj);
if (obj.contains("error")) {
const QJsonObject error = obj["error"].toObject();
const QString message = error["message"].toString();
fail(message.isEmpty()
? QStringLiteral("LLM server returned an error")
: message);
return;
}
for (const QJsonValue& choiceValue : obj["choices"].toArray()) {
if (!m_streaming)
continue;
const QJsonObject delta =
choiceValue.toObject()["delta"].toObject();
m_streaming->appendContent(delta["content"].toString());
QString reasoning = delta["reasoning_content"].toString();
if (reasoning.isEmpty())
reasoning = delta["reasoning"].toString();
m_streaming->appendReasoning(reasoning);
}
}
void LlmClient::refreshModels() {
const QUrl url =
QUrl::fromUserInput(completionsPath(m_endpoint, "/models"));
if (!url.isValid() || url.host().isEmpty())
return;
auto* reply = m_manager.get(QNetworkRequest(url));
connect(reply, &QNetworkReply::finished, this, [this, reply]() {
const QNetworkReply::NetworkError error = reply->error();
const QByteArray data = reply->readAll();
reply->deleteLater();
if (error != QNetworkReply::NoError) {
qWarning() << "LlmClient: failed to fetch models:" << error;
return;
}
const QJsonArray dataArr =
QJsonDocument::fromJson(data).object()["data"].toArray();
QStringList models;
QSet<QString> seen;
for (const QJsonValue& value : dataArr) {
const QString id = value.toObject()["id"].toString();
if (!id.isEmpty() && !seen.contains(id)) {
seen.insert(id);
models.append(id);
}
}
if (models.isEmpty())
return;
m_availableModels = models;
Q_EMIT availableModelsChanged();
if (m_model.isEmpty()) {
m_model = models.first();
Q_EMIT modelChanged();
}
});
}
void LlmClient::setContextSize(int size) {
if (size <= 0 || m_contextSize == size)
return;
m_contextSize = size;
Q_EMIT contextSizeChanged();
}
void LlmClient::probeContextSize() {
// llama.cpp-specific endpoint; other servers fall back to 4096.
QString base = m_endpoint.trimmed();
while (base.endsWith('/'))
base.chop(1);
const QUrl url = QUrl::fromUserInput(base + "/props");
if (!url.isValid() || url.host().isEmpty())
return;
auto* reply = m_manager.get(QNetworkRequest(url));
connect(
reply,
&QNetworkReply::finished,
this,
[this, reply]() {
const QNetworkReply::NetworkError error = reply->error();
const QByteArray data = reply->readAll();
reply->deleteLater();
int size = 0;
if (error == QNetworkReply::NoError) {
const QJsonDocument doc = QJsonDocument::fromJson(data);
if (doc.isArray()) {
for (const auto& value : doc.array()) {
const QJsonObject slot = value.toObject();
if (slot.contains("n_ctx")) {
size = slot["n_ctx"].toInt(0);
if (size > 0)
break;
}
}
} else if (doc.isObject()) {
const QJsonObject obj = doc.object();
size = obj["n_ctx"].toInt(0);
if (size <= 0)
size = obj["default_generation_settings"].toObject()
["n_ctx"].toInt(0);
}
}
setContextSize(size > 0 ? size : 4096);
});
}
void LlmClient::updateTokenUsage(const QJsonObject& data) {
if (!m_active || m_contextSize <= 0)
return;
const QJsonObject usage = data["usage"].toObject();
if (usage.isEmpty())
return;
const double used =
usage.value("prompt_tokens").toDouble() +
usage.value("completion_tokens").toDouble();
if (used > 0)
m_active->setLastTokenCount(static_cast<int>(used));
}
} // namespace ZShell::llm