add markdown parsing + latex + tree-sitter highlighting for codeblocks in llm responses
This commit is contained in:
+540
-250
@@ -1,7 +1,9 @@
|
||||
#include "llmclient.hpp"
|
||||
|
||||
#include "generation.hpp"
|
||||
#include "message.hpp"
|
||||
#include "messagemodel.hpp"
|
||||
#include "segment.hpp"
|
||||
#include "session.hpp"
|
||||
|
||||
#include <QDebug>
|
||||
@@ -62,12 +64,19 @@ void LlmClient::setStreamingChatId(const QString& id) {
|
||||
}
|
||||
|
||||
LlmClient::LlmClient(QObject* parent) : QObject(parent) {
|
||||
m_tools = new ToolRegistry(this);
|
||||
connect(
|
||||
m_tools,
|
||||
&ToolRegistry::enabledChanged,
|
||||
this,
|
||||
&LlmClient::toolsEnabledChanged);
|
||||
probeContextSize();
|
||||
}
|
||||
|
||||
LlmClient::~LlmClient() {
|
||||
if (m_reply)
|
||||
m_reply->abort();
|
||||
m_tools->cancelAll();
|
||||
endStream();
|
||||
}
|
||||
|
||||
@@ -111,10 +120,6 @@ void LlmClient::startGeneration(
|
||||
return;
|
||||
}
|
||||
|
||||
QNetworkRequest request(url);
|
||||
request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
|
||||
request.setRawHeader("Accept", "text/event-stream");
|
||||
|
||||
const auto* model = session->messagesModel();
|
||||
const int targetRow =
|
||||
model->rowOf(qobject_cast<ChatMessage*>(target->parent()));
|
||||
@@ -124,25 +129,50 @@ void LlmClient::startGeneration(
|
||||
return;
|
||||
}
|
||||
|
||||
// Context: every message older than the target, oldest first.
|
||||
QJsonArray messages;
|
||||
for (int row = model->rowCount() - 1; row > targetRow; --row) {
|
||||
const auto* message = model->at(row);
|
||||
const auto* generation = message->activeGeneration();
|
||||
if (!generation)
|
||||
continue;
|
||||
QJsonObject messageObj;
|
||||
messageObj[QStringLiteral("role")] =
|
||||
message->role() == ChatMessage::Role::User
|
||||
? QStringLiteral("user")
|
||||
: QStringLiteral("assistant");
|
||||
messageObj[QStringLiteral("content")] = generation->content();
|
||||
if (!generation->reasoning().isEmpty())
|
||||
messageObj[QStringLiteral("reasoning_content")] =
|
||||
generation->reasoning();
|
||||
messages.append(messageObj);
|
||||
m_transcript = QJsonArray();
|
||||
m_callBuilders.clear();
|
||||
m_callResults.clear();
|
||||
m_finishReason.clear();
|
||||
m_round = 0;
|
||||
m_contentMark = 0;
|
||||
m_reasoningMark = 0;
|
||||
|
||||
sendRound();
|
||||
}
|
||||
|
||||
void LlmClient::sendRound() {
|
||||
if (!m_active || !m_streaming)
|
||||
return;
|
||||
m_finishReason.clear();
|
||||
m_callBuilders.clear();
|
||||
m_callResults.clear();
|
||||
m_roundDone = false;
|
||||
m_toolPhase = false;
|
||||
m_pendingCalls = 0;
|
||||
m_buffer.clear();
|
||||
|
||||
const QUrl url =
|
||||
QUrl::fromUserInput(completionsPath(m_endpoint, "/chat/completions"));
|
||||
if (!url.isValid() || url.host().isEmpty()) {
|
||||
fail(QStringLiteral("Invalid LLM endpoint: %1").arg(m_endpoint));
|
||||
return;
|
||||
}
|
||||
|
||||
const auto* model = m_active->messagesModel();
|
||||
const int targetRow =
|
||||
model->rowOf(qobject_cast<ChatMessage*>(m_streaming->parent()));
|
||||
if (targetRow < 0) {
|
||||
fail(QStringLiteral("Internal error: generation target is not in the "
|
||||
"session"));
|
||||
return;
|
||||
}
|
||||
|
||||
// Context: every message older than the target, oldest first, plus
|
||||
// the tool exchanges of the current turn so far.
|
||||
QJsonArray messages = buildContextMessages(m_active, targetRow);
|
||||
for (const QJsonValue& value : m_transcript)
|
||||
messages.append(value);
|
||||
|
||||
QJsonObject body;
|
||||
QJsonObject streamOptions;
|
||||
streamOptions[QStringLiteral("include_usage")] = true;
|
||||
@@ -152,8 +182,14 @@ void LlmClient::startGeneration(
|
||||
body[QStringLiteral("temperature")] = m_temperature;
|
||||
if (!m_model.isEmpty())
|
||||
body[QStringLiteral("model")] = m_model;
|
||||
const QJsonArray toolSpecs = m_tools->specifications();
|
||||
if (!toolSpecs.isEmpty())
|
||||
body[QStringLiteral("tools")] = toolSpecs;
|
||||
|
||||
QNetworkRequest request(url);
|
||||
request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
|
||||
request.setRawHeader("Accept", "text/event-stream");
|
||||
|
||||
m_buffer.clear();
|
||||
m_reply = m_manager.post(request, QJsonDocument(body).toJson());
|
||||
|
||||
connect(m_reply, &QNetworkReply::readyRead, this, [this]() {
|
||||
@@ -178,14 +214,493 @@ void LlmClient::startGeneration(
|
||||
if (!m_streaming)
|
||||
return;
|
||||
|
||||
if (error == QNetworkReply::NoError ||
|
||||
error == QNetworkReply::OperationCanceledError)
|
||||
finalize();
|
||||
if (error == QNetworkReply::NoError)
|
||||
roundFinished();
|
||||
else if (error == QNetworkReply::OperationCanceledError)
|
||||
finishTurn();
|
||||
else
|
||||
fail(serverErrorMessage(responseBody, errorString));
|
||||
});
|
||||
}
|
||||
|
||||
QJsonArray LlmClient::buildContextMessages(
|
||||
ChatSession* session, int stopBeforeRow) const {
|
||||
const auto* model = session->messagesModel();
|
||||
QJsonArray messages;
|
||||
for (int row = model->rowCount() - 1; row > stopBeforeRow; --row) {
|
||||
const auto* message = model->at(row);
|
||||
const auto* generation = message->activeGeneration();
|
||||
if (!generation)
|
||||
continue;
|
||||
|
||||
if (message->role() == ChatMessage::Role::User) {
|
||||
QJsonObject user;
|
||||
user[QStringLiteral("role")] = QStringLiteral("user");
|
||||
user[QStringLiteral("content")] = generation->content();
|
||||
messages.append(user);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Assistant message: replay its tool calls (and their results)
|
||||
// so the model keeps the full history of the turn.
|
||||
QList<const LlmSegment*> toolSegments;
|
||||
for (const auto* segment : generation->segments())
|
||||
if (segment->type() == LlmSegment::Type::ToolCall)
|
||||
toolSegments.append(segment);
|
||||
|
||||
QJsonObject assistant;
|
||||
assistant[QStringLiteral("role")] = QStringLiteral("assistant");
|
||||
if (toolSegments.isEmpty())
|
||||
assistant[QStringLiteral("content")] = generation->content();
|
||||
else if (!generation->content().isEmpty())
|
||||
assistant[QStringLiteral("content")] = generation->content();
|
||||
if (!generation->reasoning().isEmpty())
|
||||
assistant[QStringLiteral("reasoning_content")] =
|
||||
generation->reasoning();
|
||||
|
||||
if (toolSegments.isEmpty()) {
|
||||
messages.append(assistant);
|
||||
continue;
|
||||
}
|
||||
QJsonArray calls;
|
||||
for (const auto* segment : toolSegments) {
|
||||
QJsonObject function;
|
||||
function[QStringLiteral("name")] = segment->name();
|
||||
function[QStringLiteral("arguments")] = segment->arguments();
|
||||
QJsonObject call;
|
||||
call[QStringLiteral("id")] = segment->toolCallId();
|
||||
call[QStringLiteral("type")] = QStringLiteral("function");
|
||||
call[QStringLiteral("function")] = function;
|
||||
calls.append(call);
|
||||
}
|
||||
assistant[QStringLiteral("tool_calls")] = calls;
|
||||
messages.append(assistant);
|
||||
for (const auto* segment : toolSegments) {
|
||||
QJsonObject toolMessage;
|
||||
toolMessage[QStringLiteral("role")] = QStringLiteral("tool");
|
||||
toolMessage[QStringLiteral("tool_call_id")] =
|
||||
segment->toolCallId();
|
||||
toolMessage[QStringLiteral("content")] = segment->result();
|
||||
messages.append(toolMessage);
|
||||
}
|
||||
}
|
||||
return messages;
|
||||
}
|
||||
|
||||
void LlmClient::applyToolCallDelta(const QJsonObject& call) {
|
||||
if (!m_streaming)
|
||||
return;
|
||||
const int index = call[QStringLiteral("index")].toInt(-1);
|
||||
if (index < 0)
|
||||
return;
|
||||
while (m_callBuilders.size() <= index)
|
||||
m_callBuilders.append(ToolCallBuilder{});
|
||||
ToolCallBuilder& builder = m_callBuilders[index];
|
||||
builder.seen = true;
|
||||
const QString id = call[QStringLiteral("id")].toString();
|
||||
if (!id.isEmpty())
|
||||
builder.id = id;
|
||||
const QJsonObject function = call[QStringLiteral("function")].toObject();
|
||||
const QString name = function[QStringLiteral("name")].toString();
|
||||
if (!name.isEmpty())
|
||||
builder.name = name;
|
||||
const QString arguments =
|
||||
function[QStringLiteral("arguments")].toString();
|
||||
if (!arguments.isEmpty())
|
||||
builder.arguments += arguments;
|
||||
|
||||
if (!builder.segment) {
|
||||
// A new call: close the in-flight text segments and open a
|
||||
// running tool-call segment so the UI can track it live.
|
||||
m_streaming->closeOpenSegments();
|
||||
builder.segment =
|
||||
m_streaming->beginToolCall(builder.name, builder.id);
|
||||
}
|
||||
builder.segment->setName(builder.name);
|
||||
builder.segment->setToolCallId(builder.id);
|
||||
if (!arguments.isEmpty())
|
||||
builder.segment->appendArguments(arguments);
|
||||
}
|
||||
|
||||
void LlmClient::roundFinished() {
|
||||
// [DONE] and the reply's finished signal both funnel here; only the
|
||||
// first may act.
|
||||
if (m_roundDone || !m_streaming)
|
||||
return;
|
||||
m_roundDone = true;
|
||||
|
||||
bool hasCalls = false;
|
||||
for (const auto& builder : m_callBuilders)
|
||||
if (builder.seen)
|
||||
hasCalls = true;
|
||||
if (m_finishReason != QLatin1String("tool_calls") && !hasCalls) {
|
||||
finishTurn();
|
||||
return;
|
||||
}
|
||||
if (m_round >= kMaxToolRounds) {
|
||||
qWarning() << "LlmClient: tool round limit reached, ending turn";
|
||||
finishTurn();
|
||||
return;
|
||||
}
|
||||
|
||||
m_streaming->closeOpenSegments();
|
||||
|
||||
// Record the assistant's tool-call message in the transcript so the
|
||||
// next round (and the model) can see it.
|
||||
const QString content = m_streaming->content();
|
||||
const QString reasoning = m_streaming->reasoning();
|
||||
QJsonObject assistant;
|
||||
assistant[QStringLiteral("role")] = QStringLiteral("assistant");
|
||||
if (content.size() > m_contentMark) {
|
||||
assistant[QStringLiteral("content")] = content.mid(m_contentMark);
|
||||
m_contentMark = content.size();
|
||||
}
|
||||
if (reasoning.size() > m_reasoningMark) {
|
||||
assistant[QStringLiteral("reasoning_content")] =
|
||||
reasoning.mid(m_reasoningMark);
|
||||
m_reasoningMark = reasoning.size();
|
||||
}
|
||||
QJsonArray calls;
|
||||
for (const auto& builder : m_callBuilders) {
|
||||
if (!builder.seen)
|
||||
continue;
|
||||
QJsonObject function;
|
||||
function[QStringLiteral("name")] = builder.name;
|
||||
function[QStringLiteral("arguments")] = builder.arguments;
|
||||
QJsonObject call;
|
||||
call[QStringLiteral("id")] = builder.id;
|
||||
call[QStringLiteral("type")] = QStringLiteral("function");
|
||||
call[QStringLiteral("function")] = function;
|
||||
calls.append(call);
|
||||
}
|
||||
assistant[QStringLiteral("tool_calls")] = calls;
|
||||
m_transcript.append(assistant);
|
||||
m_round++;
|
||||
|
||||
executeAllCalls();
|
||||
}
|
||||
|
||||
void LlmClient::executeAllCalls() {
|
||||
m_toolPhase = true;
|
||||
m_pendingCalls = 0;
|
||||
m_callResults = QList<ToolCallResult>(m_callBuilders.size());
|
||||
|
||||
for (int i = 0; i < m_callBuilders.size(); ++i) {
|
||||
const auto& call = m_callBuilders.at(i);
|
||||
if (!call.seen)
|
||||
continue;
|
||||
|
||||
LlmTool* tool = m_tools->tool(call.name);
|
||||
QJsonObject args;
|
||||
QString errorText;
|
||||
if (!tool) {
|
||||
errorText = QStringLiteral("Error: unknown tool '%1'")
|
||||
.arg(call.name);
|
||||
} else if (!call.arguments.isEmpty()) {
|
||||
const QJsonDocument doc =
|
||||
QJsonDocument::fromJson(call.arguments.toUtf8());
|
||||
if (!doc.isObject()) {
|
||||
errorText =
|
||||
QStringLiteral("Error: tool arguments are not valid "
|
||||
"JSON: %1")
|
||||
.arg(call.arguments);
|
||||
} else {
|
||||
args = doc.object();
|
||||
}
|
||||
}
|
||||
if (!errorText.isEmpty()) {
|
||||
m_callResults[i] = { errorText, false };
|
||||
if (LlmSegment* segment = call.segment)
|
||||
segment->finishTool(errorText, false);
|
||||
continue;
|
||||
}
|
||||
|
||||
++m_pendingCalls;
|
||||
tool->execute(
|
||||
args,
|
||||
[this, i, call](const QJsonObject& result) {
|
||||
if (!m_streaming)
|
||||
return;
|
||||
const bool success =
|
||||
result.contains(QStringLiteral("output"));
|
||||
const QString content = success
|
||||
? result[QStringLiteral("output")].toString()
|
||||
: QStringLiteral("Error: ") +
|
||||
result[QStringLiteral("error")].toString();
|
||||
m_callResults[i] = { content, success };
|
||||
if (LlmSegment* segment = call.segment)
|
||||
segment->finishTool(content, success);
|
||||
if (--m_pendingCalls == 0)
|
||||
flushCallResults();
|
||||
});
|
||||
}
|
||||
if (m_pendingCalls == 0)
|
||||
flushCallResults();
|
||||
}
|
||||
|
||||
void LlmClient::flushCallResults() {
|
||||
if (!m_toolPhase)
|
||||
return;
|
||||
m_toolPhase = false;
|
||||
if (!m_streaming)
|
||||
return;
|
||||
for (int i = 0; i < m_callBuilders.size(); ++i) {
|
||||
const auto& call = m_callBuilders.at(i);
|
||||
if (!call.seen)
|
||||
continue;
|
||||
QJsonObject toolMessage;
|
||||
toolMessage[QStringLiteral("role")] = QStringLiteral("tool");
|
||||
toolMessage[QStringLiteral("tool_call_id")] = call.id;
|
||||
toolMessage[QStringLiteral("content")] =
|
||||
m_callResults.at(i).content;
|
||||
m_transcript.append(toolMessage);
|
||||
}
|
||||
sendRound();
|
||||
}
|
||||
|
||||
void LlmClient::stop() {
|
||||
if (!m_busy)
|
||||
return;
|
||||
if (m_toolPhase) {
|
||||
m_tools->cancelAll();
|
||||
if (m_streaming) {
|
||||
for (auto* segment : m_streaming->segments()) {
|
||||
if (segment->type() == LlmSegment::Type::ToolCall &&
|
||||
segment->running())
|
||||
segment->finishTool(
|
||||
QStringLiteral("Cancelled"), false);
|
||||
}
|
||||
}
|
||||
finishTurn();
|
||||
return;
|
||||
}
|
||||
if (m_reply)
|
||||
m_reply->abort();
|
||||
}
|
||||
|
||||
void LlmClient::endStream() {
|
||||
if (!m_streaming)
|
||||
return;
|
||||
auto* generation = m_streaming;
|
||||
auto* session = m_active;
|
||||
m_streaming = nullptr;
|
||||
m_active = nullptr;
|
||||
generation->setStreaming(false);
|
||||
if (generation->content().isEmpty() &&
|
||||
generation->reasoning().isEmpty() &&
|
||||
generation->toolCallCount() == 0) {
|
||||
if (auto* message = qobject_cast<ChatMessage*>(generation->parent())) {
|
||||
if (message->generationCount() <= 1) {
|
||||
if (session)
|
||||
session->removeMessage(message);
|
||||
} else {
|
||||
message->removeGeneration(generation);
|
||||
}
|
||||
}
|
||||
}
|
||||
setBusy(false);
|
||||
setStreamingChatId(QString());
|
||||
}
|
||||
|
||||
void LlmClient::finishTurn() {
|
||||
if (!m_streaming)
|
||||
return;
|
||||
ChatSession* session = m_active;
|
||||
endStream();
|
||||
if (session && m_pendingClear == session) {
|
||||
session->clearMessages();
|
||||
m_pendingClear.clear();
|
||||
}
|
||||
if (session)
|
||||
session->persist();
|
||||
}
|
||||
|
||||
void LlmClient::clearOnFinish(ChatSession* session) {
|
||||
m_pendingClear = session;
|
||||
}
|
||||
|
||||
void LlmClient::sessionRemoved(ChatSession* session) {
|
||||
if (m_pendingClear == session)
|
||||
m_pendingClear.clear();
|
||||
if (m_active == session) {
|
||||
stop();
|
||||
endStream();
|
||||
}
|
||||
}
|
||||
|
||||
void LlmClient::fail(const QString& message) {
|
||||
qWarning() << "LlmClient:" << message;
|
||||
finishTurn();
|
||||
Q_EMIT errorOccurred(message);
|
||||
}
|
||||
|
||||
void LlmClient::drainBuffer() {
|
||||
while (true) {
|
||||
const qsizetype newline = m_buffer.indexOf('\n');
|
||||
if (newline < 0)
|
||||
break;
|
||||
const QByteArray line = m_buffer.left(newline).trimmed();
|
||||
m_buffer.remove(0, newline + 1);
|
||||
handleLine(line);
|
||||
}
|
||||
}
|
||||
|
||||
void LlmClient::handleLine(const QByteArray& line) {
|
||||
if (!m_streaming || line.isEmpty() || !line.startsWith("data:"))
|
||||
return;
|
||||
|
||||
const QByteArray data = line.mid(5).trimmed();
|
||||
if (data == "[DONE]") {
|
||||
roundFinished();
|
||||
return;
|
||||
}
|
||||
|
||||
const QJsonDocument doc = QJsonDocument::fromJson(data);
|
||||
if (!doc.isObject())
|
||||
return;
|
||||
const QJsonObject obj = doc.object();
|
||||
updateTokenUsage(obj);
|
||||
|
||||
if (obj.contains("error")) {
|
||||
const QJsonObject error = obj["error"].toObject();
|
||||
const QString message = error["message"].toString();
|
||||
fail(message.isEmpty()
|
||||
? QStringLiteral("LLM server returned an error")
|
||||
: message);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const QJsonValue& choiceValue : obj["choices"].toArray()) {
|
||||
if (!m_streaming)
|
||||
continue;
|
||||
const QJsonObject choice = choiceValue.toObject();
|
||||
const QJsonObject delta = choice["delta"].toObject();
|
||||
|
||||
const QString finishReason =
|
||||
choice[QStringLiteral("finish_reason")].toString();
|
||||
if (!finishReason.isEmpty())
|
||||
m_finishReason = finishReason;
|
||||
|
||||
m_streaming->appendContent(delta["content"].toString());
|
||||
QString reasoning =
|
||||
delta["reasoning_content"].toString();
|
||||
if (reasoning.isEmpty())
|
||||
reasoning = delta["reasoning"].toString();
|
||||
m_streaming->appendReasoning(reasoning);
|
||||
|
||||
for (const QJsonValue& callValue :
|
||||
delta["tool_calls"].toArray()) {
|
||||
if (!m_streaming)
|
||||
break;
|
||||
applyToolCallDelta(callValue.toObject());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void LlmClient::refreshModels() {
|
||||
const QUrl url =
|
||||
QUrl::fromUserInput(completionsPath(m_endpoint, "/models"));
|
||||
if (!url.isValid() || url.host().isEmpty())
|
||||
return;
|
||||
|
||||
auto* reply = m_manager.get(QNetworkRequest(url));
|
||||
connect(reply, &QNetworkReply::finished, this, [this, reply]() {
|
||||
const QNetworkReply::NetworkError error = reply->error();
|
||||
const QByteArray data = reply->readAll();
|
||||
reply->deleteLater();
|
||||
|
||||
if (error != QNetworkReply::NoError) {
|
||||
qWarning() << "LlmClient: failed to fetch models:" << error;
|
||||
return;
|
||||
}
|
||||
const QJsonArray dataArr =
|
||||
QJsonDocument::fromJson(data).object()["data"].toArray();
|
||||
|
||||
QStringList models;
|
||||
QSet<QString> seen;
|
||||
for (const QJsonValue& value : dataArr) {
|
||||
const QString id = value.toObject()["id"].toString();
|
||||
if (!id.isEmpty() && !seen.contains(id)) {
|
||||
seen.insert(id);
|
||||
models.append(id);
|
||||
}
|
||||
}
|
||||
if (models.isEmpty())
|
||||
return;
|
||||
|
||||
m_availableModels = models;
|
||||
Q_EMIT availableModelsChanged();
|
||||
|
||||
if (m_model.isEmpty()) {
|
||||
m_model = models.first();
|
||||
Q_EMIT modelChanged();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void LlmClient::setContextSize(int size) {
|
||||
if (size <= 0 || m_contextSize == size)
|
||||
return;
|
||||
m_contextSize = size;
|
||||
Q_EMIT contextSizeChanged();
|
||||
}
|
||||
|
||||
void LlmClient::probeContextSize() {
|
||||
// llama.cpp-specific endpoint; other servers fall back to 4096.
|
||||
QString base = m_endpoint.trimmed();
|
||||
while (base.endsWith('/'))
|
||||
base.chop(1);
|
||||
const QUrl url = QUrl::fromUserInput(base + "/props");
|
||||
if (!url.isValid() || url.host().isEmpty())
|
||||
return;
|
||||
|
||||
auto* reply = m_manager.get(QNetworkRequest(url));
|
||||
connect(
|
||||
reply,
|
||||
&QNetworkReply::finished,
|
||||
this,
|
||||
[this, reply]() {
|
||||
const QNetworkReply::NetworkError error = reply->error();
|
||||
const QByteArray data = reply->readAll();
|
||||
reply->deleteLater();
|
||||
|
||||
int size = 0;
|
||||
if (error == QNetworkReply::NoError) {
|
||||
const QJsonDocument doc = QJsonDocument::fromJson(data);
|
||||
if (doc.isArray()) {
|
||||
for (const auto& value : doc.array()) {
|
||||
const QJsonObject slot = value.toObject();
|
||||
if (slot.contains("n_ctx")) {
|
||||
size = slot["n_ctx"].toInt(0);
|
||||
if (size > 0)
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (doc.isObject()) {
|
||||
const QJsonObject obj = doc.object();
|
||||
size = obj["n_ctx"].toInt(0);
|
||||
if (size <= 0)
|
||||
size = obj["default_generation_settings"].toObject()
|
||||
["n_ctx"].toInt(0);
|
||||
}
|
||||
}
|
||||
setContextSize(size > 0 ? size : 4096);
|
||||
});
|
||||
}
|
||||
|
||||
void LlmClient::updateTokenUsage(const QJsonObject& data) {
|
||||
if (!m_active || m_contextSize <= 0)
|
||||
return;
|
||||
const QJsonObject usage = data["usage"].toObject();
|
||||
if (usage.isEmpty())
|
||||
return;
|
||||
const double used =
|
||||
usage.value("prompt_tokens").toDouble() +
|
||||
usage.value("completion_tokens").toDouble();
|
||||
if (used > 0)
|
||||
m_active->setLastTokenCount(static_cast<int>(used));
|
||||
}
|
||||
|
||||
void LlmClient::shortRequest(
|
||||
const QString& tag,
|
||||
const QString& systemPrompt,
|
||||
@@ -340,229 +855,4 @@ void LlmClient::requestIcon(ChatSession* session, const QString& userText) {
|
||||
});
|
||||
}
|
||||
|
||||
void LlmClient::stop() {
|
||||
if (!m_busy)
|
||||
return;
|
||||
if (m_reply)
|
||||
m_reply->abort();
|
||||
}
|
||||
|
||||
void LlmClient::endStream() {
|
||||
if (!m_streaming)
|
||||
return;
|
||||
auto* generation = m_streaming;
|
||||
auto* session = m_active;
|
||||
m_streaming = nullptr;
|
||||
m_active = nullptr;
|
||||
generation->setStreaming(false);
|
||||
if (generation->content().isEmpty() &&
|
||||
generation->reasoning().isEmpty()) {
|
||||
if (auto* message = qobject_cast<ChatMessage*>(generation->parent())) {
|
||||
if (message->generationCount() <= 1) {
|
||||
if (session)
|
||||
session->removeMessage(message);
|
||||
} else {
|
||||
message->removeGeneration(generation);
|
||||
}
|
||||
}
|
||||
}
|
||||
setBusy(false);
|
||||
setStreamingChatId(QString());
|
||||
}
|
||||
|
||||
void LlmClient::finalize() {
|
||||
if (!m_streaming)
|
||||
return;
|
||||
ChatSession* session = m_active;
|
||||
endStream();
|
||||
if (session && m_pendingClear == session) {
|
||||
session->clearMessages();
|
||||
m_pendingClear.clear();
|
||||
}
|
||||
if (session)
|
||||
session->persist();
|
||||
}
|
||||
|
||||
void LlmClient::clearOnFinish(ChatSession* session) {
|
||||
m_pendingClear = session;
|
||||
}
|
||||
|
||||
void LlmClient::sessionRemoved(ChatSession* session) {
|
||||
if (m_pendingClear == session)
|
||||
m_pendingClear.clear();
|
||||
if (m_active == session) {
|
||||
stop();
|
||||
endStream();
|
||||
}
|
||||
}
|
||||
|
||||
void LlmClient::fail(const QString& message) {
|
||||
qWarning() << "LlmClient:" << message;
|
||||
|
||||
if (m_streaming) {
|
||||
ChatSession* session = m_active;
|
||||
endStream();
|
||||
if (session && m_pendingClear == session) {
|
||||
session->clearMessages();
|
||||
m_pendingClear.clear();
|
||||
}
|
||||
if (session)
|
||||
session->persist();
|
||||
}
|
||||
Q_EMIT errorOccurred(message);
|
||||
}
|
||||
|
||||
void LlmClient::drainBuffer() {
|
||||
while (true) {
|
||||
const qsizetype newline = m_buffer.indexOf('\n');
|
||||
if (newline < 0)
|
||||
break;
|
||||
const QByteArray line = m_buffer.left(newline).trimmed();
|
||||
m_buffer.remove(0, newline + 1);
|
||||
handleLine(line);
|
||||
}
|
||||
}
|
||||
|
||||
void LlmClient::handleLine(const QByteArray& line) {
|
||||
if (!m_streaming || line.isEmpty() || !line.startsWith("data:"))
|
||||
return;
|
||||
|
||||
const QByteArray data = line.mid(5).trimmed();
|
||||
if (data == "[DONE]") {
|
||||
finalize();
|
||||
return;
|
||||
}
|
||||
|
||||
const QJsonDocument doc = QJsonDocument::fromJson(data);
|
||||
if (!doc.isObject())
|
||||
return;
|
||||
const QJsonObject obj = doc.object();
|
||||
updateTokenUsage(obj);
|
||||
|
||||
if (obj.contains("error")) {
|
||||
const QJsonObject error = obj["error"].toObject();
|
||||
const QString message = error["message"].toString();
|
||||
fail(message.isEmpty()
|
||||
? QStringLiteral("LLM server returned an error")
|
||||
: message);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const QJsonValue& choiceValue : obj["choices"].toArray()) {
|
||||
if (!m_streaming)
|
||||
continue;
|
||||
const QJsonObject delta =
|
||||
choiceValue.toObject()["delta"].toObject();
|
||||
m_streaming->appendContent(delta["content"].toString());
|
||||
QString reasoning = delta["reasoning_content"].toString();
|
||||
if (reasoning.isEmpty())
|
||||
reasoning = delta["reasoning"].toString();
|
||||
m_streaming->appendReasoning(reasoning);
|
||||
}
|
||||
}
|
||||
|
||||
void LlmClient::refreshModels() {
|
||||
const QUrl url =
|
||||
QUrl::fromUserInput(completionsPath(m_endpoint, "/models"));
|
||||
if (!url.isValid() || url.host().isEmpty())
|
||||
return;
|
||||
|
||||
auto* reply = m_manager.get(QNetworkRequest(url));
|
||||
connect(reply, &QNetworkReply::finished, this, [this, reply]() {
|
||||
const QNetworkReply::NetworkError error = reply->error();
|
||||
const QByteArray data = reply->readAll();
|
||||
reply->deleteLater();
|
||||
|
||||
if (error != QNetworkReply::NoError) {
|
||||
qWarning() << "LlmClient: failed to fetch models:" << error;
|
||||
return;
|
||||
}
|
||||
const QJsonArray dataArr =
|
||||
QJsonDocument::fromJson(data).object()["data"].toArray();
|
||||
|
||||
QStringList models;
|
||||
QSet<QString> seen;
|
||||
for (const QJsonValue& value : dataArr) {
|
||||
const QString id = value.toObject()["id"].toString();
|
||||
if (!id.isEmpty() && !seen.contains(id)) {
|
||||
seen.insert(id);
|
||||
models.append(id);
|
||||
}
|
||||
}
|
||||
if (models.isEmpty())
|
||||
return;
|
||||
|
||||
m_availableModels = models;
|
||||
Q_EMIT availableModelsChanged();
|
||||
|
||||
if (m_model.isEmpty()) {
|
||||
m_model = models.first();
|
||||
Q_EMIT modelChanged();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void LlmClient::setContextSize(int size) {
|
||||
if (size <= 0 || m_contextSize == size)
|
||||
return;
|
||||
m_contextSize = size;
|
||||
Q_EMIT contextSizeChanged();
|
||||
}
|
||||
|
||||
void LlmClient::probeContextSize() {
|
||||
// llama.cpp-specific endpoint; other servers fall back to 4096.
|
||||
QString base = m_endpoint.trimmed();
|
||||
while (base.endsWith('/'))
|
||||
base.chop(1);
|
||||
const QUrl url = QUrl::fromUserInput(base + "/props");
|
||||
if (!url.isValid() || url.host().isEmpty())
|
||||
return;
|
||||
|
||||
auto* reply = m_manager.get(QNetworkRequest(url));
|
||||
connect(
|
||||
reply,
|
||||
&QNetworkReply::finished,
|
||||
this,
|
||||
[this, reply]() {
|
||||
const QNetworkReply::NetworkError error = reply->error();
|
||||
const QByteArray data = reply->readAll();
|
||||
reply->deleteLater();
|
||||
|
||||
int size = 0;
|
||||
if (error == QNetworkReply::NoError) {
|
||||
const QJsonDocument doc = QJsonDocument::fromJson(data);
|
||||
if (doc.isArray()) {
|
||||
for (const auto& value : doc.array()) {
|
||||
const QJsonObject slot = value.toObject();
|
||||
if (slot.contains("n_ctx")) {
|
||||
size = slot["n_ctx"].toInt(0);
|
||||
if (size > 0)
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (doc.isObject()) {
|
||||
const QJsonObject obj = doc.object();
|
||||
size = obj["n_ctx"].toInt(0);
|
||||
if (size <= 0)
|
||||
size = obj["default_generation_settings"].toObject()
|
||||
["n_ctx"].toInt(0);
|
||||
}
|
||||
}
|
||||
setContextSize(size > 0 ? size : 4096);
|
||||
});
|
||||
}
|
||||
|
||||
void LlmClient::updateTokenUsage(const QJsonObject& data) {
|
||||
if (!m_active || m_contextSize <= 0)
|
||||
return;
|
||||
const QJsonObject usage = data["usage"].toObject();
|
||||
if (usage.isEmpty())
|
||||
return;
|
||||
const double used =
|
||||
usage.value("prompt_tokens").toDouble() +
|
||||
usage.value("completion_tokens").toDouble();
|
||||
if (used > 0)
|
||||
m_active->setLastTokenCount(static_cast<int>(used));
|
||||
}
|
||||
|
||||
} // namespace ZShell::llm
|
||||
|
||||
Reference in New Issue
Block a user