#include "llmclient.hpp" #include "attachment.hpp" #include "filetool.hpp" #include "generation.hpp" #include "message.hpp" #include "messagemodel.hpp" #include "prompts.hpp" #include "session.hpp" #include "shelltool.hpp" #include "webfetchtool.hpp" #include "workspace.hpp" #include "config/llm.hpp" #include "config/rootnodes.hpp" #include #include #include #include #include #include #include #include #include #include #include namespace ZShell::llm { namespace { constexpr int kRefreshIntervalMs = 60'000; constexpr int kProbeTimeoutMs = 2000; int contextSizeFromErrorMessage(const QString& message) { static const QRegularExpression re( QLatin1String("available context size \\((\\d+)")); const auto match = re.match(message); if (match.hasMatch()) return match.captured(1).toInt(); return 0; } } // namespace QString LlmClient::completionsPath( const QString& endpoint, const QString& subpath) { QString base = endpoint.trimmed(); while (base.endsWith('/')) base.chop(1); if (!base.endsWith("/v1")) base += "/v1"; return base + subpath; } QString LlmClient::jsonErrorMessage(const QByteArray& body) { const QJsonDocument doc = QJsonDocument::fromJson(body); if (!doc.isObject()) return QString(); const QJsonValue errorValue = doc.object().value("error"); if (errorValue.isObject()) return errorValue.toObject().value("message").toString(); if (errorValue.isString()) return errorValue.toString(); return QString(); } QString LlmClient::serverErrorMessage( const QByteArray& body, const QString& fallback) { const QString message = jsonErrorMessage(body); if (!message.isEmpty()) return message; return fallback.isEmpty() ? QStringLiteral("Request to LLM server failed") : fallback; } void LlmClient::setServerError(const QString& value) { if (m_serverError == value) return; m_serverError = value; Q_EMIT serverErrorChanged(); } void LlmClient::setBusy(bool value) { if (m_busy == value) return; m_busy = value; Q_EMIT busyChanged(); } void LlmClient::setModel(const QString& value) { if (m_model == value) return; m_model = value; Q_EMIT modelChanged(); if (m_model.isEmpty()) refreshModels(); } void LlmClient::setContextSize(int size) { if (size <= 0 || m_tools->contextSize() == size) return; m_tools->setContextSize(size); Q_EMIT contextSizeChanged(); } void LlmClient::setToolsEnabled(bool value) { if (m_tools->enabled() == value) return; m_tools->setEnabled(value); Q_EMIT toolsEnabledChanged(); if (value) refreshAllStates(); } void LlmClient::setSandboxEnabled(bool value) { if (m_tools->sandboxEnabled() == value) return; m_tools->setSandboxEnabled(value); Q_EMIT sandboxEnabledChanged(); } LlmClient::LlmClient(QObject* parent) : QObject(parent) { m_tools = new ToolRegistry(this); m_store = new ChatStore(this, this); m_tools->registerTool(new WebFetchTool(m_tools, m_tools)); m_tools->registerTool(new FileReadTool(m_tools, m_tools)); m_tools->registerTool(new ShellTool(m_tools, m_tools)); auto* llm = config::ConfigSingleton::instance()->llm(); if (llm) { m_endpoint = llm->endpoint(); m_model = llm->model(); m_temperature = llm->temperature(); m_configEnabled = llm->enabled(); m_tools->setEnabled(llm->tools()); m_tools->setSandboxEnabled(llm->sandbox()); connect(llm, &config::Llm::endpointChanged, this, [this, llm]() { m_endpoint = llm->endpoint(); refreshAllStates(); }); connect(llm, &config::Llm::modelChanged, this, [this, llm]() { setModel(llm->model()); }); connect(llm, &config::Llm::temperatureChanged, this, [this, llm]() { m_temperature = llm->temperature(); }); connect(llm, &config::Llm::toolsChanged, this, [this, llm]() { setToolsEnabled(llm->tools()); }); connect(llm, &config::Llm::sandboxChanged, this, [this, llm]() { setSandboxEnabled(llm->sandbox()); }); connect(llm, &config::Llm::enabledChanged, this, [this, llm]() { m_configEnabled = llm->enabled(); if (m_configEnabled) { m_refreshTimer.setInterval(kRefreshIntervalMs); m_refreshTimer.start(); refreshAllStates(); } else { m_refreshTimer.stop(); } }); } connect( m_store, &ChatStore::sessionRemoved, this, [this](ChatSession* session) { if (m_turn && m_turn->session() == session) { stop(); endStream(); } }); m_refreshTimer.setInterval(kRefreshIntervalMs); connect( &m_refreshTimer, &QTimer::timeout, this, &LlmClient::refreshAllStates); if (m_configEnabled) { m_refreshTimer.start(); refreshAllStates(); } } LlmClient::~LlmClient() { if (m_reply) { QNetworkReply* reply = m_reply; m_reply = nullptr; reply->abort(); } m_tools->cancelAll(); } void LlmClient::selectModel(const QString& id) { if (id.isEmpty()) return; if (auto* config = config::ConfigSingleton::instance()) config->llm()->set_model(id); setModel(id); } ToolContext LlmClient::toolContext() const { ToolContext context; if (m_turn) { context.workspace = m_turn->session()->workspace()->path(); context.outputBudgetChars = m_turn->outputBudgetChars(); } return context; } void LlmClient::startGeneration(ChatSession* session, ChatGeneration* target) { if (m_busy || !session || !target || !m_configEnabled) return; m_turn = std::make_unique(session, target, *m_tools); if (m_turn->targetRow() < 0) { fail(QStringLiteral( "Internal error: generation target is not in the session")); return; } target->setStreaming(true); setBusy(true); probeContextSize([this, session, target]() { if (!m_turn || m_turn->session() != session || m_turn->generation() != target) return; refreshModels(); sendRound(); }); } void LlmClient::sendRound() { if (!m_turn) return; probeContextSize(); const QUrl url = QUrl::fromUserInput(completionsPath(m_endpoint, "/chat/completions")); if (!url.isValid() || url.host().isEmpty()) { fail(QStringLiteral("Invalid LLM endpoint: %1").arg(m_endpoint)); return; } Turn* turn = m_turn.get(); QJsonArray messages = turn->contextMessages(); QJsonObject body; QJsonObject streamOptions; streamOptions[QStringLiteral("include_usage")] = true; body[QStringLiteral("stream_options")] = streamOptions; body[QStringLiteral("messages")] = messages; body[QStringLiteral("stream")] = true; body[QStringLiteral("temperature")] = m_temperature; if (!m_model.isEmpty()) body[QStringLiteral("model")] = m_model; if (m_tools->enabled()) { const QJsonArray toolSpecs = m_tools->specifications(toolContext()); if (!toolSpecs.isEmpty()) body[QStringLiteral("tools")] = toolSpecs; } QNetworkRequest request(url); request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json"); request.setRawHeader("Accept", "text/event-stream"); QNetworkReply* reply = m_manager.post(request, QJsonDocument(body).toJson()); if (!reply) { fail(QStringLiteral("Failed to start request to LLM server")); return; } m_reply = reply; connect(reply, &QNetworkReply::readyRead, this, [this, reply, turn]() { if (m_reply != reply || m_turn.get() != turn) return; m_buffer.append(reply->readAll()); drainBuffer(); }); connect(reply, &QNetworkReply::finished, this, [this, reply, turn]() { if (m_reply != reply) { reply->deleteLater(); return; } m_reply = nullptr; const QNetworkReply::NetworkError error = reply->error(); const QString errorString = reply->errorString(); const QByteArray responseBody = reply->readAll(); m_buffer.append(responseBody); reply->deleteLater(); if (m_turn.get() != turn) return; drainBuffer(); if (error == QNetworkReply::NoError) { if (turn->streamDone) { roundFinished(); return; } const QByteArray tail = m_buffer; m_buffer.clear(); const QString serverMessage = jsonErrorMessage(tail.simplified()); fail( serverMessage.isEmpty() ? QStringLiteral( "LLM server closed the stream before " "completing the response") : serverMessage); return; } if (error == QNetworkReply::OperationCanceledError) { finishTurn(); return; } const QString message = serverErrorMessage(responseBody, errorString); const int reported = contextSizeFromErrorMessage(message); if (reported > 0 && reported != m_tools->contextSize() && !turn->contextRetryUsed) { qWarning() << "LlmClient: prompt overflow, server reports" << reported << "tokens; retrying round"; setContextSize(reported); turn->contextRetryUsed = true; sendRound(); return; } fail(message); }); } void LlmClient::applyToolCallDelta(const QJsonObject& call) { Turn* turn = m_turn.get(); if (!turn) return; const int index = call[QStringLiteral("index")].toInt(-1); if (index < 0) return; while (turn->calls().size() <= index) turn->calls().append(Turn::Call{}); Turn::Call& builder = turn->calls()[index]; builder.seen = true; const QString id = call[QStringLiteral("id")].toString(); if (!id.isEmpty()) builder.id = id; const QJsonObject function = call[QStringLiteral("function")].toObject(); const QString name = function[QStringLiteral("name")].toString(); if (!name.isEmpty()) builder.name = name; const QString arguments = function[QStringLiteral("arguments")].toString(); if (!arguments.isEmpty()) builder.arguments += arguments; if (!builder.segment) { turn->generation()->closeOpenSegments(); builder.segment = turn->generation()->beginToolCall(builder.name, builder.id); } builder.segment->setName(builder.name); builder.segment->setToolCallId(builder.id); if (!arguments.isEmpty()) builder.segment->appendArguments(arguments); } void LlmClient::roundFinished() { Turn* turn = m_turn.get(); if (!turn || turn->roundDone) return; turn->roundDone = true; if (!turn->roundHasCalls()) { finishTurn(); return; } if (turn->finishReason != QLatin1String("tool_calls")) { qWarning() << "LlmClient: stream ended" << turn->finishReason << "with incomplete tool calls, ending turn"; for (const auto& builder : turn->calls()) { if (!builder.seen || !builder.segment) continue; builder.segment->finish( QStringLiteral( "Stream truncated before the tool call completed"), false); } finishTurn(); return; } if (turn->round >= kMaxToolRounds) { qWarning() << "LlmClient: tool round limit reached, ending turn"; finishTurn(); return; } turn->generation()->closeOpenSegments(); turn->appendToTranscript(turn->roundAssistantMessage()); ++turn->round; setApprovalPending(true); } void LlmClient::executeAllCalls() { Turn* turn = m_turn.get(); if (!turn) return; turn->toolPhase = true; turn->pendingCalls = 0; turn->results() = QList(turn->calls().size()); for (int i = 0; i < turn->calls().size(); ++i) { const Turn::Call& call = turn->calls().at(i); if (!call.seen) continue; LlmTool* tool = m_tools->tool(call.name); QJsonObject args; QString errorText; if (!tool) { errorText = QStringLiteral("Error: unknown tool '%1'").arg(call.name); } else if (!call.arguments.isEmpty()) { const QJsonDocument doc = QJsonDocument::fromJson(call.arguments.toUtf8()); if (!doc.isObject()) { errorText = QStringLiteral( "Error: tool arguments are not valid JSON: %1") .arg(call.arguments); } else { args = doc.object(); } } if (!errorText.isEmpty()) { turn->results()[i] = {false, errorText}; if (call.segment) call.segment->finish(errorText, false); continue; } ToolContext context = toolContext(); context.toolCallId = call.id; ++turn->pendingCalls; tool->execute(context, args, [this, turn, i](const ToolResult& result) { if (m_turn.get() != turn) return; turn->results()[i] = result; if (turn->calls().at(i).segment) turn->calls().at(i).segment->finish(result.text, result.ok); if (--turn->pendingCalls == 0) flushCallResults(); }); } if (turn->pendingCalls == 0) flushCallResults(); } void LlmClient::flushCallResults() { Turn* turn = m_turn.get(); if (!turn || !turn->toolPhase) return; turn->toolPhase = false; for (const QJsonValue& value : turn->toolMessages()) turn->appendToTranscript(value.toObject()); sendRound(); } void LlmClient::setApprovalPending(bool value) { Turn* turn = m_turn.get(); if (!turn) return; if (turn->approvalPending == value) return; turn->approvalPending = value; turn->generation()->setApprovalPending(value); } void LlmClient::approveTools(ChatGeneration* generation) { Turn* turn = m_turn.get(); if (!turn || !turn->approvalPending || turn->generation() != generation) return; setApprovalPending(false); for (auto* call : generation->toolCalls()) { if (call->status() != LlmToolCall::Status::Pending) continue; call->setStatus(LlmToolCall::Status::Running); call->begin(); } executeAllCalls(); } void LlmClient::denyTools(ChatGeneration* generation) { Turn* turn = m_turn.get(); if (!turn || !turn->approvalPending || turn->generation() != generation) return; setApprovalPending(false); for (auto* call : generation->toolCalls()) if (call->status() == LlmToolCall::Status::Pending) call->finish("Denied by user", false); finishTurn(); } void LlmClient::finishPendingCalls(const QString& resultText) { Turn* turn = m_turn.get(); if (!turn) return; for (auto* call : turn->generation()->toolCalls()) if (call->status() == LlmToolCall::Status::Pending) call->finish(resultText, false); } void LlmClient::stop() { if (!m_busy) return; Turn* turn = m_turn.get(); if (!turn) return; if (turn->approvalPending) { setApprovalPending(false); finishPendingCalls(QStringLiteral("Cancelled")); finishTurn(); return; } if (turn->toolPhase) { m_tools->cancelAll(); for (auto* call : turn->generation()->toolCalls()) if (call->running()) call->finish(QStringLiteral("Cancelled"), false); finishTurn(); return; } if (m_reply) { QNetworkReply* reply = m_reply; m_reply = nullptr; reply->abort(); } endStream(); } void LlmClient::endStream() { Turn* turn = m_turn.get(); if (!turn) return; ChatSession* session = turn->session(); ChatGeneration* generation = turn->generation(); if (m_reply) { QNetworkReply* reply = m_reply; m_reply = nullptr; reply->abort(); } generation->setStreaming(false); setApprovalPending(false); for (auto* call : generation->toolCalls()) if (call->status() == LlmToolCall::Status::Pending) call->finish(QStringLiteral("Cancelled"), false); if (generation->content().isEmpty() && generation->reasoning().isEmpty() && generation->toolCallCount() == 0 && !generation->error()) { ChatMessage* message = generation->message(); if (message) { if (message->generationCount() <= 1) session->removeMessage(message); else message->removeGeneration(generation); } } m_turn.reset(); setBusy(false); } void LlmClient::finishTurn() { ChatSession* session = m_turn ? m_turn->session() : nullptr; endStream(); if (session) { m_store->applyPendingClear(session); session->persist(); } if (m_configEnabled) refreshAllStates(); } void LlmClient::fail(const QString& message) { qWarning() << "LlmClient:" << message; ChatSession* session = m_turn ? m_turn->session() : nullptr; if (Turn* turn = m_turn.get()) turn->generation()->setError(message); finishTurn(); if (session && !message.isEmpty()) Q_EMIT generationFailed(session, message); } void LlmClient::drainBuffer() { while (true) { const qsizetype newline = m_buffer.indexOf('\n'); if (newline < 0) break; const QByteArray line = m_buffer.left(newline).trimmed(); m_buffer.remove(0, newline + 1); handleLine(line); } } void LlmClient::handleLine(const QByteArray& line) { Turn* turn = m_turn.get(); if (!turn || line.isEmpty() || !line.startsWith("data:")) return; const QByteArray data = line.mid(5).trimmed(); if (data == "[DONE]") { turn->streamDone = true; roundFinished(); return; } const QJsonDocument doc = QJsonDocument::fromJson(data); if (!doc.isObject()) return; const QJsonObject obj = doc.object(); updateTokenUsage(obj); if (obj.contains("error")) { const QJsonValue errorValue = obj["error"]; QString message; if (errorValue.isObject()) message = errorValue.toObject()["message"].toString(); else if (errorValue.isString()) message = errorValue.toString(); fail( message.isEmpty() ? QStringLiteral("LLM server returned an error") : message); return; } for (const QJsonValue& choiceValue : obj["choices"].toArray()) { if (!m_turn) break; const QJsonObject choice = choiceValue.toObject(); const QJsonObject delta = choice["delta"].toObject(); const QString finishReason = choice["finish_reason"].toString(); if (!finishReason.isEmpty()) turn->finishReason = finishReason; turn->generation()->appendContent(delta["content"].toString()); QString reasoning = delta["reasoning_content"].toString(); if (reasoning.isEmpty()) reasoning = delta["reasoning"].toString(); turn->generation()->appendReasoning(reasoning); for (const QJsonValue& callValue : delta["tool_calls"].toArray()) { if (!m_turn) break; applyToolCallDelta(callValue.toObject()); } } } void LlmClient::refreshAllStates() { if (!m_configEnabled) return; if (m_endpoint.trimmed().isEmpty()) { setServerError(QStringLiteral("LLM endpoint is not configured")); return; } probeContextSize([this]() { refreshModels(); }); } void LlmClient::refreshModels() { if (!m_configEnabled) return; const QUrl url = QUrl::fromUserInput(completionsPath(m_endpoint, "/models")); if (!url.isValid() || url.host().isEmpty()) { setServerError(QStringLiteral("LLM endpoint is not configured")); return; } auto* reply = m_manager.get(QNetworkRequest(url)); connect(reply, &QNetworkReply::finished, this, [this, reply]() { const QNetworkReply::NetworkError error = reply->error(); const QString errorString = reply->errorString(); const QByteArray data = reply->readAll(); reply->deleteLater(); if (error != QNetworkReply::NoError) { const QString serverMessage = jsonErrorMessage(data); setServerError( serverMessage.isEmpty() ? QStringLiteral("Cannot reach LLM server: %1") .arg(errorString) : serverMessage); return; } const QJsonArray dataArr = QJsonDocument::fromJson(data).object()["data"].toArray(); QStringList models; QSet seen; for (const QJsonValue& value : dataArr) { const QString id = value.toObject()["id"].toString(); if (!id.isEmpty() && !seen.contains(id)) { seen.insert(id); models.append(id); } } if (models.isEmpty()) { setServerError(QStringLiteral( "LLM server is running but has no models loaded")); return; } setServerError(QString()); m_availableModels = models; Q_EMIT availableModelsChanged(); if (m_model.isEmpty() || !models.contains(m_model)) { if (auto* config = config::ConfigSingleton::instance()) config->llm()->set_model(models.first()); setModel(models.first()); } }); } void LlmClient::probeContextSize(std::function done) { if (!m_configEnabled) { if (done) done(); return; } QString base = m_endpoint.trimmed(); while (base.endsWith('/')) base.chop(1); const QUrl url = QUrl::fromUserInput(base + "/props"); if (!url.isValid() || url.host().isEmpty()) { setServerError( QStringLiteral("Invalid LLM endpoint: %1").arg(m_endpoint)); if (done) done(); return; } auto* reply = m_manager.get(QNetworkRequest(url)); auto settled = std::make_shared(false); const auto finish = [this, reply, settled, done]() { if (*settled) return; *settled = true; const QNetworkReply::NetworkError error = reply->error(); const QString errorString = reply->errorString(); const QByteArray data = reply->readAll(); reply->deleteLater(); int size = 0; if (error == QNetworkReply::NoError) { setServerError(QString()); const QJsonDocument doc = QJsonDocument::fromJson(data); if (doc.isArray()) { for (const auto& value : doc.array()) { const QJsonObject slot = value.toObject(); if (slot.contains("n_ctx")) { size = slot["n_ctx"].toInt(0); if (size > 0) break; } } } else if (doc.isObject()) { const QJsonObject obj = doc.object(); size = obj["n_ctx"].toInt(0); if (size <= 0) size = obj["default_generation_settings"] .toObject()["n_ctx"] .toInt(0); } } else { const QString serverMessage = jsonErrorMessage(data); setServerError( serverMessage.isEmpty() ? QStringLiteral("Cannot reach LLM server: %1") .arg(errorString) : serverMessage); } if (size > 0) setContextSize(size); else if (m_tools->contextSize() <= 0) setContextSize(4096); if (done) done(); }; connect(reply, &QNetworkReply::finished, this, finish); QTimer::singleShot(kProbeTimeoutMs, this, [reply, settled]() { if (!*settled) reply->abort(); }); } void LlmClient::updateTokenUsage(const QJsonObject& data) { if (!m_turn || m_tools->contextSize() <= 0) return; const QJsonObject usage = data["usage"].toObject(); if (usage.isEmpty()) return; const double used = usage.value("prompt_tokens").toDouble() + usage.value("completion_tokens").toDouble(); if (used > 0) m_turn->session()->setUsageTokens(static_cast(used)); } void LlmClient::shortRequest( const QString& tag, const QString& systemPrompt, const QString& userText, std::function onResult) { const QUrl url = QUrl::fromUserInput(completionsPath(m_endpoint, "/chat/completions")); if (!url.isValid() || url.host().isEmpty()) return; qInfo() << "LlmClient:" << tag << "request POST" << url.toString() << "model=" << m_model; QNetworkRequest request(url); request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json"); request.setRawHeader("Accept", "application/json"); QJsonArray messages; QJsonObject system; system[QStringLiteral("role")] = QStringLiteral("system"); system[QStringLiteral("content")] = systemPrompt; messages.append(system); QJsonObject user; user[QStringLiteral("role")] = QStringLiteral("user"); user[QStringLiteral("content")] = userText.simplified().mid(0, 512); messages.append(user); QJsonObject body; if (!m_model.isEmpty()) body[QStringLiteral("model")] = m_model; body[QStringLiteral("stream")] = false; body[QStringLiteral("temperature")] = 0.3; body[QStringLiteral("max_tokens")] = 128; QJsonObject templateKwargs; templateKwargs[QStringLiteral("enable_thinking")] = false; body[QStringLiteral("chat_template_kwargs")] = templateKwargs; body[QStringLiteral("messages")] = messages; auto* reply = m_manager.post(request, QJsonDocument(body).toJson()); connect( reply, &QNetworkReply::finished, this, [this, reply, tag, onResult = std::move(onResult)]() { const QByteArray data = reply->readAll(); reply->deleteLater(); if (reply->error() != QNetworkReply::NoError) return; const QJsonDocument doc = QJsonDocument::fromJson(data); const QJsonArray choices = doc.object()[QStringLiteral("choices")].toArray(); if (choices.isEmpty()) return; const QString result = choices.at(0) .toObject()[QStringLiteral("message")] .toObject()[QStringLiteral("content")] .toString() .trimmed(); onResult(result); }); } void LlmClient::requestTitle(ChatSession* session, const QString& userText) { shortRequest( QStringLiteral("title"), prompts::titleSystemPrompt(), userText, [this, session = QPointer(session)](QString title) { if (!session) { qWarning() << "LlmClient: title request: session gone"; return; } title = prompts::cleanTitle(title); if (title.length() < 2) { qWarning() << "LlmClient: title rejected (too short)" << title; return; } qInfo() << "LlmClient: suggesting title" << title; session->setTitle(title); }); } void LlmClient::requestIcon(ChatSession* session, const QString& userText) { const QStringList icons = prompts::iconNames(); shortRequest( QStringLiteral("icon"), prompts::iconSystemPrompt(), userText, [this, session = QPointer(session), icons](QString name) { if (!session) { qWarning() << "LlmClient: icon request: session gone"; return; } name = name.simplified().toLower(); const auto isQuote = [](QChar c) { return c == QLatin1Char('"') || c == QLatin1Char('\''); }; while (name.size() >= 2 && isQuote(name.at(0)) && isQuote(name.at(name.size() - 1))) name = name.mid(1, name.size() - 2).simplified(); name.replace(QLatin1Char(' '), QLatin1Char('_')); if (!icons.contains(name)) { qWarning() << "LlmClient: icon not in list, using default" << name; name = QStringLiteral("chat"); } qInfo() << "LlmClient: suggesting icon" << name; session->setIcon(name); }); } } // namespace ZShell::llm