#include "llmclient.hpp" #include "generation.hpp" #include "message.hpp" #include "messagemodel.hpp" #include "segment.hpp" #include "session.hpp" #include #include #include #include #include #include #include #include namespace ZShell::llm { QString LlmClient::completionsPath( const QString& endpoint, const QString& subpath) { QString base = endpoint.trimmed(); while (base.endsWith('/')) base.chop(1); if (!base.endsWith("/v1")) base += "/v1"; return base + subpath; } QString LlmClient::serverErrorMessage( const QByteArray& body, const QString& fallback) { const QJsonDocument doc = QJsonDocument::fromJson(body); if (doc.isObject()) { const QJsonObject obj = doc.object(); if (obj.contains("error")) { const QJsonValue errorValue = obj["error"]; if (errorValue.isObject()) { const QString message = errorValue.toObject()["message"].toString(); if (!message.isEmpty()) return message; } else if (!errorValue.toString().isEmpty()) { return errorValue.toString(); } } } return fallback.isEmpty() ? QStringLiteral("Request to LLM server failed") : fallback; } void LlmClient::setBusy(bool value) { if (m_busy == value) return; m_busy = value; Q_EMIT busyChanged(); } void LlmClient::setStreamingChatId(const QString& id) { if (m_streamingChatId == id) return; m_streamingChatId = id; Q_EMIT streamingChatIdChanged(); } LlmClient::LlmClient(QObject* parent) : QObject(parent) { m_tools = new ToolRegistry(this); connect( m_tools, &ToolRegistry::enabledChanged, this, &LlmClient::toolsEnabledChanged); probeContextSize(); } LlmClient::~LlmClient() { if (m_reply) m_reply->abort(); m_tools->cancelAll(); endStream(); } void LlmClient::setEndpoint(const QString& value) { if (m_endpoint == value) return; m_endpoint = value; Q_EMIT endpointChanged(); probeContextSize(); if (m_model.isEmpty()) refreshModels(); } void LlmClient::setModel(const QString& value) { if (m_model == value) return; m_model = value; Q_EMIT modelChanged(); if (m_model.isEmpty()) refreshModels(); } void LlmClient::setTemperature(double value) { m_temperature = value; } void LlmClient::startGeneration( ChatSession* session, ChatGeneration* target) { if (m_busy || !session || !target) return; m_active = session; m_streaming = target; m_streaming->setStreaming(true); setBusy(true); setStreamingChatId(session->id()); const QUrl url = QUrl::fromUserInput(completionsPath(m_endpoint, "/chat/completions")); if (!url.isValid() || url.host().isEmpty()) { fail(QStringLiteral("Invalid LLM endpoint: %1").arg(m_endpoint)); return; } const auto* model = session->messagesModel(); const int targetRow = model->rowOf(qobject_cast(target->parent())); if (targetRow < 0) { fail(QStringLiteral("Internal error: generation target is not in the " "session")); return; } m_transcript = QJsonArray(); m_callBuilders.clear(); m_callResults.clear(); m_finishReason.clear(); m_round = 0; m_contentMark = 0; m_reasoningMark = 0; sendRound(); } void LlmClient::sendRound() { if (!m_active || !m_streaming) return; m_finishReason.clear(); m_callBuilders.clear(); m_callResults.clear(); m_roundDone = false; m_toolPhase = false; m_pendingCalls = 0; m_buffer.clear(); const QUrl url = QUrl::fromUserInput(completionsPath(m_endpoint, "/chat/completions")); if (!url.isValid() || url.host().isEmpty()) { fail(QStringLiteral("Invalid LLM endpoint: %1").arg(m_endpoint)); return; } const auto* model = m_active->messagesModel(); const int targetRow = model->rowOf(qobject_cast(m_streaming->parent())); if (targetRow < 0) { fail(QStringLiteral("Internal error: generation target is not in the " "session")); return; } // Context: every message older than the target, oldest first, plus // the tool exchanges of the current turn so far. QJsonArray messages = buildContextMessages(m_active, targetRow); for (const QJsonValue& value : m_transcript) messages.append(value); QJsonObject body; QJsonObject streamOptions; streamOptions[QStringLiteral("include_usage")] = true; body[QStringLiteral("stream_options")] = streamOptions; body[QStringLiteral("messages")] = messages; body[QStringLiteral("stream")] = true; body[QStringLiteral("temperature")] = m_temperature; if (!m_model.isEmpty()) body[QStringLiteral("model")] = m_model; const QJsonArray toolSpecs = m_tools->specifications(); if (!toolSpecs.isEmpty()) body[QStringLiteral("tools")] = toolSpecs; QNetworkRequest request(url); request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json"); request.setRawHeader("Accept", "text/event-stream"); m_reply = m_manager.post(request, QJsonDocument(body).toJson()); connect(m_reply, &QNetworkReply::readyRead, this, [this]() { if (m_reply) m_buffer.append(m_reply->readAll()); drainBuffer(); }); connect(m_reply, &QNetworkReply::finished, this, [this]() { QNetworkReply* reply = m_reply; if (!reply) return; m_reply = nullptr; const QNetworkReply::NetworkError error = reply->error(); const QString errorString = reply->errorString(); // Data not already consumed by readyRead is only reachable here. const QByteArray responseBody = reply->readAll(); m_buffer.append(responseBody); reply->deleteLater(); drainBuffer(); if (!m_streaming) return; if (error == QNetworkReply::NoError) roundFinished(); else if (error == QNetworkReply::OperationCanceledError) finishTurn(); else fail(serverErrorMessage(responseBody, errorString)); }); } QJsonArray LlmClient::buildContextMessages( ChatSession* session, int stopBeforeRow) const { const auto* model = session->messagesModel(); QJsonArray messages; for (int row = model->rowCount() - 1; row > stopBeforeRow; --row) { const auto* message = model->at(row); const auto* generation = message->activeGeneration(); if (!generation) continue; if (message->role() == ChatMessage::Role::User) { QJsonObject user; user[QStringLiteral("role")] = QStringLiteral("user"); user[QStringLiteral("content")] = generation->content(); messages.append(user); continue; } // Assistant message: replay its tool calls (and their results) // so the model keeps the full history of the turn. QList toolSegments; for (const auto* segment : generation->segments()) if (segment->type() == LlmSegment::Type::ToolCall) toolSegments.append(segment); QJsonObject assistant; assistant[QStringLiteral("role")] = QStringLiteral("assistant"); if (toolSegments.isEmpty()) assistant[QStringLiteral("content")] = generation->content(); else if (!generation->content().isEmpty()) assistant[QStringLiteral("content")] = generation->content(); if (!generation->reasoning().isEmpty()) assistant[QStringLiteral("reasoning_content")] = generation->reasoning(); if (toolSegments.isEmpty()) { messages.append(assistant); continue; } QJsonArray calls; for (const auto* segment : toolSegments) { QJsonObject function; function[QStringLiteral("name")] = segment->name(); function[QStringLiteral("arguments")] = segment->arguments(); QJsonObject call; call[QStringLiteral("id")] = segment->toolCallId(); call[QStringLiteral("type")] = QStringLiteral("function"); call[QStringLiteral("function")] = function; calls.append(call); } assistant[QStringLiteral("tool_calls")] = calls; messages.append(assistant); for (const auto* segment : toolSegments) { QJsonObject toolMessage; toolMessage[QStringLiteral("role")] = QStringLiteral("tool"); toolMessage[QStringLiteral("tool_call_id")] = segment->toolCallId(); toolMessage[QStringLiteral("content")] = segment->result(); messages.append(toolMessage); } } return messages; } void LlmClient::applyToolCallDelta(const QJsonObject& call) { if (!m_streaming) return; const int index = call[QStringLiteral("index")].toInt(-1); if (index < 0) return; while (m_callBuilders.size() <= index) m_callBuilders.append(ToolCallBuilder{}); ToolCallBuilder& builder = m_callBuilders[index]; builder.seen = true; const QString id = call[QStringLiteral("id")].toString(); if (!id.isEmpty()) builder.id = id; const QJsonObject function = call[QStringLiteral("function")].toObject(); const QString name = function[QStringLiteral("name")].toString(); if (!name.isEmpty()) builder.name = name; const QString arguments = function[QStringLiteral("arguments")].toString(); if (!arguments.isEmpty()) builder.arguments += arguments; if (!builder.segment) { // A new call: close the in-flight text segments and open a // running tool-call segment so the UI can track it live. m_streaming->closeOpenSegments(); builder.segment = m_streaming->beginToolCall(builder.name, builder.id); } builder.segment->setName(builder.name); builder.segment->setToolCallId(builder.id); if (!arguments.isEmpty()) builder.segment->appendArguments(arguments); } void LlmClient::roundFinished() { // [DONE] and the reply's finished signal both funnel here; only the // first may act. if (m_roundDone || !m_streaming) return; m_roundDone = true; bool hasCalls = false; for (const auto& builder : m_callBuilders) if (builder.seen) hasCalls = true; if (m_finishReason != QLatin1String("tool_calls") && !hasCalls) { finishTurn(); return; } if (m_round >= kMaxToolRounds) { qWarning() << "LlmClient: tool round limit reached, ending turn"; finishTurn(); return; } m_streaming->closeOpenSegments(); // Record the assistant's tool-call message in the transcript so the // next round (and the model) can see it. const QString content = m_streaming->content(); const QString reasoning = m_streaming->reasoning(); QJsonObject assistant; assistant[QStringLiteral("role")] = QStringLiteral("assistant"); if (content.size() > m_contentMark) { assistant[QStringLiteral("content")] = content.mid(m_contentMark); m_contentMark = content.size(); } if (reasoning.size() > m_reasoningMark) { assistant[QStringLiteral("reasoning_content")] = reasoning.mid(m_reasoningMark); m_reasoningMark = reasoning.size(); } QJsonArray calls; for (const auto& builder : m_callBuilders) { if (!builder.seen) continue; QJsonObject function; function[QStringLiteral("name")] = builder.name; function[QStringLiteral("arguments")] = builder.arguments; QJsonObject call; call[QStringLiteral("id")] = builder.id; call[QStringLiteral("type")] = QStringLiteral("function"); call[QStringLiteral("function")] = function; calls.append(call); } assistant[QStringLiteral("tool_calls")] = calls; m_transcript.append(assistant); m_round++; executeAllCalls(); } void LlmClient::executeAllCalls() { m_toolPhase = true; m_pendingCalls = 0; m_callResults = QList(m_callBuilders.size()); for (int i = 0; i < m_callBuilders.size(); ++i) { const auto& call = m_callBuilders.at(i); if (!call.seen) continue; LlmTool* tool = m_tools->tool(call.name); QJsonObject args; QString errorText; if (!tool) { errorText = QStringLiteral("Error: unknown tool '%1'") .arg(call.name); } else if (!call.arguments.isEmpty()) { const QJsonDocument doc = QJsonDocument::fromJson(call.arguments.toUtf8()); if (!doc.isObject()) { errorText = QStringLiteral("Error: tool arguments are not valid " "JSON: %1") .arg(call.arguments); } else { args = doc.object(); } } if (!errorText.isEmpty()) { m_callResults[i] = { errorText, false }; if (LlmSegment* segment = call.segment) segment->finishTool(errorText, false); continue; } ++m_pendingCalls; tool->execute( args, [this, i, call](const QJsonObject& result) { if (!m_streaming) return; const bool success = result.contains(QStringLiteral("output")); const QString content = success ? result[QStringLiteral("output")].toString() : QStringLiteral("Error: ") + result[QStringLiteral("error")].toString(); m_callResults[i] = { content, success }; if (LlmSegment* segment = call.segment) segment->finishTool(content, success); if (--m_pendingCalls == 0) flushCallResults(); }); } if (m_pendingCalls == 0) flushCallResults(); } void LlmClient::flushCallResults() { if (!m_toolPhase) return; m_toolPhase = false; if (!m_streaming) return; for (int i = 0; i < m_callBuilders.size(); ++i) { const auto& call = m_callBuilders.at(i); if (!call.seen) continue; QJsonObject toolMessage; toolMessage[QStringLiteral("role")] = QStringLiteral("tool"); toolMessage[QStringLiteral("tool_call_id")] = call.id; toolMessage[QStringLiteral("content")] = m_callResults.at(i).content; m_transcript.append(toolMessage); } sendRound(); } void LlmClient::stop() { if (!m_busy) return; if (m_toolPhase) { m_tools->cancelAll(); if (m_streaming) { for (auto* segment : m_streaming->segments()) { if (segment->type() == LlmSegment::Type::ToolCall && segment->running()) segment->finishTool( QStringLiteral("Cancelled"), false); } } finishTurn(); return; } if (m_reply) m_reply->abort(); } void LlmClient::endStream() { if (!m_streaming) return; auto* generation = m_streaming; auto* session = m_active; m_streaming = nullptr; m_active = nullptr; generation->setStreaming(false); if (generation->content().isEmpty() && generation->reasoning().isEmpty() && generation->toolCallCount() == 0) { if (auto* message = qobject_cast(generation->parent())) { if (message->generationCount() <= 1) { if (session) session->removeMessage(message); } else { message->removeGeneration(generation); } } } setBusy(false); setStreamingChatId(QString()); } void LlmClient::finishTurn() { if (!m_streaming) return; ChatSession* session = m_active; endStream(); if (session && m_pendingClear == session) { session->clearMessages(); m_pendingClear.clear(); } if (session) session->persist(); } void LlmClient::clearOnFinish(ChatSession* session) { m_pendingClear = session; } void LlmClient::sessionRemoved(ChatSession* session) { if (m_pendingClear == session) m_pendingClear.clear(); if (m_active == session) { stop(); endStream(); } } void LlmClient::fail(const QString& message) { qWarning() << "LlmClient:" << message; finishTurn(); Q_EMIT errorOccurred(message); } void LlmClient::drainBuffer() { while (true) { const qsizetype newline = m_buffer.indexOf('\n'); if (newline < 0) break; const QByteArray line = m_buffer.left(newline).trimmed(); m_buffer.remove(0, newline + 1); handleLine(line); } } void LlmClient::handleLine(const QByteArray& line) { if (!m_streaming || line.isEmpty() || !line.startsWith("data:")) return; const QByteArray data = line.mid(5).trimmed(); if (data == "[DONE]") { roundFinished(); return; } const QJsonDocument doc = QJsonDocument::fromJson(data); if (!doc.isObject()) return; const QJsonObject obj = doc.object(); updateTokenUsage(obj); if (obj.contains("error")) { const QJsonObject error = obj["error"].toObject(); const QString message = error["message"].toString(); fail(message.isEmpty() ? QStringLiteral("LLM server returned an error") : message); return; } for (const QJsonValue& choiceValue : obj["choices"].toArray()) { if (!m_streaming) continue; const QJsonObject choice = choiceValue.toObject(); const QJsonObject delta = choice["delta"].toObject(); const QString finishReason = choice[QStringLiteral("finish_reason")].toString(); if (!finishReason.isEmpty()) m_finishReason = finishReason; m_streaming->appendContent(delta["content"].toString()); QString reasoning = delta["reasoning_content"].toString(); if (reasoning.isEmpty()) reasoning = delta["reasoning"].toString(); m_streaming->appendReasoning(reasoning); for (const QJsonValue& callValue : delta["tool_calls"].toArray()) { if (!m_streaming) break; applyToolCallDelta(callValue.toObject()); } } } void LlmClient::refreshModels() { const QUrl url = QUrl::fromUserInput(completionsPath(m_endpoint, "/models")); if (!url.isValid() || url.host().isEmpty()) return; auto* reply = m_manager.get(QNetworkRequest(url)); connect(reply, &QNetworkReply::finished, this, [this, reply]() { const QNetworkReply::NetworkError error = reply->error(); const QByteArray data = reply->readAll(); reply->deleteLater(); if (error != QNetworkReply::NoError) { qWarning() << "LlmClient: failed to fetch models:" << error; return; } const QJsonArray dataArr = QJsonDocument::fromJson(data).object()["data"].toArray(); QStringList models; QSet seen; for (const QJsonValue& value : dataArr) { const QString id = value.toObject()["id"].toString(); if (!id.isEmpty() && !seen.contains(id)) { seen.insert(id); models.append(id); } } if (models.isEmpty()) return; m_availableModels = models; Q_EMIT availableModelsChanged(); if (m_model.isEmpty()) { m_model = models.first(); Q_EMIT modelChanged(); } }); } void LlmClient::setContextSize(int size) { if (size <= 0 || m_contextSize == size) return; m_contextSize = size; Q_EMIT contextSizeChanged(); } void LlmClient::probeContextSize() { // llama.cpp-specific endpoint; other servers fall back to 4096. QString base = m_endpoint.trimmed(); while (base.endsWith('/')) base.chop(1); const QUrl url = QUrl::fromUserInput(base + "/props"); if (!url.isValid() || url.host().isEmpty()) return; auto* reply = m_manager.get(QNetworkRequest(url)); connect( reply, &QNetworkReply::finished, this, [this, reply]() { const QNetworkReply::NetworkError error = reply->error(); const QByteArray data = reply->readAll(); reply->deleteLater(); int size = 0; if (error == QNetworkReply::NoError) { const QJsonDocument doc = QJsonDocument::fromJson(data); if (doc.isArray()) { for (const auto& value : doc.array()) { const QJsonObject slot = value.toObject(); if (slot.contains("n_ctx")) { size = slot["n_ctx"].toInt(0); if (size > 0) break; } } } else if (doc.isObject()) { const QJsonObject obj = doc.object(); size = obj["n_ctx"].toInt(0); if (size <= 0) size = obj["default_generation_settings"].toObject() ["n_ctx"].toInt(0); } } setContextSize(size > 0 ? size : 4096); }); } void LlmClient::updateTokenUsage(const QJsonObject& data) { if (!m_active || m_contextSize <= 0) return; const QJsonObject usage = data["usage"].toObject(); if (usage.isEmpty()) return; const double used = usage.value("prompt_tokens").toDouble() + usage.value("completion_tokens").toDouble(); if (used > 0) m_active->setLastTokenCount(static_cast(used)); } void LlmClient::shortRequest( const QString& tag, const QString& systemPrompt, const QString& userText, std::function onResult) { const QUrl url = QUrl::fromUserInput(completionsPath(m_endpoint, "/chat/completions")); if (!url.isValid() || url.host().isEmpty()) return; qInfo() << "LlmClient:" << tag << "request POST" << url.toString() << "model=" << m_model; QNetworkRequest request(url); request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json"); request.setRawHeader("Accept", "application/json"); QJsonArray messages; QJsonObject system; system[QStringLiteral("role")] = QStringLiteral("system"); system[QStringLiteral("content")] = systemPrompt; messages.append(system); QJsonObject user; user[QStringLiteral("role")] = QStringLiteral("user"); user[QStringLiteral("content")] = userText.simplified().mid(0, 512); messages.append(user); QJsonObject body; if (!m_model.isEmpty()) body[QStringLiteral("model")] = m_model; body[QStringLiteral("stream")] = false; body[QStringLiteral("temperature")] = 0.3; body[QStringLiteral("max_tokens")] = 128; QJsonObject templateKwargs; templateKwargs[QStringLiteral("enable_thinking")] = false; body[QStringLiteral("chat_template_kwargs")] = templateKwargs; body[QStringLiteral("messages")] = messages; auto* reply = m_manager.post(request, QJsonDocument(body).toJson()); connect( reply, &QNetworkReply::finished, this, [this, reply, tag, onResult = std::move(onResult)]() { const QByteArray data = reply->readAll(); reply->deleteLater(); qInfo() << "LlmClient:" << tag << "request finished" << "error=" << reply->error() << reply->errorString() << "http=" << reply->attribute( QNetworkRequest::HttpStatusCodeAttribute) .toInt() << "response=" << QString::fromUtf8(data.left(400)).simplified(); if (reply->error() != QNetworkReply::NoError) return; const QJsonDocument doc = QJsonDocument::fromJson(data); const QJsonArray choices = doc.object()[QStringLiteral("choices")].toArray(); if (choices.isEmpty()) return; const QString result = choices.at(0).toObject()[QStringLiteral("message")].toObject() [QStringLiteral("content")].toString() .trimmed(); qInfo() << "LlmClient:" << tag << "raw result" << result; onResult(result); }); } void LlmClient::requestTitle(ChatSession* session, const QString& userText) { shortRequest( QStringLiteral("title"), QStringLiteral( "Write a short, concise title for a chat conversation starting " "with the user's message below. At most six words, no quotation " "marks, no trailing punctuation. Reply with the title only."), userText, [this, session = QPointer(session)](QString title) { if (!session) { qWarning() << "LlmClient: title request: session gone"; return; } const auto isQuote = [](QChar c) { return c == QLatin1Char('"') || c == QLatin1Char('\'') || c == QChar(u'\u201C') || c == QChar(u'\u201D') || c == QChar(u'\u2018') || c == QChar(u'\u2019'); }; while (title.size() >= 2 && isQuote(title.at(0)) && isQuote(title.at(title.size() - 1))) title = title.mid(1, title.size() - 2).simplified(); while (!title.isEmpty() && (title.endsWith(QLatin1Char('.')) || title.endsWith(QLatin1Char('!')) || title.endsWith(QLatin1Char('?')))) title.chop(1); if (title.size() < 2) { qWarning() << "LlmClient: title rejected (too short)" << title; return; } if (title.size() > 48) title = title.left(47) + QStringLiteral("…"); qInfo() << "LlmClient: suggesting title" << title; Q_EMIT titleSuggested(session, title); }); } void LlmClient::requestIcon(ChatSession* session, const QString& userText) { const QStringList icons = { "chat", "lightbulb", "code", "description", "article", "school", "work", "build", "science", "palette", "music_note", "sports_esports", "takeout_dining", "flight", "photo_camera", "psychology_alt", "favorite", "savings", "gamepad", "auto_awesome", }; const QString prompt = QStringLiteral( "Pick the single icon name from this list that best matches the " "topic of the user's message below: %1. Reply with only the icon " "name, exactly as written in the list, and nothing else.") .arg(icons.join(QStringLiteral(", "))); shortRequest( QStringLiteral("icon"), prompt, userText, [this, session = QPointer(session), icons]( QString name) { if (!session) { qWarning() << "LlmClient: icon request: session gone"; return; } name = name.simplified().toLower(); const auto isQuote = [](QChar c) { return c == QLatin1Char('"') || c == QLatin1Char('\''); }; while (name.size() >= 2 && isQuote(name.at(0)) && isQuote(name.at(name.size() - 1))) name = name.mid(1, name.size() - 2).simplified(); name.replace(QLatin1Char(' '), QLatin1Char('_')); if (!icons.contains(name)) { qWarning() << "LlmClient: icon not in list, using default" << name; name = QStringLiteral("chat"); } qInfo() << "LlmClient: suggesting icon" << name; Q_EMIT iconSuggested(session, name); }); } } // namespace ZShell::llm