fix: truncate webfetch content to fit context

Plugins/ZShell/Llm/webfetchtool.cpp

@@ -1,5 +1,7 @@
 #include "webfetchtool.hpp"
 
+#include <QDir>
+#include <QFile>
 #include <QJsonArray>
 #include <QNetworkReply>
 #include <QNetworkRequest>
@@ -8,6 +10,9 @@
 
 namespace ZShell::llm {
 
+const QString WebFetchTool::StoragePath =
+	QStringLiteral("/tmp/zshell-llm/webfetch");
+
 namespace {
 
 const char* kUserAgent =
@@ -46,7 +51,10 @@ QString WebFetchTool::description() const {
 	return QStringLiteral(
 		"Fetch content from an HTTP or HTTPS URL and return it as plain "
 		"text or raw HTML. HTML pages are reduced to their visible text "
-		"by default. This tool is read-only.");
+		"by default. Only a limited amount of content is returned inline; "
+		"the full output of every call is saved under "
+		"/tmp/zshell-llm/webfetch/, where it can be paged through with "
+		"the readfile tool. This tool is read-only.");
 }
 
 QJsonObject WebFetchTool::parameters() const {
@@ -102,9 +110,12 @@ void WebFetchTool::completeJob(Job* job, QJsonObject result) {
 }
 
 void WebFetchTool::execute(
-	const QJsonObject& args, std::function<void(const QJsonObject&)> done) {
+	const QString& toolCallId,
+	const QJsonObject& args,
+	std::function<void(const QJsonObject&)> done) {
 	auto* job = new Job;
 	job->done = std::move(done);
+	job->toolCallId = toolCallId;
 	m_jobs.append(job);
 
 	auto fail = [this, job](const QString& message) {
@@ -233,13 +244,64 @@ void WebFetchTool::execute(
 		if (mime.contains(QLatin1String("text/html")) &&
 			job->format == QLatin1String("text"))
 			content = extractTextFromHtml(content);
-		if (content.size() > MaxOutputChars)
-			content = content.left(MaxOutputChars) +
-					  QStringLiteral("\n[... truncated ...]");
-		completeJob(job, makeOutput(content));
+
+		const QString savedPath = saveToFile(*job, content, mime);
+
+		QString output = content;
+		const int budget = inlineBudgetChars();
+		if (content.size() > budget) {
+			output = content.left(budget);
+			const QString note = QStringLiteral(
+				"\n\n[... truncated: showing %1 of %2 characters")
+								.arg(budget)
+								.arg(content.size());
+			if (!savedPath.isEmpty())
+				output += note +
+						  QStringLiteral(
+							  ". The full content is saved to %1; use "
+							  "the readfile tool to read the rest.")
+							  .arg(savedPath);
+			else
+				output += note + QStringLiteral(
+								   ". The remaining content is not "
+								   "available.");
+		}
+		completeJob(job, makeOutput(output));
 	});
 }
 
+QString WebFetchTool::saveToFile(
+	const Job& job, const QString& content, const QString& mime) const {
+	QDir dir(StoragePath);
+	if (!dir.exists() && !dir.mkpath(QStringLiteral(".")))
+		return QString();
+
+	QString fileName;
+	fileName.reserve(job.toolCallId.size());
+	for (const QChar& c : job.toolCallId)
+		fileName += (c.isLetterOrNumber() || c == QLatin1Char('-') ||
+					  c == QLatin1Char('_'))
+						? c
+						: QLatin1Char('_');
+	if (fileName.isEmpty()) fileName = QStringLiteral("fetch");
+
+	QString extension = QStringLiteral("txt");
+	if (job.format == QLatin1String("html"))
+		extension = QStringLiteral("html");
+	else if (mime.contains(QLatin1String("json")))
+		extension = QStringLiteral("json");
+	else if (mime.contains(QLatin1String("xml")))
+		extension = QStringLiteral("xml");
+
+	const QString path =
+		dir.filePath(fileName + QLatin1Char('.') + extension);
+	QFile file(path);
+	if (!file.open(QIODevice::WriteOnly | QIODevice::Truncate))
+		return QString();
+	file.write(content.toUtf8());
+	return path;
+}
+
 void WebFetchTool::cancel() {
 	for (auto* job : m_jobs) {
 		job->timer->stop();