zach/z-bar-qt
1
1
Fork 0
Code Issues 23 Pull Requests 2 Actions Packages Projects 1 Releases Wiki Activity

async parsing + tex caching

Browse Source
This commit is contained in:
zach
2026-08-26 02:06:10 +02:00
parent e18875a752
commit 3946541f87
32 changed files with 2730 additions and 489 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Plugins/ZShell/Llm/codehighlighter.cpp
+131 -129
View File
@@ -4,10 +4,17 @@
#include <tree_sitter/api.h>
#include <QCoreApplication>
#include <QDir>
#include <QHash>
#include <QMap>
#include <QMutexLocker>
#include <QPointer>
#include <QStringList>
#include <QThreadPool>
#include <QVariantMap>
#include <cstring>
#include <dlfcn.h>
namespace ZShell::llm {
@@ -57,77 +64,21 @@ const char* roleName(Role role) {
using LanguageFn = const TSLanguage* (*)();
// Query sources, indexed by Grammar::queries.
struct QuerySources {
std::vector<std::string> sources;
int c, cpp, python, javascript, typescriptExtended, typescript, bash, json, rust, go, yaml, toml, sql, cppExtended;
};
const QuerySources& querySources() {
static const QuerySources sources = [] {
QuerySources qs;
// javascript + typescript concatenated: the TS grammar reuses the
// JS node names, so the JS query usually compiles against it and
// gives full coverage; the TS-only query is the fallback.
const std::string tsExtended =
std::string(hq::javascript) + "\n" + std::string(hq::typescript);
// Same for cpp: the C++ grammar is a superset of C, and the C++
// query only covers the C++-specific delta. Base C coverage
// (keywords, types, calls, strings, comments) comes from the C
// query.
const std::string cppExtended =
std::string(hq::c) + "\n" + std::string(hq::cpp);
qs.sources.reserve(14);
qs.sources.push_back(hq::c);
qs.sources.push_back(hq::cpp);
qs.sources.push_back(hq::python);
qs.sources.push_back(hq::javascript);
qs.sources.push_back(tsExtended);
qs.sources.push_back(hq::typescript);
qs.sources.push_back(hq::bash);
qs.sources.push_back(hq::json);
qs.sources.push_back(hq::rust);
qs.sources.push_back(hq::go);
qs.sources.push_back(hq::yaml);
qs.sources.push_back(hq::toml);
qs.sources.push_back(hq::sql);
qs.sources.push_back(cppExtended);
qs.c = 0;
qs.cpp = 1;
qs.python = 2;
qs.javascript = 3;
qs.typescriptExtended = 4;
qs.typescript = 5;
qs.bash = 6;
qs.json = 7;
qs.rust = 8;
qs.go = 9;
qs.yaml = 10;
qs.toml = 11;
qs.sql = 12;
qs.cppExtended = 13;
return qs;
}();
return sources;
}
// Grammar registry generated by CMake from the installed grammars and
// their highlight queries (see CMakeLists.txt).
const QHash<QString, CodeHighlighter::Grammar>& grammars() {
static const QHash<QString, CodeHighlighter::Grammar> grammars = [] {
const auto& qs = querySources();
QHash<QString, CodeHighlighter::Grammar> map;
map.insert("c", {"libtree-sitter-c.so", "tree_sitter_c", {qs.c}});
map.insert("cpp", {"libtree-sitter-cpp.so", "tree_sitter_cpp", {qs.cppExtended, qs.cpp}});
map.insert("python", {"libtree-sitter-python.so", "tree_sitter_python", {qs.python}});
map.insert("javascript", {"libtree-sitter-javascript.so", "tree_sitter_javascript", {qs.javascript}});
map.insert("typescript", {"libtree-sitter-typescript.so", "tree_sitter_typescript", {qs.typescriptExtended, qs.typescript}});
map.insert("tsx", {"libtree-sitter-tsx.so", "tree_sitter_tsx", {qs.typescriptExtended, qs.typescript}});
map.insert("bash", {"libtree-sitter-bash.so", "tree_sitter_bash", {qs.bash}});
map.insert("json", {"libtree-sitter-json.so", "tree_sitter_json", {qs.json}});
map.insert("rust", {"libtree-sitter-rust.so", "tree_sitter_rust", {qs.rust}});
map.insert("go", {"libtree-sitter-go.so", "tree_sitter_go", {qs.go}});
map.insert("yaml", {"libtree-sitter-yaml.so", "tree_sitter_yaml", {qs.yaml}});
map.insert("toml", {"libtree-sitter-toml.so", "tree_sitter_toml", {qs.toml}});
map.insert("sql", {"libtree-sitter-sql.so", "tree_sitter_sql", {qs.sql}});
for (const auto& g : hq::grammars) {
CodeHighlighter::Grammar grammar;
for (int i = 0; i < g.nCandidates; ++i) {
grammar.libs.push_back(g.candidates[i].lib);
grammar.symbols.push_back(g.candidates[i].symbol);
}
for (int i = 0; i < g.nQueries; ++i)
grammar.queries.push_back(g.queries[i]);
map.insert(g.id, std::move(grammar));
}
return map;
}();
return grammars;
@@ -170,6 +121,8 @@ const QHash<QString, QString>& CodeHighlighter::aliases() {
map.insert("shell-session", "bash");
map.insert("zsh", "bash");
map.insert("console", "bash");
map.insert("qml", "qmljs");
map.insert("qmljs", "qmljs");
map.insert("json", "json");
map.insert("jsonc", "json");
map.insert("rust", "rust");
@@ -190,10 +143,6 @@ const QHash<QString, QString>& CodeHighlighter::aliases() {
return aliases;
}
const std::vector<std::string>& CodeHighlighter::querySources() const {
return hl::querySources().sources;
}
uint8_t CodeHighlighter::roleFor(const char* name, uint32_t length) {
const QString n = QString::fromUtf8(name, length);
if (n == "comment")
@@ -204,15 +153,18 @@ uint8_t CodeHighlighter::roleFor(const char* name, uint32_t length) {
return hl::Role::String;
if (n.startsWith("number"))
return hl::Role::Number;
if (n.startsWith("constant"))
if (n.startsWith("constant") || n == "boolean" || n == "bool"
|| n.startsWith("character"))
return hl::Role::Constant;
if (n.startsWith("keyword"))
return hl::Role::Keyword;
if (n == "type" || n.startsWith("type."))
return hl::Role::Type;
if (n == "namespace" || n == "module")
if (n.startsWith("namespace") || n.startsWith("module")
|| n == "support.type" || n == "support.namespace")
return hl::Role::Type;
if (n.startsWith("function") || n == "constructor")
if (n.startsWith("function") || n == "constructor"
|| n.startsWith("support.function"))
return hl::Role::Function;
if (n == "method" || n == "method.builtin")
return hl::Role::Method;
@@ -220,7 +172,8 @@ uint8_t CodeHighlighter::roleFor(const char* name, uint32_t length) {
return hl::Role::Macro;
if (n.startsWith("preproc"))
return hl::Role::Preproc;
if (n == "operator" || n == "punctuation.operator" || n.startsWith("operator."))
if (n == "operator" || n == "punctuation.operator" || n.startsWith("operator.")
|| n.startsWith("punctuation"))
return hl::Role::Operator;
if (n == "property" || n == "field" || n.startsWith("property."))
return hl::Role::Property;
@@ -228,6 +181,13 @@ uint8_t CodeHighlighter::roleFor(const char* name, uint32_t length) {
return hl::Role::Label;
if (n.startsWith("attribute") || n == "annotation")
return hl::Role::Attribute;
// HTML tag names, CSS variables and friends.
if (n == "tag" || (n.startsWith("tag.") && n != "tag.delimiter"))
return hl::Role::Keyword;
if (n.startsWith("variable"))
return hl::Role::Constant;
if (n.startsWith("support"))
return hl::Role::Function;
return hl::Role::None;
}
@@ -235,15 +195,45 @@ const char* CodeHighlighter::roleName(uint8_t role) {
return hl::roleName(static_cast<hl::Role>(role));
}
QVariantList CodeHighlighter::highlight(const QString& code, const QString& language) const {
void CodeHighlighter::highlight(
const QString& code, const QString& language, QObject* target, int token) {
QThreadPool::globalInstance()->start([this, target, token, code, language]() {
const QVariantList spans = doHighlight(code, language);
// The target item may be long gone by now (delegates are
// recreated constantly while chats load); a destroyed target is
// simply skipped. Deliver through the app instance (never
// destroyed) and re-check there: posting to `target` from the
// pool thread would race with its destruction.
QPointer<QObject> guard(target);
QMetaObject::invokeMethod(
QCoreApplication::instance(),
[guard, token, spans]() {
if (!guard)
return;
// QML functions are only invokable by their generic
// QVariant overload, so pass untyped arguments.
QMetaObject::invokeMethod(
guard, "onHighlightSpans",
Q_ARG(QVariant, token), Q_ARG(QVariant, spans));
},
Qt::QueuedConnection);
});
}
QVariantList CodeHighlighter::doHighlight(
const QString& code, const QString& language) const {
QVariantList spans;
if (code.isEmpty())
return spans;
const QString id = aliases().value(language.trimmed().toLower());
if (id.isEmpty())
return spans;
const QString tag = language.trimmed().toLower();
const QString id = [&] {
const QString alias = aliases().value(tag);
return alias.isEmpty() ? tag : alias; // unknown tags = grammar id
}();
const Grammar& grammar = hl::grammars().value(id);
if (grammar.libs.empty())
return spans;
// Guard against pathological blocks; highlighting is best-effort.
static constexpr size_t kMaxBytes = 512 * 1024;
@@ -251,59 +241,72 @@ QVariantList CodeHighlighter::highlight(const QString& code, const QString& lang
if (static_cast<size_t>(utf8.size()) > kMaxBytes)
return spans;
State& state = m_states[id];
if (state.bad)
return spans;
if (!state.lang) {
// Missing library is retriable (it may be installed while the
// shell runs); ABI/query failures below are not.
state.lib = dlopen(grammar.lib.toUtf8().constData(), RTLD_NOW | RTLD_LOCAL);
if (!state.lib)
return spans;
auto* symbol = reinterpret_cast<hl::LanguageFn>(
dlsym(state.lib, grammar.symbol.toUtf8().constData()));
if (!symbol) {
dlclose(state.lib);
state.lib = nullptr;
return spans;
const TSLanguage* lang = nullptr;
TSQuery* query = nullptr;
{
QMutexLocker locker(&m_stateMutex);
auto& state = m_states[id];
// A missing library is retriable (it may be installed while the
// shell runs); an ABI mismatch on every candidate is not. Cache
// successes and permanent failures; leave retriable misses out.
if (!state || (!state->lang && !state->bad)) {
std::shared_ptr<State> fresh = std::make_shared<State>();
bool abiMismatch = false;
for (size_t i = 0; i < grammar.libs.size(); ++i) {
void* lib = dlopen(grammar.libs[i].c_str(),
RTLD_NOW | RTLD_LOCAL);
if (!lib)
continue;
auto* symbol = reinterpret_cast<hl::LanguageFn>(
dlsym(lib, grammar.symbols[i].c_str()));
if (!symbol) {
dlclose(lib);
continue;
}
const TSLanguage* candidate = symbol();
const uint32_t version = ts_language_abi_version(candidate);
if (version < TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION ||
version > TREE_SITTER_LANGUAGE_VERSION) {
dlclose(lib);
abiMismatch = true;
continue;
}
fresh->lib = lib;
fresh->lang = candidate;
break;
}
if (fresh->lang) {
// Candidates in priority order; first that compiles wins.
for (const char* source : grammar.queries) {
TSQueryError errorType = TSQueryErrorNone;
uint32_t errorOffset = 0;
TSQuery* candidate = ts_query_new(
static_cast<const TSLanguage*>(fresh->lang),
source,
static_cast<uint32_t>(std::strlen(source)),
&errorOffset,
&errorType);
if (!candidate)
continue;
fresh->query = candidate;
break;
}
if (!fresh->query)
fresh->bad = true;
} else if (abiMismatch) {
fresh->bad = true;
}
if (fresh->lang || fresh->bad)
state = std::move(fresh);
}
const TSLanguage* lang = symbol();
const uint32_t version = ts_language_abi_version(lang);
if (version < TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION ||
version > TREE_SITTER_LANGUAGE_VERSION) {
state.bad = true;
if (!state || state->bad || !state->lang)
return spans;
}
state.lang = lang;
}
if (!state.query) {
// Candidates in priority order; first that compiles wins.
for (const int candidate : grammar.queries) {
const std::string& source =
querySources()[static_cast<size_t>(candidate)];
TSQueryError errorType = TSQueryErrorNone;
uint32_t errorOffset = 0;
TSQuery* query = ts_query_new(
static_cast<const TSLanguage*>(state.lang),
source.data(),
static_cast<uint32_t>(source.size()),
&errorOffset,
&errorType);
if (!query)
continue;
state.query = query;
break;
}
if (!state.query) {
state.bad = true;
return spans;
}
lang = static_cast<const TSLanguage*>(state->lang);
query = static_cast<TSQuery*>(state->query);
}
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, static_cast<const TSLanguage*>(state.lang));
ts_parser_set_language(parser, lang);
TSTree* tree = ts_parser_parse_string(
parser, nullptr, utf8.constData(), static_cast<uint32_t>(utf8.size()));
if (!tree) {
@@ -311,7 +314,6 @@ QVariantList CodeHighlighter::highlight(const QString& code, const QString& lang
return spans;
}
TSQuery* query = static_cast<TSQuery*>(state.query);
TSQueryCursor* cursor = ts_query_cursor_new();
ts_query_cursor_exec(cursor, query, ts_tree_root_node(tree));