zach/z-bar-qt
1
1
Fork 0
Code Issues 24 Pull Requests 2 Actions Packages Projects 1 Releases Wiki Activity

async parsing + tex caching

Browse Source
This commit is contained in:
zach
2026-08-26 02:06:10 +02:00
parent e18875a752
commit 3946541f87
32 changed files with 2730 additions and 489 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Plugins/ZShell/Llm/codehighlighter.hpp
+36 -16
View File
@@ -1,11 +1,13 @@
#pragma once
#include <QMutex>
#include <QObject>
#include <QString>
#include <QVariantList>
#include <QtQml>
#include <cstdint>
#include <memory>
#include <vector>
class QQmlEngine;
@@ -15,37 +17,51 @@ namespace ZShell::llm {
// Syntax highlighting for LLM code blocks via tree-sitter.
//
// The tree-sitter runtime is linked. Grammar libraries
// (libtree-sitter-<lang>.so) are dlopen()'d lazily, so a missing
// grammar package degrades that language to plain text instead of
// breaking the build or the app. Highlight queries are embedded at
// build time (highlight-queries/*.scm, vendored from the grammar
// repos, MIT).
// The tree-sitter runtime is linked. Grammar libraries are dlopen()'d
// lazily, so a missing grammar degrades that language to plain text
// instead of breaking the build or the app. At configure time CMake
// discovers installed grammars (system packages plus the parsers
// Neovim's nvim-treesitter installs) and pairs each with highlight
// queries (vendored, Neovim's, or fetched from
// tree-sitter/highlighting); both are embedded in the generated
// highlight-queries.hpp.
//
// For each grammar the first loadable (ABI-compatible) library wins
// and the first query that compiles against it wins, so a version
// skew between a library and its query degrades gracefully.
//
// The fence language the LLM wrote (```cpp, ```python, ...) is mapped
// to a grammar through an alias table.
// to a grammar through an alias table; tags not in the table are used
// as grammar ids as-is.
//
// highlight() returns a list of span maps:
// highlight() parses the code off the GUI thread and delivers a list of
// span maps by calling target's "onHighlightSpans(token, spans)" method
// (on the GUI thread):
// { "start": int, "length": int, "kind": QString }
// where kind is a semantic role (keyword, string, comment, number,
// function, type, ...) that QML maps to theme colors. An empty list
// means "no highlighting" (unknown language or grammar not installed).
// token is passed back unchanged so the caller can drop results for
// superseded code; a destroyed target is simply skipped.
class CodeHighlighter : public QObject {
Q_OBJECT
QML_ELEMENT
QML_SINGLETON
public:
Q_INVOKABLE QVariantList highlight(const QString& code, const QString& language) const;
Q_INVOKABLE void highlight(
const QString& code, const QString& language, QObject* target, int token);
static CodeHighlighter* create(QQmlEngine*, QJSEngine*);
struct Grammar {
QString lib; // soname to dlopen
QString symbol; // tree_sitter_<lang>() entry point
// Candidate query sources, first wins. Lets typescript reuse
// the javascript query when it compiles against the grammar.
std::vector<int> queries; // indices into querySources()
// Library candidates in priority order (system package, then
// Neovim copies); libs[i] pairs with symbols[i].
std::vector<std::string> libs;
std::vector<std::string> symbols;
// Candidate query sources in priority order; first that
// compiles against the loaded grammar wins.
std::vector<const char*> queries;
};
private:
@@ -57,12 +73,16 @@ class CodeHighlighter : public QObject {
};
[[nodiscard]] static const QHash<QString, QString>& aliases();
[[nodiscard]] const std::vector<std::string>& querySources() const;
// Maps a tree-sitter capture name to a role index (0 = unstyled).
[[nodiscard]] static uint8_t roleFor(const char* name, uint32_t length);
[[nodiscard]] static const char* roleName(uint8_t role);
// The parsing work; runs on worker threads, so the per-language
// state must be initialized under m_stateMutex and is shared as an
// immutable object afterwards.
[[nodiscard]] QVariantList doHighlight(const QString& code, const QString& language) const;
mutable QHash<QString, State> m_states;
mutable QHash<QString, std::shared_ptr<const State>> m_states;
mutable QMutex m_stateMutex;
static CodeHighlighter* s_instance;
};