add markdown parsing + latex + tree-sitter highlighting for codeblocks in llm responses

2026-08-24 23:55:51 +02:00
parent aae3757daf
commit e18875a752
53 changed files with 5043 additions and 618 deletions
@@ -1,17 +1,76 @@
+# cmark-gfm: markdown -> block structure (pkg-config; no CMake config
+# package is installed).
+pkg_check_modules(CMARK_GFM REQUIRED IMPORTED_TARGET libcmark-gfm)
+
+# tree-sitter runtime for code block highlighting (grammars are
+# dlopen()'d at runtime and optional).
+pkg_check_modules(TREE_SITTER REQUIRED IMPORTED_TARGET tree-sitter)
+
+# JKQTMathText (JKQtPlotter) for LaTeX rendering. The config files live
+# in a shared JKQTPlotter6 directory, not one named after the package.
+find_path(JKQTPlotter6_CMAKE_DIR
+	NAMES JKQTMathText6Config.cmake
+	HINTS /usr/lib/cmake/JKQTPlotter6 /usr/local/lib/cmake/JKQTPlotter6
+	DOC "Directory containing the JKQtPlotter cmake package files")
+find_package(JKQTMathText6 REQUIRED PATHS "${JKQTPlotter6_CMAKE_DIR}")
+
+# Embed the vendored highlight queries as C++ string literals.
+set(HIGHLIGHT_QUERY_LANGS c cpp python javascript typescript bash json rust go yaml toml sql)
+set(HIGHLIGHT_QUERIES_HPP "${CMAKE_CURRENT_BINARY_DIR}/highlight-queries.hpp")
+set(_hl_header "#pragma once\n\n// Vendored tree-sitter highlight queries (MIT; see the per-file\n// source headers in the highlight-queries/ directory).\nnamespace ZShell::llm::hq {\n")
+foreach(_hl_lang IN LISTS HIGHLIGHT_QUERY_LANGS)
+	file(READ "${CMAKE_CURRENT_SOURCE_DIR}/highlight-queries/${_hl_lang}.scm" _hl_src)
+	string(APPEND _hl_header "inline constexpr const char* ${_hl_lang} = R\"ZSQUERY(${_hl_src})ZSQUERY\";\n")
+endforeach()
+string(APPEND _hl_header "}\n")
+file(WRITE "${HIGHLIGHT_QUERIES_HPP}" "${_hl_header}")
+
+# Embed the vendored Latin Modern fonts (GUST Font License; provenance
+# in fonts/latinmodern/GUST-FONT-LICENSE.txt) as byte arrays.
+set(LM_FONTS
+	lmroman10-regular
+	lmroman10-italic
+	lmroman10-bold
+	lmroman10-bolditalic
+	latinmodern-math)
+set(LM_FONTS_HPP "${CMAKE_CURRENT_BINARY_DIR}/latinmodern-fonts.hpp")
+set(_lm_header "#pragma once\n\n// Vendored Latin Modern fonts (GUST Font License; see\n// fonts/latinmodern/GUST-FONT-LICENSE.txt).\nnamespace ZShell::llm::lmfont {\n")
+foreach(_lm_font IN LISTS LM_FONTS)
+	string(REPLACE "-" "_" _lm_sym "${_lm_font}")
+	file(READ "${CMAKE_CURRENT_SOURCE_DIR}/fonts/latinmodern/${_lm_font}.otf" _lm_hex HEX)
+	string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," _lm_bytes "${_lm_hex}")
+	string(APPEND _lm_header "inline const unsigned char ${_lm_sym}[] = { ${_lm_bytes} };\n")
+endforeach()
+string(APPEND _lm_header "}\n")
+file(WRITE "${LM_FONTS_HPP}" "${_lm_header}")
+
 qml_module(ZShell-llm
 	URI ZShell.Llm
 	SOURCES
 		chat.hpp chat.cpp
 		chatstore.hpp chatstore.cpp
+		codehighlighter.hpp codehighlighter.cpp
 		generation.hpp generation.cpp
 		llmclient.hpp llmclient.cpp
+		markdownblock.hpp
+		markdownparser.hpp markdownparser.cpp
+		mathtext.hpp mathtext.cpp
 		message.hpp message.cpp
 		messagemodel.hpp messagemodel.cpp
+		segment.hpp segment.cpp
 		session.hpp session.cpp
+		tool.hpp tool.cpp
+		webfetchtool.hpp webfetchtool.cpp
 	LIBRARIES
 		Qt::Network
 		Qt::Sql
+		Qt::Gui
+		Qt::Widgets
 		ZShell-config
+		JKQTPlotter::JKQTMathText
+		PkgConfig::CMARK_GFM
+		PkgConfig::TREE_SITTER
 )

+target_include_directories(ZShell-llm PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 target_include_directories(ZShell-llm PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../Config)