async parsing + tex caching
This commit is contained in:
@@ -14,15 +14,253 @@ find_path(JKQTPlotter6_CMAKE_DIR
|
||||
DOC "Directory containing the JKQtPlotter cmake package files")
|
||||
find_package(JKQTMathText6 REQUIRED PATHS "${JKQTPlotter6_CMAKE_DIR}")
|
||||
|
||||
# Embed the vendored highlight queries as C++ string literals.
|
||||
set(HIGHLIGHT_QUERY_LANGS c cpp python javascript typescript bash json rust go yaml toml sql)
|
||||
set(HIGHLIGHT_QUERIES_HPP "${CMAKE_CURRENT_BINARY_DIR}/highlight-queries.hpp")
|
||||
set(_hl_header "#pragma once\n\n// Vendored tree-sitter highlight queries (MIT; see the per-file\n// source headers in the highlight-queries/ directory).\nnamespace ZShell::llm::hq {\n")
|
||||
foreach(_hl_lang IN LISTS HIGHLIGHT_QUERY_LANGS)
|
||||
file(READ "${CMAKE_CURRENT_SOURCE_DIR}/highlight-queries/${_hl_lang}.scm" _hl_src)
|
||||
string(APPEND _hl_header "inline constexpr const char* ${_hl_lang} = R\"ZSQUERY(${_hl_src})ZSQUERY\";\n")
|
||||
# --- tree-sitter grammar discovery (configure time) ---
|
||||
#
|
||||
# Discover installed tree-sitter grammars — system packages
|
||||
# (libtree-sitter-<lang>.so) and the parsers Neovim's nvim-treesitter
|
||||
# installs (~/.local/share/nvim/site/parser/*.so) — and pair each with
|
||||
# highlight queries. Query sources, in priority order:
|
||||
# 1. vendored highlight-queries/*.scm (version-pinned; see the
|
||||
# per-file source headers, MIT),
|
||||
# 2. Neovim's own queries (version-matched to its parsers),
|
||||
# 3. tree-sitter/highlighting from GitHub (cached in the build dir).
|
||||
# The result is embedded as highlight-queries.hpp. Re-run cmake to pick
|
||||
# up grammars installed later.
|
||||
#
|
||||
# Candidate entry points are read from the .so with `nm` rather than
|
||||
# assumed to be tree_sitter_<id>; a missing entry point just makes that
|
||||
# candidate fail at runtime.
|
||||
function(_ts_entry_point out file)
|
||||
# OUTPUT_VARIABLE + OUTPUT_QUIET loses the output on CMake 4, so
|
||||
# capture through a temp file.
|
||||
get_filename_component(_ts_nm_base "${file}" NAME)
|
||||
set(_ts_nm_file "${CMAKE_CURRENT_BINARY_DIR}/ts-entry-${_ts_nm_base}")
|
||||
execute_process(
|
||||
COMMAND nm -D --defined-only "${file}"
|
||||
RESULT_VARIABLE _ts_nm_rc
|
||||
OUTPUT_FILE "${_ts_nm_file}"
|
||||
ERROR_FILE "${_ts_nm_file}.err")
|
||||
set(_ts_sym "tree_sitter_missing")
|
||||
if(_ts_nm_rc EQUAL 0 AND EXISTS "${_ts_nm_file}")
|
||||
file(READ "${_ts_nm_file}" _ts_nm_out)
|
||||
file(REMOVE "${_ts_nm_file}" "${_ts_nm_file}.err")
|
||||
# The library also exports the external scanner functions; the
|
||||
# entry point is the one without the _external suffix.
|
||||
string(REGEX MATCHALL " T tree_sitter_[A-Za-z0-9_]+" _ts_syms "${_ts_nm_out}")
|
||||
foreach(_ts_s IN LISTS _ts_syms)
|
||||
string(REPLACE " T " "" _ts_s "${_ts_s}")
|
||||
if(NOT _ts_s MATCHES "_external")
|
||||
set(_ts_sym "${_ts_s}")
|
||||
break()
|
||||
endif()
|
||||
endforeach()
|
||||
endif()
|
||||
set(${out} "${_ts_sym}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
# Append a query text to <files> (PARENT_SCOPE) unless it duplicates a
|
||||
# hash in <hashes>. The text is written to the build tree right away:
|
||||
# the s-expression `;` comments would split CMake list items and the
|
||||
# query `(` `)` break bracket arguments, so query text only ever lives
|
||||
# in variables and files, never in lists.
|
||||
# Expand Neovim's "; inherits:" directives by appending the inherited
|
||||
# query set's highlights file (recursively; a visited set prevents
|
||||
# cycles). Several languages are stubs that inherit the real query
|
||||
# (html inherits html_tags, qmljs inherits ecma, ...).
|
||||
function(_ts_expand_inherits query_dir text outVar)
|
||||
set(result "${text}")
|
||||
set(_visited "")
|
||||
set(_depth 0)
|
||||
while(_depth LESS 8)
|
||||
# Do not match the leading `;`: a MATCHALL result that itself
|
||||
# contains a semicolon is re-split into list items.
|
||||
string(REGEX MATCHALL "inherits:[ \t]*[A-Za-z0-9_]+" _inh "${result}")
|
||||
if(NOT _inh)
|
||||
break()
|
||||
endif()
|
||||
set(_added FALSE)
|
||||
foreach(_entry IN LISTS _inh)
|
||||
string(REGEX REPLACE "^inherits:[ \t]*" "" _name "${_entry}")
|
||||
set(_file "${query_dir}/${_name}/highlights.scm")
|
||||
if(NOT EXISTS "${_file}" OR _file IN_LIST _visited)
|
||||
continue()
|
||||
endif()
|
||||
list(APPEND _visited "${_file}")
|
||||
file(READ "${_file}" _inh_text)
|
||||
string(APPEND result "\n${_inh_text}")
|
||||
set(_added TRUE)
|
||||
endforeach()
|
||||
if(NOT _added)
|
||||
break()
|
||||
endif()
|
||||
math(EXPR _depth "${_depth} + 1")
|
||||
endwhile()
|
||||
set(${outVar} "${result}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
function(_ts_add_query sid text filesVar hashesVar)
|
||||
# filesVar/hashesVar hold the caller's variable names.
|
||||
set(files "${${filesVar}}")
|
||||
set(hashes "${${hashesVar}}")
|
||||
string(SHA256 _ts_qh "${text}")
|
||||
if(_ts_qh IN_LIST hashes)
|
||||
return()
|
||||
endif()
|
||||
list(APPEND hashes "${_ts_qh}")
|
||||
list(LENGTH files _ts_qi)
|
||||
set(_ts_qfile "${CMAKE_CURRENT_BINARY_DIR}/ts-queries/${sid}_${_ts_qi}.scm")
|
||||
file(WRITE "${_ts_qfile}" "${text}")
|
||||
list(APPEND files "${_ts_qfile}")
|
||||
set(${filesVar} "${files}" PARENT_SCOPE)
|
||||
set(${hashesVar} "${hashes}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
|
||||
set(_ts_candidates "") # entries: <id>|<cmake-safe id>|<lib>|<symbol>
|
||||
file(GLOB _ts_sys_files
|
||||
"/usr/lib/libtree-sitter-*.so" "/usr/local/lib/libtree-sitter-*.so")
|
||||
foreach(_ts_file IN LISTS _ts_sys_files)
|
||||
get_filename_component(_ts_name "${_ts_file}" NAME)
|
||||
string(REGEX REPLACE "^libtree-sitter-(.+)\.so$" "\\1" _ts_id "${_ts_name}")
|
||||
string(REPLACE "-" "_" _ts_sid "${_ts_id}")
|
||||
_ts_entry_point(_ts_sym "${_ts_file}")
|
||||
list(APPEND _ts_candidates
|
||||
"${_ts_id}|${_ts_sid}|libtree-sitter-${_ts_id}.so|${_ts_sym}")
|
||||
endforeach()
|
||||
string(APPEND _hl_header "}\n")
|
||||
set(_ts_nvim_parser_dirs
|
||||
"$ENV{HOME}/.local/share/nvim/site/parser"
|
||||
"$ENV{HOME}/.local/share/nvim/runtime/parser"
|
||||
"$ENV{HOME}/.local/share/nvim/lazy/nvim-treesitter/parser")
|
||||
set(_ts_nvim_query_dirs
|
||||
"$ENV{HOME}/.local/share/nvim/site/queries"
|
||||
"$ENV{HOME}/.local/share/nvim/lazy/nvim-treesitter/runtime/queries")
|
||||
foreach(_ts_dir IN LISTS _ts_nvim_parser_dirs)
|
||||
file(GLOB _ts_dir_files "${_ts_dir}/*.so")
|
||||
foreach(_ts_file IN LISTS _ts_dir_files)
|
||||
get_filename_component(_ts_name "${_ts_file}" NAME)
|
||||
string(REGEX REPLACE "\\.so$" "" _ts_id "${_ts_name}")
|
||||
string(REPLACE "-" "_" _ts_sid "${_ts_id}")
|
||||
_ts_entry_point(_ts_sym "${_ts_file}")
|
||||
list(APPEND _ts_candidates
|
||||
"${_ts_id}|${_ts_sid}|${_ts_file}|${_ts_sym}")
|
||||
endforeach()
|
||||
endforeach()
|
||||
|
||||
set(_ts_ids "")
|
||||
foreach(_ts_c IN LISTS _ts_candidates)
|
||||
string(REPLACE "|" ";" _ts_parts "${_ts_c}")
|
||||
list(GET _ts_parts 0 _ts_id)
|
||||
if(NOT _ts_id IN_LIST _ts_ids)
|
||||
list(APPEND _ts_ids "${_ts_id}")
|
||||
endif()
|
||||
endforeach()
|
||||
list(SORT _ts_ids)
|
||||
|
||||
set(HIGHLIGHT_QUERIES_HPP "${CMAKE_CURRENT_BINARY_DIR}/highlight-queries.hpp")
|
||||
set(_hl_header
|
||||
"#pragma once\n\n// Generated by CMake. Discoverd tree-sitter grammars and their\n// highlight queries: vendored highlight-queries/*.scm (MIT), Neovim\n// nvim-treesitter queries, and tree-sitter/highlighting (MIT).\nnamespace ZShell::llm::hq {\nstruct Candidate { const char* lib; const char* symbol; };\nstruct Grammar { const char* id; int nCandidates; const Candidate* candidates; int nQueries; const char* const* queries; };\n")
|
||||
set(_ts_grammar_rows "")
|
||||
set(_ts_vendored
|
||||
c cpp python javascript typescript tsx bash json rust go yaml toml sql)
|
||||
foreach(_ts_id IN LISTS _ts_ids)
|
||||
string(REPLACE "-" "_" _ts_sid "${_ts_id}")
|
||||
|
||||
# Query candidates in priority order (deduped; identical texts are
|
||||
# skipped, the nvim site and plugin copies are usually the same
|
||||
# file).
|
||||
set(_ts_qfiles "")
|
||||
set(_ts_qhashes "")
|
||||
if(_ts_id STREQUAL "cpp")
|
||||
# The C++ grammar is a superset of C and its query only covers
|
||||
# the C++ delta; base C coverage comes from the C query.
|
||||
file(READ "${CMAKE_CURRENT_SOURCE_DIR}/highlight-queries/c.scm" _ts_qa)
|
||||
file(READ "${CMAKE_CURRENT_SOURCE_DIR}/highlight-queries/cpp.scm" _ts_qb)
|
||||
_ts_add_query(${_ts_sid} "${_ts_qa}\n${_ts_qb}" _ts_qfiles _ts_qhashes)
|
||||
elseif(_ts_id STREQUAL "typescript" OR _ts_id STREQUAL "tsx")
|
||||
# The TS grammars reuse the JS node names; the JS query usually
|
||||
# compiles against them and gives full coverage.
|
||||
file(READ "${CMAKE_CURRENT_SOURCE_DIR}/highlight-queries/javascript.scm" _ts_qa)
|
||||
file(READ "${CMAKE_CURRENT_SOURCE_DIR}/highlight-queries/typescript.scm" _ts_qb)
|
||||
_ts_add_query(${_ts_sid} "${_ts_qa}\n${_ts_qb}" _ts_qfiles _ts_qhashes)
|
||||
_ts_add_query(${_ts_sid} "${_ts_qb}" _ts_qfiles _ts_qhashes)
|
||||
elseif(_ts_id IN_LIST _ts_vendored)
|
||||
file(READ
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/highlight-queries/${_ts_id}.scm" _ts_q)
|
||||
_ts_add_query(${_ts_sid} "${_ts_q}" _ts_qfiles _ts_qhashes)
|
||||
endif()
|
||||
foreach(_ts_qd IN LISTS _ts_nvim_query_dirs)
|
||||
if(EXISTS "${_ts_qd}/${_ts_id}/highlights.scm")
|
||||
file(READ "${_ts_qd}/${_ts_id}/highlights.scm" _ts_q)
|
||||
_ts_expand_inherits("${_ts_qd}" "${_ts_q}" _ts_q)
|
||||
_ts_add_query(${_ts_sid} "${_ts_q}" _ts_qfiles _ts_qhashes)
|
||||
endif()
|
||||
endforeach()
|
||||
if(NOT _ts_qfiles)
|
||||
# Last resort: fetch from the tree-sitter/highlighting repo.
|
||||
# Version-skewed against locally installed grammars, so only
|
||||
# used when nothing local exists. Cached; offline builds simply
|
||||
# drop the language.
|
||||
set(_ts_dl "${CMAKE_CURRENT_BINARY_DIR}/ts-query-downloads/${_ts_sid}.scm")
|
||||
if(NOT EXISTS "${_ts_dl}")
|
||||
file(DOWNLOAD
|
||||
"https://raw.githubusercontent.com/tree-sitter/highlighting/main/queries/${_ts_id}/highlight.scm"
|
||||
"${_ts_dl}" STATUS _ts_dl_status TIMEOUT 30)
|
||||
list(GET _ts_dl_status 0 _ts_dl_rc)
|
||||
if(NOT _ts_dl_rc EQUAL 0)
|
||||
file(REMOVE "${_ts_dl}")
|
||||
endif()
|
||||
endif()
|
||||
if(EXISTS "${_ts_dl}")
|
||||
file(READ "${_ts_dl}" _ts_q)
|
||||
_ts_add_query(${_ts_sid} "${_ts_q}" _ts_qfiles _ts_qhashes)
|
||||
endif()
|
||||
endif()
|
||||
list(LENGTH _ts_qfiles _ts_nq)
|
||||
if(_ts_nq EQUAL 0)
|
||||
continue()
|
||||
endif()
|
||||
|
||||
# Emit query sources.
|
||||
set(_ts_qn 0)
|
||||
set(_ts_q_ptrs "")
|
||||
foreach(_ts_qfile IN LISTS _ts_qfiles)
|
||||
file(READ "${_ts_qfile}" _ts_q)
|
||||
string(APPEND _hl_header
|
||||
"inline constexpr const char* q_${_ts_sid}_${_ts_qn} = R\"ZSQUERY(${_ts_q})ZSQUERY\";\n")
|
||||
string(APPEND _ts_q_ptrs "q_${_ts_sid}_${_ts_qn}, ")
|
||||
math(EXPR _ts_qn "${_ts_qn} + 1")
|
||||
endforeach()
|
||||
string(APPEND _hl_header
|
||||
"inline constexpr const char* const q_${_ts_sid}[] = { ${_ts_q_ptrs} };\n")
|
||||
|
||||
# Emit library candidates (system package first, then Neovim).
|
||||
string(APPEND _hl_header "inline constexpr Candidate cand_${_ts_sid}[] = {\n")
|
||||
set(_ts_nc 0)
|
||||
foreach(_ts_c IN LISTS _ts_candidates)
|
||||
string(REPLACE "|" ";" _ts_parts "${_ts_c}")
|
||||
list(GET _ts_parts 0 _ts_cid)
|
||||
if(_ts_cid STREQUAL _ts_id)
|
||||
list(GET _ts_parts 2 _ts_lib)
|
||||
list(GET _ts_parts 3 _ts_sym)
|
||||
string(APPEND _hl_header
|
||||
" { R\"ZSLIB(${_ts_lib})ZSLIB\", R\"ZSSYM(${_ts_sym})ZSSYM\" },\n")
|
||||
math(EXPR _ts_nc "${_ts_nc} + 1")
|
||||
endif()
|
||||
endforeach()
|
||||
string(APPEND _hl_header "};\n")
|
||||
if(_ts_nc EQUAL 0)
|
||||
# Queries but no library: pointless, drop the language.
|
||||
string(APPEND _hl_header "") # (arrays stay; grammar row is skipped)
|
||||
continue()
|
||||
endif()
|
||||
|
||||
list(APPEND _ts_grammar_rows
|
||||
"{ \"${_ts_id}\", ${_ts_nc}, cand_${_ts_sid}, ${_ts_nq}, q_${_ts_sid} },")
|
||||
endforeach()
|
||||
string(APPEND _hl_header "inline constexpr Grammar grammars[] = {\n")
|
||||
foreach(_ts_row IN LISTS _ts_grammar_rows)
|
||||
string(APPEND _hl_header " ${_ts_row}\n")
|
||||
endforeach()
|
||||
string(APPEND _hl_header "};\n}\n")
|
||||
file(WRITE "${HIGHLIGHT_QUERIES_HPP}" "${_hl_header}")
|
||||
|
||||
# Embed the vendored Latin Modern fonts (GUST Font License; provenance
|
||||
|
||||
Reference in New Issue
Block a user