feat: skip two-way copy for cuda devices when decoding video wallpapers + cache wallpapers as blobs in new database

Plugins/ZShell/Internal/CMakeLists.txt

@@ -22,11 +22,13 @@ qml_module(ZShell-internal
 		linearindicatormanager.hpp linearindicatormanager.cpp
 		strokecanvasitem.hpp strokecanvasitem.cpp
 		strokecanvasrenderer.hpp strokecanvasrenderer.cpp
+		videoplayer.hpp videoplayer.cpp
 		cursor.hpp
 	LIBRARIES
 		Qt::Gui
 		Qt::Quick
 		Qt::Concurrent
+		Qt::Sql
 		Qt::Core
 		Qt::Network
 		Qt::DBus
@@ -40,4 +42,20 @@ qml_module(ZShell-internal
 		PkgConfig::WAYLAND_EGL
 		PkgConfig::GL
 		PkgConfig::MPV
+		PkgConfig::AVCODEC
+		PkgConfig::AVFORMAT
+		PkgConfig::AVUTIL
 )
+
+qt_add_resources(ZShell-internal "wallpaper_shaders"
+	PREFIX "/"
+	FILES
+		shaders/wallpaper.vert
+		shaders/wallpaper.frag
+		shaders/wallpaper_video.frag
+)
+
+if(CUDAToolkit_FOUND)
+	target_link_libraries(ZShell-internal PRIVATE CUDA::cuda_driver)
+	target_compile_definitions(ZShell-internal PRIVATE ZHAVE_VIDEO=1)
+endif()

Plugins/ZShell/Internal/shaders/wallpaper.frag

@@ -0,0 +1,11 @@
+uniform sampler2D source;
+uniform vec2 size;
+uniform vec4 crop;
+uniform float alpha;
+out vec4 outColor;
+
+void main() {
+	vec2 p = vec2(gl_FragCoord.x, size.y - gl_FragCoord.y) / size;
+	vec4 c = texture(source, crop.xy + p * crop.zw);
+	outColor = vec4(c.rgb, c.a * alpha);
+}

Plugins/ZShell/Internal/shaders/wallpaper.vert

@@ -0,0 +1,4 @@
+void main() {
+	vec2 corner = vec2(float((gl_VertexID << 1) & 2), float(gl_VertexID & 2));
+	gl_Position = vec4(corner * 2.0 - 1.0, 0.0, 1.0);
+}

Plugins/ZShell/Internal/shaders/wallpaper_video.frag

@@ -0,0 +1,105 @@
+uniform sampler2D luma;
+uniform sampler2D chroma;
+uniform vec2 size;
+uniform vec2 srcSize;
+uniform vec4 crop;
+uniform float alpha;
+uniform int matrix;
+uniform int range;
+uniform vec2 chromaOff;
+out vec4 outColor;
+
+float noise(vec2 p) {
+	return fract(sin(dot(p, vec2(12.9898, 78.233))) * 43758.5453);
+}
+
+const float PI = 3.1415926535893;
+
+float sinc(float x) {
+	x *= PI;
+	return abs(x) < 1e-6 ? 1.0 : sin(x) / x;
+}
+
+float lanczos(float x) {
+	if (abs(x) >= 3.0)
+		return 0.0;
+	return sinc(x) * sinc(x / 3.0);
+}
+
+float hermite(float x) {
+	float a = abs(x);
+	if (a >= 1.0)
+		return 0.0;
+	return 1.0 - 3.0 * a * a + 2.0 * a * a * a;
+}
+
+vec4 resample(sampler2D tex, vec2 plane, vec2 dest, vec2 uv) {
+	vec2 pt = 1.0 / plane;
+	float wx[6];
+	float wy[6];
+	vec2 base;
+
+	int upx = dest.x >= plane.x ? 1 : 0;
+	int tx = upx == 1 ? 6 : 2;
+	float fsx = max(1.0, plane.x / dest.x);
+	float f = fract(uv.x * plane.x - 0.5);
+	base.x = uv.x - f * pt.x - pt.x * (float(tx) * 0.5 - 1.0);
+	float sx = 0.0;
+	for (int i = 0; i < 6; ++i) {
+		if (i >= tx) { wx[i] = 0.0; continue; }
+		float x = f - (float(i) - float(tx) * 0.5 + 1.0);
+		wx[i] = upx == 1 ? lanczos(x / fsx) : hermite(x / fsx);
+		sx += wx[i];
+	}
+	if (sx != 0.0)
+		for (int i = 0; i < 6; ++i) wx[i] /= sx;
+
+	int upy = dest.y >= plane.y ? 1 : 0;
+	int ty = upy == 1 ? 6 : 2;
+	float fsy = max(1.0, plane.y / dest.y);
+	f = fract(uv.y * plane.y - 0.5);
+	base.y = uv.y - f * pt.y - pt.y * (float(ty) * 0.5 - 1.0);
+	float sy = 0.0;
+	for (int i = 0; i < 6; ++i) {
+		if (i >= ty) { wy[i] = 0.0; continue; }
+		float y = f - (float(i) - float(ty) * 0.5 + 1.0);
+		wy[i] = upy == 1 ? lanczos(y / fsy) : hermite(y / fsy);
+		sy += wy[i];
+	}
+	if (sy != 0.0)
+		for (int i = 0; i < 6; ++i) wy[i] /= sy;
+
+	vec4 col = vec4(0.0);
+	for (int i = 0; i < 6; ++i) {
+		if (wx[i] == 0.0)
+			continue;
+		for (int j = 0; j < 6; ++j) {
+			if (wy[j] == 0.0)
+				continue;
+			vec2 c = base + vec2(float(i) * pt.x, float(j) * pt.y);
+			col += (wx[i] * wy[j]) * texture(tex, c);
+		}
+	}
+	return col;
+}
+
+void main() {
+	vec2 p = vec2(gl_FragCoord.x, size.y - gl_FragCoord.y) / size;
+	vec2 uv = crop.xy + p * crop.zw;
+	float Yv = resample(luma, srcSize, size, uv).r;
+	vec2 ch = resample(chroma, srcSize * 0.5, size, uv + chromaOff).rg;
+	vec3 yuv;
+	if (range == 0) {
+		yuv = vec3((Yv - 16.0 / 255.0) / (219.0 / 255.0),
+			(ch.r - 128.0 / 255.0) / (224.0 / 255.0),
+			(ch.g - 128.0 / 255.0) / (224.0 / 255.0));
+	} else {
+		yuv = vec3(Yv, ch.r - 0.5, ch.g - 0.5);
+	}
+	mat3 m = matrix == 1
+		? mat3(1.0, 1.0, 1.0, 0.0, -0.187324, 1.855600, 1.574800, -0.468124, 0.0)
+		: mat3(1.0, 1.0, 1.0, 0.0, -0.344136, 1.772000, 1.402000, -0.714136, 0.0);
+	vec3 rgb = m * yuv;
+	rgb += (noise(gl_FragCoord.xy) - 0.5) / 255.0;
+	outColor = vec4(clamp(rgb, 0.0, 1.0), alpha);
+}

Plugins/ZShell/Internal/videoplayer.cpp

@@ -0,0 +1,529 @@
+#include "videoplayer.hpp"
+
+#ifdef ZHAVE_VIDEO
+
+#define EGL_EGLEXT_PROTOTYPES
+#define GL_GLEXT_PROTOTYPES
+
+#include <EGL/egl.h>
+#include <GL/gl.h>
+#include <GL/glext.h>
+#include <cuda.h>
+
+#include <algorithm>
+#include <chrono>
+#include <climits>
+
+#include <QtLogging>
+
+extern "C" CUresult cuGraphicsGLRegisterImage(
+	CUgraphicsResource* resource,
+	GLuint image,
+	GLenum target,
+	unsigned int flags);
+
+Q_LOGGING_CATEGORY(videoLog, "ZShell.Internal.wallpaper")
+
+namespace {
+
+enum AVPixelFormat chooseFormat(
+	AVCodecContext* ctx, const AVPixelFormat* pix_fmts) {
+	(void)ctx;
+	for (const AVPixelFormat* p = pix_fmts; *p != AV_PIX_FMT_NONE; ++p)
+		if (*p == AV_PIX_FMT_CUDA) return AV_PIX_FMT_CUDA;
+	return pix_fmts[0];
+}
+
+} // namespace
+
+VideoPlayer::VideoPlayer(EGLDisplay display) : m_eglDisplay(display) {}
+
+VideoPlayer::~VideoPlayer() {
+	stop();
+
+	std::lock_guard<std::mutex> lock(m_mtx);
+	if (m_pending) {
+		av_frame_unref(m_pending);
+		av_frame_free(&m_pending);
+	}
+	if (m_displayed) {
+		av_frame_unref(m_displayed);
+		av_frame_free(&m_displayed);
+	}
+
+	if (m_cuCtx) {
+		CUcontext prev = nullptr;
+		cuCtxPushCurrent(m_cuCtx);
+		if (m_resY) {
+			cuGraphicsUnregisterResource(m_resY);
+			m_resY = nullptr;
+		}
+		if (m_resUV) {
+			cuGraphicsUnregisterResource(m_resUV);
+			m_resUV = nullptr;
+		}
+		if (m_texY) {
+			glDeleteTextures(1, &m_texY);
+			m_texY = 0;
+		}
+		if (m_texUV) {
+			glDeleteTextures(1, &m_texUV);
+			m_texUV = 0;
+		}
+		cuCtxPopCurrent(&prev);
+		cuCtxDestroy(m_cuCtx);
+		m_cuCtx = nullptr;
+	}
+	m_texSize = QSize();
+}
+
+void VideoPlayer::setReadyCallback(std::function<void()> callback) {
+	m_readyCb = std::move(callback);
+}
+
+void VideoPlayer::setFrameCallback(std::function<void()> callback) {
+	m_frameCb = std::move(callback);
+}
+
+bool VideoPlayer::openDecoder(const QString& path) {
+	if (avformat_open_input(
+			&m_fmt, path.toUtf8().constData(), nullptr, nullptr) < 0) {
+		qCWarning(
+			videoLog,
+			"VideoPlayer: could not open %s",
+			path.toUtf8().constData());
+		return false;
+	}
+	if (avformat_find_stream_info(m_fmt, nullptr) < 0) {
+		qCWarning(videoLog, "VideoPlayer: could not read stream info");
+		return false;
+	}
+
+	m_streamIndex = -1;
+	for (unsigned i = 0; i < m_fmt->nb_streams; ++i)
+		if (m_fmt->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+			m_streamIndex = int(i);
+			break;
+		}
+	if (m_streamIndex < 0) {
+		qCWarning(videoLog, "VideoPlayer: no video stream");
+		return false;
+	}
+
+	AVCodecParameters* par = m_fmt->streams[m_streamIndex]->codecpar;
+	const AVCodec* codec = avcodec_find_decoder(par->codec_id);
+	const AVCodec* hwcodec = nullptr;
+	void* it = nullptr;
+	while (const AVCodec* c = av_codec_iterate(&it)) {
+		if (c->id != par->codec_id || !av_codec_is_decoder(c)) continue;
+		for (int i = 0;
+			 const AVCodecHWConfig* cfg = avcodec_get_hw_config(c, i);
+			 ++i)
+			if (cfg->device_type == AV_HWDEVICE_TYPE_CUDA &&
+				(cfg->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) {
+				hwcodec = c;
+				break;
+			}
+		if (hwcodec) break;
+	}
+	if (hwcodec) codec = hwcodec;
+	if (!codec) {
+		qCWarning(
+			videoLog,
+			"VideoPlayer: no decoder for codec id %d",
+			int(par->codec_id));
+		return false;
+	}
+
+	m_codec = avcodec_alloc_context3(codec);
+	if (!m_codec) return false;
+	if (avcodec_parameters_to_context(m_codec, par) < 0) {
+		qCWarning(videoLog, "VideoPlayer: could not copy codec params");
+		return false;
+	}
+
+	m_hwDeviceCtx = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_CUDA);
+	if (!m_hwDeviceCtx) {
+		qCWarning(
+			videoLog, "VideoPlayer: could not allocate cuda hwdevice context");
+		return false;
+	}
+	AVCUDADeviceContext* cudaHw = reinterpret_cast<AVCUDADeviceContext*>(
+		reinterpret_cast<AVHWDeviceContext*>(m_hwDeviceCtx->data)->hwctx);
+	if (m_cuCtx) cudaHw->cuda_ctx = m_cuCtx;
+	if (av_hwdevice_ctx_init(m_hwDeviceCtx) < 0) {
+		qCWarning(
+			videoLog, "VideoPlayer: could not create cuda hwdevice context");
+		av_buffer_unref(&m_hwDeviceCtx);
+		return false;
+	}
+	m_codec->hw_device_ctx = av_buffer_ref(m_hwDeviceCtx);
+	m_codec->get_format = chooseFormat;
+
+	if (avcodec_open2(m_codec, codec, nullptr) < 0) {
+		qCWarning(videoLog, "VideoPlayer: could not open codec");
+		return false;
+	}
+
+	return true;
+}
+
+void VideoPlayer::closeDecoder() {
+	if (m_codec) {
+		avcodec_free_context(&m_codec);
+		m_codec = nullptr;
+	}
+	if (m_hwDeviceCtx) {
+		av_buffer_unref(&m_hwDeviceCtx);
+		m_hwDeviceCtx = nullptr;
+	}
+	if (m_fmt) {
+		avformat_close_input(&m_fmt);
+		m_fmt = nullptr;
+	}
+	m_streamIndex = -1;
+}
+
+void VideoPlayer::decodeLoop(const QString& path) {
+	if (!openDecoder(path)) {
+		m_threadRunning = false;
+		return;
+	}
+
+	AVRational fr = m_fmt->streams[m_streamIndex]->avg_frame_rate;
+	if (fr.num <= 0 || fr.den <= 0) fr = m_codec->framerate;
+	if (fr.num <= 0 || fr.den <= 0) fr = AVRational{30, 1};
+	const int64_t intervalUs =
+		av_rescale_q(1, AVRational{fr.den, fr.num}, AVRational{1, 1000000});
+
+	AVFrame* frame = av_frame_alloc();
+	if (!frame) {
+		m_threadRunning = false;
+		return;
+	}
+
+	AVPacket pkt{};
+
+	while (m_threadRunning) {
+		int ret = av_read_frame(m_fmt, &pkt);
+		if (ret == AVERROR_EOF) {
+			if (avformat_seek_file(
+					m_fmt, m_streamIndex, INT64_MIN, 0, INT64_MAX, 0) < 0) {
+				qCWarning(videoLog, "VideoPlayer: loop seek failed; stopping");
+				break;
+			}
+			avcodec_flush_buffers(m_codec);
+			continue;
+		}
+		if (ret < 0) {
+			qCWarning(videoLog, "VideoPlayer: read error; stopping");
+			break;
+		}
+		if (pkt.stream_index != m_streamIndex) {
+			av_packet_unref(&pkt);
+			continue;
+		}
+
+		if (avcodec_send_packet(m_codec, &pkt) < 0) {
+			av_packet_unref(&pkt);
+			continue;
+		}
+		av_packet_unref(&pkt);
+
+		while (m_threadRunning && avcodec_receive_frame(m_codec, frame) == 0) {
+			bool wasEmpty = false;
+			{
+				std::lock_guard<std::mutex> lock(m_mtx);
+				if (!m_pending) m_pending = av_frame_alloc();
+				av_frame_unref(m_pending);
+				if (av_frame_ref(m_pending, frame) >= 0) {
+					wasEmpty = !m_haveFrame;
+					m_haveFrame = true;
+				}
+			}
+			if (wasEmpty && m_readyCb) m_readyCb();
+			if (m_frameCb) m_frameCb();
+			av_frame_unref(frame);
+		}
+
+		std::this_thread::sleep_for(std::chrono::microseconds(intervalUs));
+	}
+
+	av_frame_free(&frame);
+	closeDecoder();
+	m_threadRunning = false;
+}
+
+bool VideoPlayer::start(const QString& path) {
+	stop();
+
+	if (!ensureCudaContext()) return false;
+
+	m_threadRunning = true;
+	m_thread = std::thread([this, path]() { decodeLoop(path); });
+	return true;
+}
+
+void VideoPlayer::stop() {
+	m_threadRunning = false;
+	if (m_thread.joinable()) m_thread.join();
+
+	closeDecoder();
+
+	std::lock_guard<std::mutex> lock(m_mtx);
+	if (m_pending) {
+		av_frame_unref(m_pending);
+		m_pending = nullptr;
+	}
+	if (m_displayed) {
+		av_frame_unref(m_displayed);
+		m_displayed = nullptr;
+	}
+	m_haveFrame = false;
+	m_size = QSize();
+}
+
+QSize VideoPlayer::frameSize() const {
+	std::lock_guard<std::mutex> lock(m_mtx);
+	return m_size;
+}
+
+void VideoPlayer::deleteTexture() {
+	std::lock_guard<std::mutex> lock(m_mtx);
+	CUcontext prev = nullptr;
+	if (m_cuCtx) cuCtxPushCurrent(m_cuCtx);
+	if (m_resY) {
+		cuGraphicsUnregisterResource(m_resY);
+		m_resY = nullptr;
+	}
+	if (m_resUV) {
+		cuGraphicsUnregisterResource(m_resUV);
+		m_resUV = nullptr;
+	}
+	if (m_texY) {
+		glDeleteTextures(1, &m_texY);
+		m_texY = 0;
+	}
+	if (m_texUV) {
+		glDeleteTextures(1, &m_texUV);
+		m_texUV = 0;
+	}
+	if (m_cuCtx) cuCtxPopCurrent(&prev);
+	m_texSize = QSize();
+}
+
+bool VideoPlayer::ensureCudaContext() {
+	if (m_cuCtx) return true;
+
+	if (cuInit(0) != CUDA_SUCCESS) {
+		qCWarning(videoLog, "VideoPlayer: cuInit failed");
+		return false;
+	}
+	CUdevice dev = 0;
+	if (cuDeviceGet(&dev, 0) != CUDA_SUCCESS) {
+		qCWarning(videoLog, "VideoPlayer: cuDeviceGet failed");
+		return false;
+	}
+	if (cuCtxCreate_v4(&m_cuCtx, nullptr, 0, dev) != CUDA_SUCCESS) {
+		m_cuCtx = nullptr;
+		qCWarning(videoLog, "VideoPlayer: cuCtxCreate failed");
+		return false;
+	}
+	CUcontext popped = nullptr;
+	cuCtxPopCurrent(&popped);
+	return true;
+}
+
+bool VideoPlayer::ensureTextures(unsigned w, unsigned h, int compBytes) {
+	if (!ensureCudaContext()) return false;
+	CUcontext prev = nullptr;
+	cuCtxPushCurrent(m_cuCtx);
+
+	const QSize sz = QSize(int(w), int(h));
+	if (m_texSize == sz && m_texComp == compBytes && m_texY != 0) {
+		cuCtxPopCurrent(&prev);
+		return true;
+	}
+
+	if (m_resY) cuGraphicsUnregisterResource(m_resY);
+	if (m_resUV) cuGraphicsUnregisterResource(m_resUV);
+	m_resY = m_resUV = nullptr;
+	if (m_texY) glDeleteTextures(1, &m_texY);
+	if (m_texUV) glDeleteTextures(1, &m_texUV);
+	m_texY = m_texUV = 0;
+
+	const bool wide = compBytes == 2;
+	auto makeTex =
+		[&](GLuint& tex, GLint internal, GLenum fmt, unsigned tw, unsigned th) {
+			glGenTextures(1, &tex);
+			glBindTexture(GL_TEXTURE_2D, tex);
+			glTexImage2D(
+				GL_TEXTURE_2D,
+				0,
+				internal,
+				GLint(tw),
+				GLint(th),
+				0,
+				fmt,
+				wide ? GL_UNSIGNED_SHORT : GL_UNSIGNED_BYTE,
+				nullptr);
+			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+		};
+	makeTex(m_texY, wide ? GL_R16 : GL_R8, GL_RED, w, h);
+	makeTex(m_texUV, wide ? GL_RG16 : GL_RG8, GL_RG, w / 2, h / 2);
+
+	CUresult r = cuGraphicsGLRegisterImage(
+		&m_resY,
+		m_texY,
+		GL_TEXTURE_2D,
+		CU_GRAPHICS_REGISTER_FLAGS_WRITE_DISCARD);
+	if (r != CUDA_SUCCESS) {
+		const char* es = nullptr;
+		cuGetErrorString(r, &es);
+		qCWarning(
+			videoLog,
+			"VideoPlayer: register luma tex failed (%s)",
+			es ? es : "?");
+		m_texSize = QSize();
+		cuCtxPopCurrent(&prev);
+		return false;
+	}
+	r = cuGraphicsGLRegisterImage(
+		&m_resUV,
+		m_texUV,
+		GL_TEXTURE_2D,
+		CU_GRAPHICS_REGISTER_FLAGS_WRITE_DISCARD);
+	if (r != CUDA_SUCCESS) {
+		qCWarning(videoLog, "VideoPlayer: register chroma tex failed");
+		cuGraphicsUnregisterResource(m_resY);
+		m_resY = nullptr;
+		m_texSize = QSize();
+		cuCtxPopCurrent(&prev);
+		return false;
+	}
+
+	m_texSize = sz;
+	m_texComp = compBytes;
+	cuCtxPopCurrent(&prev);
+	return true;
+}
+
+bool VideoPlayer::importCurrentFrame() {
+	AVFrame* frame = nullptr;
+	{
+		std::lock_guard<std::mutex> lock(m_mtx);
+		if (!m_pending || m_pending->format == AV_PIX_FMT_NONE) return false;
+		frame = m_pending;
+		m_pending = nullptr;
+	}
+
+	if (frame->format != AV_PIX_FMT_CUDA || !frame->hw_frames_ctx) {
+		static int logged = 0;
+		if (logged++ < 3)
+			qCWarning(
+				videoLog,
+				"VideoPlayer: frame not on GPU (fmt=%s hw_frames=%d)",
+				av_get_pix_fmt_name(AVPixelFormat(frame->format)),
+				frame->hw_frames_ctx != nullptr);
+		av_frame_unref(frame);
+		return false;
+	}
+
+	const AVHWFramesContext* hf =
+		reinterpret_cast<const AVHWFramesContext*>(frame->hw_frames_ctx->data);
+	const AVPixelFormat sw = hf->sw_format;
+	const bool tenBit = sw == AV_PIX_FMT_P010LE || sw == AV_PIX_FMT_P010BE ||
+						sw == AV_PIX_FMT_P012LE || sw == AV_PIX_FMT_P016LE;
+	const int compBytes = tenBit ? 2 : 1;
+	const size_t compSize = size_t(compBytes);
+
+	const unsigned w = unsigned(frame->width);
+	const unsigned h = unsigned(frame->height);
+	if (w == 0 || h == 0) {
+		av_frame_unref(frame);
+		return false;
+	}
+
+	size_t pitchY = size_t(frame->linesize[0]);
+	size_t pitchUV = frame->linesize[1] > 0 ? size_t(frame->linesize[1])
+											: pitchY;
+	if (pitchY == 0) pitchY = size_t(w) * compSize;
+
+	switch (frame->colorspace) {
+	case AVCOL_SPC_BT470BG:
+	case AVCOL_SPC_SMPTE170M:
+		m_matrix = 0;
+		break;
+	case AVCOL_SPC_BT709:
+		m_matrix = 1;
+		break;
+	default:
+		m_matrix = h >= 720 ? 1 : 0;
+		break;
+	}
+	m_range = frame->color_range == AVCOL_RANGE_JPEG ? 1 : 0;
+	m_size = QSize(int(w), int(h));
+
+	if (!ensureTextures(w, h, compBytes)) {
+		av_frame_unref(frame);
+		return false;
+	}
+
+	CUcontext prev = nullptr;
+	cuCtxPushCurrent(m_cuCtx);
+	CUgraphicsResource res[2] = {m_resY, m_resUV};
+	CUresult r = cuGraphicsMapResources(2, res, nullptr);
+	if (r != CUDA_SUCCESS) {
+		cuCtxPopCurrent(&prev);
+		av_frame_unref(frame);
+		qCWarning(videoLog, "VideoPlayer: cuGraphicsMapResources failed");
+		return false;
+	}
+
+	CUarray arrY = nullptr, arrUV = nullptr;
+	cuGraphicsSubResourceGetMappedArray(&arrY, m_resY, 0, 0);
+	cuGraphicsSubResourceGetMappedArray(&arrUV, m_resUV, 0, 0);
+
+	const size_t dstPitch = size_t(w) * compSize;
+	CUDA_MEMCPY2D mc{};
+	mc.srcMemoryType = CU_MEMORYTYPE_DEVICE;
+	mc.dstMemoryType = CU_MEMORYTYPE_ARRAY;
+	mc.srcDevice = CUdeviceptr(frame->data[0]);
+	mc.srcPitch = pitchY;
+	mc.dstArray = arrY;
+	mc.dstPitch = dstPitch;
+	mc.WidthInBytes = std::min(pitchY, dstPitch);
+	mc.Height = h;
+	r = cuMemcpy2D(&mc);
+	if (r == CUDA_SUCCESS) {
+		mc.srcDevice = CUdeviceptr(frame->data[1]);
+		mc.srcPitch = pitchUV;
+		mc.dstArray = arrUV;
+		mc.dstPitch = dstPitch;
+		mc.WidthInBytes = std::min(pitchUV, dstPitch);
+		mc.Height = h / 2;
+		r = cuMemcpy2D(&mc);
+	}
+
+	cuGraphicsUnmapResources(2, res, nullptr);
+	cuCtxPopCurrent(&prev);
+
+	if (r != CUDA_SUCCESS) {
+		av_frame_unref(frame);
+		qCWarning(videoLog, "VideoPlayer: cuMemcpy2D failed");
+		return false;
+	}
+
+	{
+		std::lock_guard<std::mutex> lock(m_mtx);
+		if (m_displayed) av_frame_unref(m_displayed);
+		m_displayed = frame;
+	}
+
+	return true;
+}
+
+#endif

Plugins/ZShell/Internal/videoplayer.hpp

@@ -0,0 +1,87 @@
+#pragma once
+
+#include <EGL/egl.h>
+
+#include <cuda.h>
+
+#include <atomic>
+#include <functional>
+#include <mutex>
+#include <thread>
+
+#include <QSize>
+#include <QString>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/buffer.h>
+#include <libavutil/frame.h>
+#include <libavutil/hwcontext.h>
+#include <libavutil/hwcontext_cuda.h>
+#include <libavutil/pixfmt.h>
+#include <libavutil/pixdesc.h>
+}
+
+class VideoPlayer {
+	public:
+	explicit VideoPlayer(EGLDisplay display);
+	~VideoPlayer();
+
+	void setReadyCallback(std::function<void()> callback);
+	void setFrameCallback(std::function<void()> callback);
+
+	bool start(const QString& path);
+	void stop();
+
+
+	bool importCurrentFrame();
+
+	void deleteTexture();
+
+	[[nodiscard]] uint32_t textureId() const { return m_texY; }
+	[[nodiscard]] uint32_t chromaTextureId() const { return m_texUV; }
+	[[nodiscard]] int colorMatrix() const { return m_matrix; }
+	[[nodiscard]] int colorRange() const { return m_range; }
+	[[nodiscard]] QSize frameSize() const;
+
+	private:
+	void decodeLoop(const QString& path);
+	bool openDecoder(const QString& path);
+	void closeDecoder();
+
+	bool ensureTextures(unsigned w, unsigned h, int compBytes);
+
+	bool ensureCudaContext();
+
+	EGLDisplay m_eglDisplay;
+	std::function<void()> m_readyCb;
+	std::function<void()> m_frameCb;
+
+	std::thread m_thread;
+	std::atomic<bool> m_threadRunning{false};
+
+	mutable std::mutex m_mtx;
+	AVFrame* m_pending = nullptr;
+	bool m_haveFrame = false;
+
+	AVFrame* m_displayed = nullptr;
+
+	AVFormatContext* m_fmt = nullptr;
+	AVCodecContext* m_codec = nullptr;
+	AVBufferRef* m_hwDeviceCtx = nullptr;
+	int m_streamIndex = -1;
+
+	CUcontext m_cuCtx = nullptr;
+	CUgraphicsResource m_resY = nullptr;
+	CUgraphicsResource m_resUV = nullptr;
+	QSize m_texSize;
+
+	uint32_t m_texY = 0;
+	uint32_t m_texUV = 0;
+
+	int m_matrix = 1;
+	int m_range = 0;
+	int m_texComp = 1;
+	QSize m_size;
+};

Plugins/ZShell/Internal/wallpaperservice.hpp

@@ -18,6 +18,11 @@
 #include <QtQml/qqml.h>
 #include <qtmetamacros.h>
 #include <atomic>
+#include <memory>
+
+#ifdef ZHAVE_VIDEO
+#include "videoplayer.hpp"
+#endif
 
 #include <qwayland-wlr-layer-shell-unstable-v1.h>
 #include <wayland-client-protocol.h>
@@ -39,6 +44,10 @@ class WallpaperService : public QObject {
 
 	Q_PROPERTY(QString path READ path WRITE setPath NOTIFY pathChanged REQUIRED)
 	Q_PROPERTY(QVariantMap crops READ crops WRITE setCrops NOTIFY cropsChanged)
+	Q_PROPERTY(QString dbPath READ dbPath WRITE setDbPath NOTIFY dbPathChanged)
+	Q_PROPERTY(
+		QSize largestBufferSize READ largestBufferSize NOTIFY
+			largestBufferSizeChanged)
 
 	public:
 	explicit WallpaperService(QObject* parent = nullptr);
@@ -47,6 +56,11 @@ class WallpaperService : public QObject {
 	[[nodiscard]] QString path() const;
 	void setPath(const QString& path);
 
+	[[nodiscard]] QSize largestBufferSize() const;
+
+	[[nodiscard]] QString dbPath() const;
+	void setDbPath(const QString& path);
+
 	void addScreen(const QString& screen);
 	void removeScreen(const QString& screen);
 
@@ -57,9 +71,13 @@ class WallpaperService : public QObject {
 
 	Q_INVOKABLE void setCrop(const QString& screen, const QVariantMap& crop);
 
+	Q_INVOKABLE void invalidateCache();
+
 	signals:
 	void pathChanged();
 	void cropsChanged();
+	void dbPathChanged();
+	void largestBufferSizeChanged();
 
 	private:
 	friend class WallpaperSurface;
@@ -77,7 +95,6 @@ class WallpaperService : public QObject {
 	void applyPath(const QString& path);
 
 	void startVideo(const QString& path);
-	void stopVideo();
 	void teardownMpv();
 
 	void initMpvRender();
@@ -87,6 +104,23 @@ class WallpaperService : public QObject {
 	static void* getMpvProcAddress(void* ctx, const char* name);
 	static void mpvUpdate(void* ctx);
 
+#ifdef ZHAVE_VIDEO
+	bool startPlayerVideo(const QString& path);
+	void stopPlayerVideo();
+	[[nodiscard]] uint32_t createVideoProgram();
+	void drawVideoTexture(
+		uint32_t lumaTex,
+		uint32_t chromaTex,
+		int matrix,
+		int range,
+		const QRectF& rect,
+		float sizeW,
+		float sizeH,
+		float srcW,
+		float srcH,
+		float alpha);
+#endif
+
 	void bindGlobals();
 	static void registryGlobal(
 		void* data,
@@ -111,16 +145,18 @@ class WallpaperService : public QObject {
 		float sizeH,
 		float alpha);
 
-	bool decode(const QString& path);
+	[[nodiscard]] QImage decodeImage(
+		const QString& path, const QSize& decodeSize) const;
 
-	bool ensureDecodeSize();
+	[[nodiscard]] QByteArray cachedBlob(const QString& path) const;
 
-	[[nodiscard]] QSize largestBufferSize();
+	void ensureImage();
+
+	void notifyBufferSizes();
 
 	void queue(WallpaperSurface* surface);
 	void queueAll();
 
-	void startDrawing();
 	void loop();
 
 	void present();
@@ -143,6 +179,7 @@ class WallpaperService : public QObject {
 	void addSurface(const QString& screen);
 
 	QString m_path;
+	QString m_dbPath;
 	QStringList m_screens;
 	QVariantMap m_crops;
 
@@ -160,9 +197,15 @@ class WallpaperService : public QObject {
 
 	QProcess* m_transcode = nullptr;
 
-	QSize m_sourceSize;
 	QSize m_decodeSize;
 
+	QImage m_stagedImage;
+	QString m_stagedPath;
+	bool m_stagedReady = false;
+	bool m_decoding = false;
+	QString m_decodePath;
+	bool m_replacementReady = false;
+
 	wl_display* m_display = nullptr;
 	wl_registry* m_registry = nullptr;
 	wl_compositor* m_compositor = nullptr;
@@ -178,10 +221,29 @@ class WallpaperService : public QObject {
 	int m_sizeUniform = -1;
 	int m_cropUniform = -1;
 	int m_alphaUniform = -1;
+	int m_sourceUniform = -1;
+
+	uint32_t m_videoProgram = 0;
+	uint32_t m_vao = 0;
+	int m_videoSizeUniform = -1;
+	int m_videoSrcSizeUniform = -1;
+	int m_videoCropUniform = -1;
+	int m_videoAlphaUniform = -1;
+	int m_videoLumaUniform = -1;
+	int m_videoChromaUniform = -1;
+	int m_videoMatrixUniform = -1;
+	int m_videoRangeUniform = -1;
+	int m_videoChromaOffUniform = -1;
+
+#ifdef ZHAVE_VIDEO
+	std::unique_ptr<VideoPlayer> m_player;
+	bool m_usePlayer = true;
+#endif
 
 	QImage m_previousImage;
 	QVariantMap m_previousCrops;
 	uint32_t m_previousTexture = 0;
+	std::vector<uint32_t> m_orphanTextures;
 	bool m_fading = false;
 	bool m_fadeClockStarted = false;
 	bool m_firstFadeDone = false;

Plugins/ZShell/Internal/wallpapersurface.cpp

@@ -185,6 +185,7 @@ void WallpaperSurface::updateBuffer() {
 
 	requestFrame();
 	m_service->queue(this);
+	m_service->notifyBufferSizes();
 }
 
 } // namespace ZShell::internal