From 92600a04683984ca689b238f3d29431985588122 Mon Sep 17 00:00:00 2001 From: Jeongjun Park Date: Sun, 29 Mar 2026 11:57:38 +0900 Subject: [PATCH] drm/vc4: txp: fix writeback dimension checks and normalize rotation If the rotation value is 0, the rotation is compared to an accurate bitmask, which can bypass the size check, and the frame buffer size validation is too weak to reliably reject write buffers that are too small. Therefore, to prevent this, normalize the rotation state, modify the size check, and reject write targets that cannot cover the programmed output size. Signed-off-by: Jeongjun Park --- drivers/gpu/drm/vc4/vc4_txp.c | 42 +++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/vc4/vc4_txp.c b/drivers/gpu/drm/vc4/vc4_txp.c index f739303b2e9ebe..9314121bd90b9a 100644 --- a/drivers/gpu/drm/vc4/vc4_txp.c +++ b/drivers/gpu/drm/vc4/vc4_txp.c @@ -250,7 +250,10 @@ static int vc4_txp_connector_atomic_check(struct drm_connector *conn, { struct drm_connector_state *conn_state; struct drm_crtc_state *crtc_state; + struct drm_display_mode *mode; struct drm_framebuffer *fb; + unsigned int rotation; + unsigned int exp_w, exp_h; int i; conn_state = drm_atomic_get_new_connector_state(state, conn); @@ -258,21 +261,31 @@ static int vc4_txp_connector_atomic_check(struct drm_connector *conn, return 0; crtc_state = drm_atomic_get_new_crtc_state(state, conn_state->crtc); - + mode = &crtc_state->adjusted_mode; fb = conn_state->writeback_job->fb; - if ((conn_state->rotation == DRM_MODE_ROTATE_0 && - fb->width != crtc_state->mode.hdisplay && - fb->height != crtc_state->mode.vdisplay) || - (conn_state->rotation == (DRM_MODE_ROTATE_0 | DRM_MODE_TRANSPOSE) && - fb->width != crtc_state->mode.vdisplay && - fb->height != crtc_state->mode.hdisplay)) { - DRM_DEBUG_KMS("Invalid framebuffer size %ux%u vs mode %ux%u\n", - fb->width, fb->height, - crtc_state->mode.hdisplay, crtc_state->mode.vdisplay); + + rotation = drm_rotation_simplify(conn_state->rotation, + DRM_MODE_ROTATE_0 | + DRM_MODE_TRANSPOSE); + + if (rotation & ~(DRM_MODE_ROTATE_0 | DRM_MODE_TRANSPOSE)) return -EINVAL; + + if (rotation & DRM_MODE_TRANSPOSE) { + exp_w = mode->vdisplay; + exp_h = mode->hdisplay; + } else { + exp_w = mode->hdisplay; + exp_h = mode->vdisplay; } - if (conn_state->rotation & DRM_MODE_TRANSPOSE && + if (fb->width != exp_w || fb->height != exp_h) { + DRM_DEBUG_KMS("Invalid framebuffer size %ux%u vs expected %ux%u\n", + fb->width, fb->height, exp_w, exp_h); + return -EINVAL; + } + + if (rotation & DRM_MODE_TRANSPOSE && (fb->format->format == DRM_FORMAT_RGB888 || fb->format->format == DRM_FORMAT_BGR888)) { DRM_DEBUG_KMS("24bpp formats not supported when transposing\n"); @@ -309,6 +322,7 @@ static void vc4_txp_connector_atomic_commit(struct drm_connector *conn, struct drm_framebuffer *fb; unsigned int hdisplay; unsigned int vdisplay; + unsigned int rotation; dma_addr_t addr; u32 ctrl; int idx; @@ -343,7 +357,11 @@ static void vc4_txp_connector_atomic_commit(struct drm_connector *conn, */ ctrl |= TXP_ALPHA_INVERT; - if (conn_state->rotation & DRM_MODE_TRANSPOSE) + rotation = drm_rotation_simplify(conn_state->rotation, + DRM_MODE_ROTATE_0 | + DRM_MODE_TRANSPOSE); + + if (rotation & DRM_MODE_TRANSPOSE) ctrl |= TXP_TRANSPOSE; if (!drm_dev_enter(drm, &idx))