aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorXi Wang <xi.wang@gmail.com>2011-12-21 05:18:33 -0500
committerDave Airlie <airlied@redhat.com>2011-12-22 12:31:02 +0000
commit8a78389651b3e411ec5a7df61404734f52d6f4eb (patch)
tree02b5e5c5f43fb89c5faab8cd0e3bf5ebee49b7ae
parent77e00f2ea94abee1ad13bdfde19cf7aa25992b0e (diff)
vmwgfx: fix incorrect VRAM size check in vmw_kms_fb_create()
Commit e133e737 didn't correctly fix the integer overflow issue. - unsigned int required_size; + u64 required_size; ... required_size = mode_cmd->pitch * mode_cmd->height; - if (unlikely(required_size > dev_priv->vram_size)) { + if (unlikely(required_size > (u64) dev_priv->vram_size)) { Note that both pitch and height are u32. Their product is still u32 and would overflow before being assigned to required_size. A correct way is to convert pitch and height to u64 before the multiplication. required_size = (u64)mode_cmd->pitch * (u64)mode_cmd->height; This patch calls the existing vmw_kms_validate_mode_vram() for validation. Signed-off-by: Xi Wang <xi.wang@gmail.com> Reviewed-and-tested-by: Thomas Hellstrom <thellstrom@vmware.com> Signed-off-by: Dave Airlie <airlied@redhat.com>
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_kms.c6
1 files changed, 3 insertions, 3 deletions
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 8aa1dbb45c6..f94b33ae221 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1093,7 +1093,6 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
struct vmw_surface *surface = NULL;
struct vmw_dma_buffer *bo = NULL;
struct ttm_base_object *user_obj;
- u64 required_size;
int ret;
/**
@@ -1102,8 +1101,9 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
* requested framebuffer.
*/
- required_size = mode_cmd->pitch * mode_cmd->height;
- if (unlikely(required_size > (u64) dev_priv->vram_size)) {
+ if (!vmw_kms_validate_mode_vram(dev_priv,
+ mode_cmd->pitch,
+ mode_cmd->height)) {
DRM_ERROR("VRAM size is too small for requested mode.\n");
return ERR_PTR(-ENOMEM);
}