aboutsummaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
authorXi Wang <xi.wang@gmail.com>2011-12-21 05:18:33 -0500
committerDave Airlie <airlied@redhat.com>2011-12-22 07:31:02 -0500
commit8a78389651b3e411ec5a7df61404734f52d6f4eb (patch)
tree02b5e5c5f43fb89c5faab8cd0e3bf5ebee49b7ae /drivers
parent77e00f2ea94abee1ad13bdfde19cf7aa25992b0e (diff)
vmwgfx: fix incorrect VRAM size check in vmw_kms_fb_create()
Commit e133e737 didn't correctly fix the integer overflow issue. - unsigned int required_size; + u64 required_size; ... required_size = mode_cmd->pitch * mode_cmd->height; - if (unlikely(required_size > dev_priv->vram_size)) { + if (unlikely(required_size > (u64) dev_priv->vram_size)) { Note that both pitch and height are u32. Their product is still u32 and would overflow before being assigned to required_size. A correct way is to convert pitch and height to u64 before the multiplication. required_size = (u64)mode_cmd->pitch * (u64)mode_cmd->height; This patch calls the existing vmw_kms_validate_mode_vram() for validation. Signed-off-by: Xi Wang <xi.wang@gmail.com> Reviewed-and-tested-by: Thomas Hellstrom <thellstrom@vmware.com> Signed-off-by: Dave Airlie <airlied@redhat.com>
Diffstat (limited to 'drivers')
-rw-r--r--drivers/gpu/drm/vmwgfx/vmwgfx_kms.c6
1 files changed, 3 insertions, 3 deletions
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 8aa1dbb45c6..f94b33ae221 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1093,7 +1093,6 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
1093 struct vmw_surface *surface = NULL; 1093 struct vmw_surface *surface = NULL;
1094 struct vmw_dma_buffer *bo = NULL; 1094 struct vmw_dma_buffer *bo = NULL;
1095 struct ttm_base_object *user_obj; 1095 struct ttm_base_object *user_obj;
1096 u64 required_size;
1097 int ret; 1096 int ret;
1098 1097
1099 /** 1098 /**
@@ -1102,8 +1101,9 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
1102 * requested framebuffer. 1101 * requested framebuffer.
1103 */ 1102 */
1104 1103
1105 required_size = mode_cmd->pitch * mode_cmd->height; 1104 if (!vmw_kms_validate_mode_vram(dev_priv,
1106 if (unlikely(required_size > (u64) dev_priv->vram_size)) { 1105 mode_cmd->pitch,
1106 mode_cmd->height)) {
1107 DRM_ERROR("VRAM size is too small for requested mode.\n"); 1107 DRM_ERROR("VRAM size is too small for requested mode.\n");
1108 return ERR_PTR(-ENOMEM); 1108 return ERR_PTR(-ENOMEM);
1109 } 1109 }