drm/vmwgfx: Prune modes based on available VRAM size

This needs to be reviewed once we support screen objects and don't rely
on VRAM for the frame-buffer.

Also fix some integer overflow issues pointed out by Michel Daenzer.

Signed-off-by: Thomas Hellstrom <thellstrom@vmware.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>

1 files changed, 9 insertions, 2 deletions

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 82bd3d8c0e4f..5fb68f3981a2 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -838,7 +838,7 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
         struct vmw_framebuffer *vfb = NULL;
         struct vmw_surface *surface = NULL;
         struct vmw_dma_buffer *bo = NULL;
-        unsigned int required_size;
+        u64 required_size;
         int ret;
 
         /**
@@ -848,7 +848,7 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
          */
 
         required_size = mode_cmd->pitch * mode_cmd->height;
-        if (unlikely(required_size > dev_priv->vram_size)) {
+        if (unlikely(required_size > (u64) dev_priv->vram_size)) {
                 DRM_ERROR("VRAM size is too small for requested mode.\n");
                 return NULL;
         }
@@ -1133,6 +1133,13 @@ out_unlock:
         return ret;
 }
 
+bool vmw_kms_validate_mode_vram(struct vmw_private *dev_priv,
+                                uint32_t pitch,
+                                uint32_t height)
+{
+        return ((u64) pitch * (u64) height) < (u64) dev_priv->vram_size;
+}
+
 u32 vmw_get_vblank_counter(struct drm_device *dev, int crtc)
 {
         return 0;