drm/vmwgfx: Prune modes based on available VRAM size

This needs to be reviewed once we support screen objects and don't rely
on VRAM for the frame-buffer.

Also fix some integer overflow issues pointed out by Michel Daenzer.

Signed-off-by: Thomas Hellstrom <thellstrom@vmware.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>

1 files changed, 19 insertions, 9 deletions

diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index 11cb39e3accb..a01c47ddb5bc 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -427,7 +427,9 @@ static int vmw_ldu_connector_fill_modes(struct drm_connector *connector,
 {
         struct vmw_legacy_display_unit *ldu = vmw_connector_to_ldu(connector);
         struct drm_device *dev = connector->dev;
+        struct vmw_private *dev_priv = vmw_priv(dev);
         struct drm_display_mode *mode = NULL;
+        struct drm_display_mode *bmode;
         struct drm_display_mode prefmode = { DRM_MODE("preferred",
                 DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -443,22 +445,30 @@ static int vmw_ldu_connector_fill_modes(struct drm_connector *connector,
                 mode->hdisplay = ldu->pref_width;
                 mode->vdisplay = ldu->pref_height;
                 mode->vrefresh = drm_mode_vrefresh(mode);
-                drm_mode_probed_add(connector, mode);
+                if (vmw_kms_validate_mode_vram(dev_priv, mode->hdisplay * 2,
+                                               mode->vdisplay)) {
+                        drm_mode_probed_add(connector, mode);
 
-                if (ldu->pref_mode) {
-                        list_del_init(&ldu->pref_mode->head);
-                        drm_mode_destroy(dev, ldu->pref_mode);
-                }
+                        if (ldu->pref_mode) {
+                                list_del_init(&ldu->pref_mode->head);
+                                drm_mode_destroy(dev, ldu->pref_mode);
+                        }
 
-                ldu->pref_mode = mode;
+                        ldu->pref_mode = mode;
+                }
         }
 
         for (i = 0; vmw_ldu_connector_builtin[i].type != 0; i++) {
-                if (vmw_ldu_connector_builtin[i].hdisplay > max_width ||
-                    vmw_ldu_connector_builtin[i].vdisplay > max_height)
+                bmode = &vmw_ldu_connector_builtin[i];
+                if (bmode->hdisplay > max_width ||
+                    bmode->vdisplay > max_height)
+                        continue;
+
+                if (!vmw_kms_validate_mode_vram(dev_priv, bmode->hdisplay * 2,
+                                                bmode->vdisplay))
                         continue;
 
-                mode = drm_mode_duplicate(dev, &vmw_ldu_connector_builtin[i]);
+                mode = drm_mode_duplicate(dev, bmode);
                 if (!mode)
                         return 0;
                 mode->vrefresh = drm_mode_vrefresh(mode);