atmel_lcdfb: disallow setting larger resolution than the framebuffer memory can handle

Signed-off-by: Stanislaw Gruszka <stf_xl@wp.pl>
Acked-by: Nicolas Ferre <nicolas.ferre@atmel.com>
Cc: Krzysztof Helt <krzysztof.h1@wp.pl>
Cc: Haavard Skinnemoen <hskinnemoen@atmel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 7 insertions, 0 deletions

diff --git a/drivers/video/atmel_lcdfb.c b/drivers/video/atmel_lcdfb.c
index d38fd5217422..f8d0a57a07cb 100644
--- a/drivers/video/atmel_lcdfb.c
+++ b/drivers/video/atmel_lcdfb.c
@@ -372,6 +372,13 @@ static int atmel_lcdfb_check_var(struct fb_var_screeninfo *var,
         var->transp.offset = var->transp.length = 0;
         var->xoffset = var->yoffset = 0;
 
+        if (info->fix.smem_len) {
+                unsigned int smem_len = (var->xres_virtual * var->yres_virtual
+                                         * ((var->bits_per_pixel + 7) / 8));
+                if (smem_len > info->fix.smem_len)
+                        return -EINVAL;
+        }
+
         /* Saturate vertical and horizontal timings at maximum values */
         var->vsync_len = min_t(u32, var->vsync_len,
                         (ATMEL_LCDC_VPW >> ATMEL_LCDC_VPW_OFFSET) + 1);