1 files changed, 239 insertions, 140 deletions

diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index 5708c056fa1b..034297640b48 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -55,12 +55,74 @@
  */
 #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
 
-/*
- * Clear vblank timestamp buffer for a crtc.
+static bool
+drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
+                          struct timeval *tvblank, unsigned flags);
+
+/**
+ * drm_update_vblank_count - update the master vblank counter
+ * @dev: DRM device
+ * @crtc: counter to update
+ *
+ * Call back into the driver to update the appropriate vblank counter
+ * (specified by @crtc).  Deal with wraparound, if it occurred, and
+ * update the last read value so we can deal with wraparound on the next
+ * call if necessary.
+ *
+ * Only necessary when going from off->on, to account for frames we
+ * didn't get an interrupt for.
+ *
+ * Note: caller must hold dev->vbl_lock since this reads & writes
+ * device vblank fields.
  */
-static void clear_vblank_timestamps(struct drm_device *dev, int crtc)
+static void drm_update_vblank_count(struct drm_device *dev, int crtc)
 {
-        memset(dev->vblank[crtc].time, 0, sizeof(dev->vblank[crtc].time));
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+        u32 cur_vblank, diff, tslot;
+        bool rc;
+        struct timeval t_vblank;
+
+        /*
+         * Interrupts were disabled prior to this call, so deal with counter
+         * wrap if needed.
+         * NOTE!  It's possible we lost a full dev->max_vblank_count events
+         * here if the register is small or we had vblank interrupts off for
+         * a long time.
+         *
+         * We repeat the hardware vblank counter & timestamp query until
+         * we get consistent results. This to prevent races between gpu
+         * updating its hardware counter while we are retrieving the
+         * corresponding vblank timestamp.
+         */
+        do {
+                cur_vblank = dev->driver->get_vblank_counter(dev, crtc);
+                rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0);
+        } while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc));
+
+        /* Deal with counter wrap */
+        diff = cur_vblank - vblank->last;
+        if (cur_vblank < vblank->last) {
+                diff += dev->max_vblank_count;
+
+                DRM_DEBUG("last_vblank[%d]=0x%x, cur_vblank=0x%x => diff=0x%x\n",
+                          crtc, vblank->last, cur_vblank, diff);
+        }
+
+        DRM_DEBUG("updating vblank count on crtc %d, missed %d\n",
+                  crtc, diff);
+
+        /* Reinitialize corresponding vblank timestamp if high-precision query
+         * available. Skip this step if query unsupported or failed. Will
+         * reinitialize delayed at next vblank interrupt in that case.
+         */
+        if (rc) {
+                tslot = atomic_read(&vblank->count) + diff;
+                vblanktimestamp(dev, crtc, tslot) = t_vblank;
+        }
+
+        smp_mb__before_atomic();
+        atomic_add(diff, &vblank->count);
+        smp_mb__after_atomic();
 }
 
 /*
@@ -71,10 +133,11 @@ static void clear_vblank_timestamps(struct drm_device *dev, int crtc)
  */
 static void vblank_disable_and_save(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         unsigned long irqflags;
         u32 vblcount;
         s64 diff_ns;
-        int vblrc;
+        bool vblrc;
         struct timeval tvblank;
         int count = DRM_TIMESTAMP_MAXRETRIES;
 
@@ -84,8 +147,28 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc)
          */
         spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
 
+        /*
+         * If the vblank interrupt was already disbled update the count
+         * and timestamp to maintain the appearance that the counter
+         * has been ticking all along until this time. This makes the
+         * count account for the entire time between drm_vblank_on() and
+         * drm_vblank_off().
+         *
+         * But only do this if precise vblank timestamps are available.
+         * Otherwise we might read a totally bogus timestamp since drivers
+         * lacking precise timestamp support rely upon sampling the system clock
+         * at vblank interrupt time. Which obviously won't work out well if the
+         * vblank interrupt is disabled.
+         */
+        if (!vblank->enabled &&
+            drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0)) {
+                drm_update_vblank_count(dev, crtc);
+                spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+                return;
+        }
+
         dev->driver->disable_vblank(dev, crtc);
-        dev->vblank[crtc].enabled = false;
+        vblank->enabled = false;
 
         /* No further vblank irq's will be processed after
          * this point. Get current hardware vblank count and
@@ -100,9 +183,9 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc)
          * delayed gpu counter increment.
          */
         do {
-                dev->vblank[crtc].last = dev->driver->get_vblank_counter(dev, crtc);
+                vblank->last = dev->driver->get_vblank_counter(dev, crtc);
                 vblrc = drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0);
-        } while (dev->vblank[crtc].last != dev->driver->get_vblank_counter(dev, crtc) && (--count) && vblrc);
+        } while (vblank->last != dev->driver->get_vblank_counter(dev, crtc) && (--count) && vblrc);
 
         if (!count)
                 vblrc = 0;
@@ -110,7 +193,7 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc)
         /* Compute time difference to stored timestamp of last vblank
          * as updated by last invocation of drm_handle_vblank() in vblank irq.
          */
-        vblcount = atomic_read(&dev->vblank[crtc].count);
+        vblcount = atomic_read(&vblank->count);
         diff_ns = timeval_to_ns(&tvblank) -
                   timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));
 
@@ -126,14 +209,18 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc)
          * available. In that case we can't account for this and just
          * hope for the best.
          */
-        if ((vblrc > 0) && (abs64(diff_ns) > 1000000)) {
-                atomic_inc(&dev->vblank[crtc].count);
+        if (vblrc && (abs64(diff_ns) > 1000000)) {
+                /* Store new timestamp in ringbuffer. */
+                vblanktimestamp(dev, crtc, vblcount + 1) = tvblank;
+
+                /* Increment cooked vblank count. This also atomically commits
+                 * the timestamp computed above.
+                 */
+                smp_mb__before_atomic();
+                atomic_inc(&vblank->count);
                 smp_mb__after_atomic();
         }
 
-        /* Invalidate all timestamps while vblank irq's are off. */
-        clear_vblank_timestamps(dev, crtc);
-
         spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
 }
 
@@ -164,14 +251,20 @@ static void vblank_disable_fn(unsigned long arg)
 void drm_vblank_cleanup(struct drm_device *dev)
 {
         int crtc;
+        unsigned long irqflags;
 
         /* Bail if the driver didn't call drm_vblank_init() */
         if (dev->num_crtcs == 0)
                 return;
 
         for (crtc = 0; crtc < dev->num_crtcs; crtc++) {
-                del_timer_sync(&dev->vblank[crtc].disable_timer);
-                vblank_disable_fn((unsigned long)&dev->vblank[crtc]);
+                struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+
+                del_timer_sync(&vblank->disable_timer);
+
+                spin_lock_irqsave(&dev->vbl_lock, irqflags);
+                vblank_disable_and_save(dev, crtc);
+                spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
         }
 
         kfree(dev->vblank);
@@ -204,11 +297,13 @@ int drm_vblank_init(struct drm_device *dev, int num_crtcs)
                 goto err;
 
         for (i = 0; i < num_crtcs; i++) {
-                dev->vblank[i].dev = dev;
-                dev->vblank[i].crtc = i;
-                init_waitqueue_head(&dev->vblank[i].queue);
-                setup_timer(&dev->vblank[i].disable_timer, vblank_disable_fn,
-                            (unsigned long)&dev->vblank[i]);
+                struct drm_vblank_crtc *vblank = &dev->vblank[i];
+
+                vblank->dev = dev;
+                vblank->crtc = i;
+                init_waitqueue_head(&vblank->queue);
+                setup_timer(&vblank->disable_timer, vblank_disable_fn,
+                            (unsigned long)vblank);
         }
 
         DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
@@ -224,7 +319,7 @@ int drm_vblank_init(struct drm_device *dev, int num_crtcs)
         return 0;
 
 err:
-        drm_vblank_cleanup(dev);
+        dev->num_crtcs = 0;
         return ret;
 }
 EXPORT_SYMBOL(drm_vblank_init);
@@ -360,9 +455,11 @@ int drm_irq_uninstall(struct drm_device *dev)
         if (dev->num_crtcs) {
                 spin_lock_irqsave(&dev->vbl_lock, irqflags);
                 for (i = 0; i < dev->num_crtcs; i++) {
-                        wake_up(&dev->vblank[i].queue);
-                        dev->vblank[i].enabled = false;
-                        dev->vblank[i].last =
+                        struct drm_vblank_crtc *vblank = &dev->vblank[i];
+
+                        wake_up(&vblank->queue);
+                        vblank->enabled = false;
+                        vblank->last =
                                 dev->driver->get_vblank_counter(dev, i);
                 }
                 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
@@ -617,7 +714,7 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
          * within vblank area, counting down the number of lines until
          * start of scanout.
          */
-        invbl = vbl_status & DRM_SCANOUTPOS_INVBL;
+        invbl = vbl_status & DRM_SCANOUTPOS_IN_VBLANK;
 
         /* Convert scanout position into elapsed time at raw_time query
          * since start of scanout at first display scanline. delta_ns
@@ -647,7 +744,7 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
 
         vbl_status = DRM_VBLANKTIME_SCANOUTPOS_METHOD;
         if (invbl)
-                vbl_status |= DRM_VBLANKTIME_INVBL;
+                vbl_status |= DRM_VBLANKTIME_IN_VBLANK;
 
         return vbl_status;
 }
@@ -679,10 +776,11 @@ static struct timeval get_drm_timestamp(void)
  * call, i.e., it isn't very precisely locked to the true vblank.
  *
  * Returns:
- * Non-zero if timestamp is considered to be very precise, zero otherwise.
+ * True if timestamp is considered to be very precise, false otherwise.
  */
-u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
-                              struct timeval *tvblank, unsigned flags)
+static bool
+drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
+                          struct timeval *tvblank, unsigned flags)
 {
         int ret;
 
@@ -694,7 +792,7 @@ u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
                 ret = dev->driver->get_vblank_timestamp(dev, crtc, &max_error,
                                                         tvblank, flags);
                 if (ret > 0)
-                        return (u32) ret;
+                        return true;
         }
 
         /* GPU high precision timestamp query unsupported or failed.
@@ -702,9 +800,8 @@ u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
          */
         *tvblank = get_drm_timestamp();
 
-        return 0;
+        return false;
 }
-EXPORT_SYMBOL(drm_get_last_vbltimestamp);
 
 /**
  * drm_vblank_count - retrieve "cooked" vblank counter value
@@ -720,9 +817,11 @@ EXPORT_SYMBOL(drm_get_last_vbltimestamp);
  */
 u32 drm_vblank_count(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+
         if (WARN_ON(crtc >= dev->num_crtcs))
                 return 0;
-        return atomic_read(&dev->vblank[crtc].count);
+        return atomic_read(&vblank->count);
 }
 EXPORT_SYMBOL(drm_vblank_count);
 
@@ -742,6 +841,7 @@ EXPORT_SYMBOL(drm_vblank_count);
 u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc,
                               struct timeval *vblanktime)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         u32 cur_vblank;
 
         if (WARN_ON(crtc >= dev->num_crtcs))
@@ -753,10 +853,10 @@ u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc,
          * a seqlock.
          */
         do {
-                cur_vblank = atomic_read(&dev->vblank[crtc].count);
+                cur_vblank = atomic_read(&vblank->count);
                 *vblanktime = vblanktimestamp(dev, crtc, cur_vblank);
                 smp_rmb();
-        } while (cur_vblank != atomic_read(&dev->vblank[crtc].count));
+        } while (cur_vblank != atomic_read(&vblank->count));
 
         return cur_vblank;
 }
@@ -805,83 +905,20 @@ void drm_send_vblank_event(struct drm_device *dev, int crtc,
 EXPORT_SYMBOL(drm_send_vblank_event);
 
 /**
- * drm_update_vblank_count - update the master vblank counter
- * @dev: DRM device
- * @crtc: counter to update
- *
- * Call back into the driver to update the appropriate vblank counter
- * (specified by @crtc).  Deal with wraparound, if it occurred, and
- * update the last read value so we can deal with wraparound on the next
- * call if necessary.
- *
- * Only necessary when going from off->on, to account for frames we
- * didn't get an interrupt for.
- *
- * Note: caller must hold dev->vbl_lock since this reads & writes
- * device vblank fields.
- */
-static void drm_update_vblank_count(struct drm_device *dev, int crtc)
-{
-        u32 cur_vblank, diff, tslot, rc;
-        struct timeval t_vblank;
-
-        /*
-         * Interrupts were disabled prior to this call, so deal with counter
-         * wrap if needed.
-         * NOTE!  It's possible we lost a full dev->max_vblank_count events
-         * here if the register is small or we had vblank interrupts off for
-         * a long time.
-         *
-         * We repeat the hardware vblank counter & timestamp query until
-         * we get consistent results. This to prevent races between gpu
-         * updating its hardware counter while we are retrieving the
-         * corresponding vblank timestamp.
-         */
-        do {
-                cur_vblank = dev->driver->get_vblank_counter(dev, crtc);
-                rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0);
-        } while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc));
-
-        /* Deal with counter wrap */
-        diff = cur_vblank - dev->vblank[crtc].last;
-        if (cur_vblank < dev->vblank[crtc].last) {
-                diff += dev->max_vblank_count;
-
-                DRM_DEBUG("last_vblank[%d]=0x%x, cur_vblank=0x%x => diff=0x%x\n",
-                          crtc, dev->vblank[crtc].last, cur_vblank, diff);
-        }
-
-        DRM_DEBUG("enabling vblank interrupts on crtc %d, missed %d\n",
-                  crtc, diff);
-
-        /* Reinitialize corresponding vblank timestamp if high-precision query
-         * available. Skip this step if query unsupported or failed. Will
-         * reinitialize delayed at next vblank interrupt in that case.
-         */
-        if (rc) {
-                tslot = atomic_read(&dev->vblank[crtc].count) + diff;
-                vblanktimestamp(dev, crtc, tslot) = t_vblank;
-        }
-
-        smp_mb__before_atomic();
-        atomic_add(diff, &dev->vblank[crtc].count);
-        smp_mb__after_atomic();
-}
-
-/**
  * drm_vblank_enable - enable the vblank interrupt on a CRTC
  * @dev: DRM device
  * @crtc: CRTC in question
  */
 static int drm_vblank_enable(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         int ret = 0;
 
         assert_spin_locked(&dev->vbl_lock);
 
         spin_lock(&dev->vblank_time_lock);
 
-        if (!dev->vblank[crtc].enabled) {
+        if (!vblank->enabled) {
                 /*
                  * Enable vblank irqs under vblank_time_lock protection.
                  * All vblank count & timestamp updates are held off
@@ -892,9 +929,9 @@ static int drm_vblank_enable(struct drm_device *dev, int crtc)
                 ret = dev->driver->enable_vblank(dev, crtc);
                 DRM_DEBUG("enabling vblank on crtc %d, ret: %d\n", crtc, ret);
                 if (ret)
-                        atomic_dec(&dev->vblank[crtc].refcount);
+                        atomic_dec(&vblank->refcount);
                 else {
-                        dev->vblank[crtc].enabled = true;
+                        vblank->enabled = true;
                         drm_update_vblank_count(dev, crtc);
                 }
         }
@@ -919,6 +956,7 @@ static int drm_vblank_enable(struct drm_device *dev, int crtc)
  */
 int drm_vblank_get(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         unsigned long irqflags;
         int ret = 0;
 
@@ -927,11 +965,11 @@ int drm_vblank_get(struct drm_device *dev, int crtc)
 
         spin_lock_irqsave(&dev->vbl_lock, irqflags);
         /* Going from 0->1 means we have to enable interrupts again */
-        if (atomic_add_return(1, &dev->vblank[crtc].refcount) == 1) {
+        if (atomic_add_return(1, &vblank->refcount) == 1) {
                 ret = drm_vblank_enable(dev, crtc);
         } else {
-                if (!dev->vblank[crtc].enabled) {
-                        atomic_dec(&dev->vblank[crtc].refcount);
+                if (!vblank->enabled) {
+                        atomic_dec(&vblank->refcount);
                         ret = -EINVAL;
                 }
         }
@@ -971,16 +1009,23 @@ EXPORT_SYMBOL(drm_crtc_vblank_get);
  */
 void drm_vblank_put(struct drm_device *dev, int crtc)
 {
-        BUG_ON(atomic_read(&dev->vblank[crtc].refcount) == 0);
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+
+        BUG_ON(atomic_read(&vblank->refcount) == 0);
 
         if (WARN_ON(crtc >= dev->num_crtcs))
                 return;
 
         /* Last user schedules interrupt disable */
-        if (atomic_dec_and_test(&dev->vblank[crtc].refcount) &&
-            (drm_vblank_offdelay > 0))
-                mod_timer(&dev->vblank[crtc].disable_timer,
-                          jiffies + ((drm_vblank_offdelay * HZ)/1000));
+        if (atomic_dec_and_test(&vblank->refcount)) {
+                if (drm_vblank_offdelay == 0)
+                        return;
+                else if (dev->vblank_disable_immediate || drm_vblank_offdelay < 0)
+                        vblank_disable_fn((unsigned long)vblank);
+                else
+                        mod_timer(&vblank->disable_timer,
+                                  jiffies + ((drm_vblank_offdelay * HZ)/1000));
+        }
 }
 EXPORT_SYMBOL(drm_vblank_put);
 
@@ -1059,6 +1104,7 @@ EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
  */
 void drm_vblank_off(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         struct drm_pending_vblank_event *e, *t;
         struct timeval now;
         unsigned long irqflags;
@@ -1067,14 +1113,25 @@ void drm_vblank_off(struct drm_device *dev, int crtc)
         if (WARN_ON(crtc >= dev->num_crtcs))
                 return;
 
-        spin_lock_irqsave(&dev->vbl_lock, irqflags);
+        spin_lock_irqsave(&dev->event_lock, irqflags);
+
+        spin_lock(&dev->vbl_lock);
         vblank_disable_and_save(dev, crtc);
-        wake_up(&dev->vblank[crtc].queue);
+        wake_up(&vblank->queue);
+
+        /*
+         * Prevent subsequent drm_vblank_get() from re-enabling
+         * the vblank interrupt by bumping the refcount.
+         */
+        if (!vblank->inmodeset) {
+                atomic_inc(&vblank->refcount);
+                vblank->inmodeset = 1;
+        }
+        spin_unlock(&dev->vbl_lock);
 
         /* Send any queued vblank events, lest the natives grow disquiet */
         seq = drm_vblank_count_and_time(dev, crtc, &now);
 
-        spin_lock(&dev->event_lock);
         list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
                 if (e->pipe != crtc)
                         continue;
@@ -1085,9 +1142,7 @@ void drm_vblank_off(struct drm_device *dev, int crtc)
                 drm_vblank_put(dev, e->pipe);
                 send_vblank_event(dev, e, seq, &now);
         }
-        spin_unlock(&dev->event_lock);
-
-        spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
+        spin_unlock_irqrestore(&dev->event_lock, irqflags);
 }
 EXPORT_SYMBOL(drm_vblank_off);
 
@@ -1124,14 +1179,35 @@ EXPORT_SYMBOL(drm_crtc_vblank_off);
  */
 void drm_vblank_on(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         unsigned long irqflags;
 
         if (WARN_ON(crtc >= dev->num_crtcs))
                 return;
 
         spin_lock_irqsave(&dev->vbl_lock, irqflags);
-        /* re-enable interrupts if there's are users left */
-        if (atomic_read(&dev->vblank[crtc].refcount) != 0)
+        /* Drop our private "prevent drm_vblank_get" refcount */
+        if (vblank->inmodeset) {
+                atomic_dec(&vblank->refcount);
+                vblank->inmodeset = 0;
+        }
+
+        /*
+         * sample the current counter to avoid random jumps
+         * when drm_vblank_enable() applies the diff
+         *
+         * -1 to make sure user will never see the same
+         * vblank counter value before and after a modeset
+         */
+        vblank->last =
+                (dev->driver->get_vblank_counter(dev, crtc) - 1) &
+                dev->max_vblank_count;
+        /*
+         * re-enable interrupts if there are users left, or the
+         * user wishes vblank interrupts to be enabled all the time.
+         */
+        if (atomic_read(&vblank->refcount) != 0 ||
+            (!dev->vblank_disable_immediate && drm_vblank_offdelay == 0))
                 WARN_ON(drm_vblank_enable(dev, crtc));
         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 }
@@ -1179,6 +1255,8 @@ EXPORT_SYMBOL(drm_crtc_vblank_on);
  */
 void drm_vblank_pre_modeset(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+
         /* vblank is not initialized (IRQ not installed ?), or has been freed */
         if (!dev->num_crtcs)
                 return;
@@ -1193,10 +1271,10 @@ void drm_vblank_pre_modeset(struct drm_device *dev, int crtc)
          * to avoid corrupting the count if multiple, mismatch calls occur),
          * so that interrupts remain enabled in the interim.
          */
-        if (!dev->vblank[crtc].inmodeset) {
-                dev->vblank[crtc].inmodeset = 0x1;
+        if (!vblank->inmodeset) {
+                vblank->inmodeset = 0x1;
                 if (drm_vblank_get(dev, crtc) == 0)
-                        dev->vblank[crtc].inmodeset |= 0x2;
+                        vblank->inmodeset |= 0x2;
         }
 }
 EXPORT_SYMBOL(drm_vblank_pre_modeset);
@@ -1211,21 +1289,22 @@ EXPORT_SYMBOL(drm_vblank_pre_modeset);
  */
 void drm_vblank_post_modeset(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         unsigned long irqflags;
 
         /* vblank is not initialized (IRQ not installed ?), or has been freed */
         if (!dev->num_crtcs)
                 return;
 
-        if (dev->vblank[crtc].inmodeset) {
+        if (vblank->inmodeset) {
                 spin_lock_irqsave(&dev->vbl_lock, irqflags);
                 dev->vblank_disable_allowed = true;
                 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 
-                if (dev->vblank[crtc].inmodeset & 0x2)
+                if (vblank->inmodeset & 0x2)
                         drm_vblank_put(dev, crtc);
 
-                dev->vblank[crtc].inmodeset = 0;
+                vblank->inmodeset = 0;
         }
 }
 EXPORT_SYMBOL(drm_vblank_post_modeset);
@@ -1277,6 +1356,7 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
                                   union drm_wait_vblank *vblwait,
                                   struct drm_file *file_priv)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
         struct drm_pending_vblank_event *e;
         struct timeval now;
         unsigned long flags;
@@ -1300,6 +1380,18 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
 
         spin_lock_irqsave(&dev->event_lock, flags);
 
+        /*
+         * drm_vblank_off() might have been called after we called
+         * drm_vblank_get(). drm_vblank_off() holds event_lock
+         * around the vblank disable, so no need for further locking.
+         * The reference from drm_vblank_get() protects against
+         * vblank disable from another source.
+         */
+        if (!vblank->enabled) {
+                ret = -EINVAL;
+                goto err_unlock;
+        }
+
         if (file_priv->event_space < sizeof e->event) {
                 ret = -EBUSY;
                 goto err_unlock;
@@ -1360,6 +1452,7 @@ err_put:
 int drm_wait_vblank(struct drm_device *dev, void *data,
                     struct drm_file *file_priv)
 {
+        struct drm_vblank_crtc *vblank;
         union drm_wait_vblank *vblwait = data;
         int ret;
         unsigned int flags, seq, crtc, high_crtc;
@@ -1389,6 +1482,8 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
         if (crtc >= dev->num_crtcs)
                 return -EINVAL;
 
+        vblank = &dev->vblank[crtc];
+
         ret = drm_vblank_get(dev, crtc);
         if (ret) {
                 DRM_DEBUG("failed to acquire vblank counter, %d\n", ret);
@@ -1421,11 +1516,11 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
 
         DRM_DEBUG("waiting on vblank count %d, crtc %d\n",
                   vblwait->request.sequence, crtc);
-        dev->vblank[crtc].last_wait = vblwait->request.sequence;
-        DRM_WAIT_ON(ret, dev->vblank[crtc].queue, 3 * HZ,
+        vblank->last_wait = vblwait->request.sequence;
+        DRM_WAIT_ON(ret, vblank->queue, 3 * HZ,
                     (((drm_vblank_count(dev, crtc) -
                        vblwait->request.sequence) <= (1 << 23)) ||
-                     !dev->vblank[crtc].enabled ||
+                     !vblank->enabled ||
                      !dev->irq_enabled));
 
         if (ret != -EINTR) {
@@ -1450,12 +1545,11 @@ static void drm_handle_vblank_events(struct drm_device *dev, int crtc)
 {
         struct drm_pending_vblank_event *e, *t;
         struct timeval now;
-        unsigned long flags;
         unsigned int seq;
 
-        seq = drm_vblank_count_and_time(dev, crtc, &now);
+        assert_spin_locked(&dev->event_lock);
 
-        spin_lock_irqsave(&dev->event_lock, flags);
+        seq = drm_vblank_count_and_time(dev, crtc, &now);
 
         list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
                 if (e->pipe != crtc)
@@ -1471,8 +1565,6 @@ static void drm_handle_vblank_events(struct drm_device *dev, int crtc)
                 send_vblank_event(dev, e, seq, &now);
         }
 
-        spin_unlock_irqrestore(&dev->event_lock, flags);
-
         trace_drm_vblank_event(crtc, seq);
 }
 
@@ -1486,6 +1578,7 @@ static void drm_handle_vblank_events(struct drm_device *dev, int crtc)
  */
 bool drm_handle_vblank(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         u32 vblcount;
         s64 diff_ns;
         struct timeval tvblank;
@@ -1497,15 +1590,18 @@ bool drm_handle_vblank(struct drm_device *dev, int crtc)
         if (WARN_ON(crtc >= dev->num_crtcs))
                 return false;
 
+        spin_lock_irqsave(&dev->event_lock, irqflags);
+
         /* Need timestamp lock to prevent concurrent execution with
          * vblank enable/disable, as this would cause inconsistent
          * or corrupted timestamps and vblank counts.
          */
-        spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
+        spin_lock(&dev->vblank_time_lock);
 
         /* Vblank irq handling disabled. Nothing to do. */
-        if (!dev->vblank[crtc].enabled) {
-                spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+        if (!vblank->enabled) {
+                spin_unlock(&dev->vblank_time_lock);
+                spin_unlock_irqrestore(&dev->event_lock, irqflags);
                 return false;
         }
 
@@ -1514,7 +1610,7 @@ bool drm_handle_vblank(struct drm_device *dev, int crtc)
          */
 
         /* Get current timestamp and count. */
-        vblcount = atomic_read(&dev->vblank[crtc].count);
+        vblcount = atomic_read(&vblank->count);
         drm_get_last_vbltimestamp(dev, crtc, &tvblank, DRM_CALLED_FROM_VBLIRQ);
 
         /* Compute time difference to timestamp of last vblank */
@@ -1538,17 +1634,20 @@ bool drm_handle_vblank(struct drm_device *dev, int crtc)
                  * the timestamp computed above.
                  */
                 smp_mb__before_atomic();
-                atomic_inc(&dev->vblank[crtc].count);
+                atomic_inc(&vblank->count);
                 smp_mb__after_atomic();
         } else {
                 DRM_DEBUG("crtc %d: Redundant vblirq ignored. diff_ns = %d\n",
                           crtc, (int) diff_ns);
         }
 
-        wake_up(&dev->vblank[crtc].queue);
+        spin_unlock(&dev->vblank_time_lock);
+
+        wake_up(&vblank->queue);
         drm_handle_vblank_events(dev, crtc);
 
-        spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+        spin_unlock_irqrestore(&dev->event_lock, irqflags);
+
         return true;
 }
 EXPORT_SYMBOL(drm_handle_vblank);