Merge tag 'topic/vblank-rework-2014-09-12' of git://anongit.freedesktop.org/drm-intel into drm-next

So updated vblank-rework pull request, now with the polish that Mario
requested applied (and reviewed by him). Also with backmerge like you've
requested for easier merging.

The neat thing this finally allows is to immediately disable the vblank
interrupt on the last drm_vblank_put if the hardware has perfectly
accurate vblank counter and timestamp readout support. On i915 that
required piles of small adjustements from Ville since depending upon the
platform and port the vblank happens at different scanout lines.

Of course this is fully opt-in and per-device (we need that since gen2
doesn't have a hw vblank counter).

* tag 'topic/vblank-rework-2014-09-12' of git://anongit.freedesktop.org/drm-intel: (22 commits)
  drm: Clarify vblank ts/scanoutpos sampling #defines
  drm: Simplify return value of drm_get_last_vbltimestamp
  drm: Only update final vblank count when precise ts is available
  drm: Really never disable vblank irqs for offdelay==0
  drm: Use vblank_disable_and_save in drm_vblank_cleanup()
  drm: Remove drm_vblank_cleanup from drm_vblank_init error path.
  drm: Store the vblank timestamp when adjusting the counter during disable
  drm: Fix confusing debug message in drm_update_vblank_count()
  drm/i915: Update scanline_offset only for active crtcs
  drm: Kick start vblank interrupts at drm_vblank_on()
  drm/i915: Opt out of vblank disable timer on >gen2
  drm: Add dev->vblank_disable_immediate flag
  drm: Disable vblank interrupt immediately when drm_vblank_offdelay<0
  drm: Fix race between drm_vblank_off() and drm_queue_vblank_event()
  drm: Fix deadlock between event_lock and vbl_lock/vblank_time_lock
  drm: Reduce the amount of dev->vblank[crtc] in the code
  drm: Avoid random vblank counter jumps if the hardware counter has been reset
  drm: Have the vblank counter account for the time between vblank irq disable and drm_vblank_off()
  drm: Move drm_update_vblank_count()
  drm: Don't clear vblank timestamps when vblank interrupt is disabled
  ...

1 files changed, 239 insertions, 140 deletions

diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index 5708c056fa1b..034297640b48 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -55,12 +55,74 @@
  */
 #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
 
-/*
- * Clear vblank timestamp buffer for a crtc.
+static bool
+drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
+                          struct timeval *tvblank, unsigned flags);
+
+/**
+ * drm_update_vblank_count - update the master vblank counter
+ * @dev: DRM device
+ * @crtc: counter to update
+ *
+ * Call back into the driver to update the appropriate vblank counter
+ * (specified by @crtc).  Deal with wraparound, if it occurred, and
+ * update the last read value so we can deal with wraparound on the next
+ * call if necessary.
+ *
+ * Only necessary when going from off->on, to account for frames we
+ * didn't get an interrupt for.
+ *
+ * Note: caller must hold dev->vbl_lock since this reads & writes
+ * device vblank fields.
  */
-static void clear_vblank_timestamps(struct drm_device *dev, int crtc)
+static void drm_update_vblank_count(struct drm_device *dev, int crtc)
 {
-        memset(dev->vblank[crtc].time, 0, sizeof(dev->vblank[crtc].time));
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+        u32 cur_vblank, diff, tslot;
+        bool rc;
+        struct timeval t_vblank;
+
+        /*
+         * Interrupts were disabled prior to this call, so deal with counter
+         * wrap if needed.
+         * NOTE!  It's possible we lost a full dev->max_vblank_count events
+         * here if the register is small or we had vblank interrupts off for
+         * a long time.
+         *
+         * We repeat the hardware vblank counter & timestamp query until
+         * we get consistent results. This to prevent races between gpu
+         * updating its hardware counter while we are retrieving the
+         * corresponding vblank timestamp.
+         */
+        do {
+                cur_vblank = dev->driver->get_vblank_counter(dev, crtc);
+                rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0);
+        } while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc));
+
+        /* Deal with counter wrap */
+        diff = cur_vblank - vblank->last;
+        if (cur_vblank < vblank->last) {
+                diff += dev->max_vblank_count;
+
+                DRM_DEBUG("last_vblank[%d]=0x%x, cur_vblank=0x%x => diff=0x%x\n",
+                          crtc, vblank->last, cur_vblank, diff);
+        }
+
+        DRM_DEBUG("updating vblank count on crtc %d, missed %d\n",
+                  crtc, diff);
+
+        /* Reinitialize corresponding vblank timestamp if high-precision query
+         * available. Skip this step if query unsupported or failed. Will
+         * reinitialize delayed at next vblank interrupt in that case.
+         */
+        if (rc) {
+                tslot = atomic_read(&vblank->count) + diff;
+                vblanktimestamp(dev, crtc, tslot) = t_vblank;
+        }
+
+        smp_mb__before_atomic();
+        atomic_add(diff, &vblank->count);
+        smp_mb__after_atomic();
 }
 
 /*
@@ -71,10 +133,11 @@ static void clear_vblank_timestamps(struct drm_device *dev, int crtc)
  */
 static void vblank_disable_and_save(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         unsigned long irqflags;
         u32 vblcount;
         s64 diff_ns;
-        int vblrc;
+        bool vblrc;
         struct timeval tvblank;
         int count = DRM_TIMESTAMP_MAXRETRIES;
 
@@ -84,8 +147,28 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc)
          */
         spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
 
+        /*
+         * If the vblank interrupt was already disbled update the count
+         * and timestamp to maintain the appearance that the counter
+         * has been ticking all along until this time. This makes the
+         * count account for the entire time between drm_vblank_on() and
+         * drm_vblank_off().
+         *
+         * But only do this if precise vblank timestamps are available.
+         * Otherwise we might read a totally bogus timestamp since drivers
+         * lacking precise timestamp support rely upon sampling the system clock
+         * at vblank interrupt time. Which obviously won't work out well if the
+         * vblank interrupt is disabled.
+         */
+        if (!vblank->enabled &&
+            drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0)) {
+                drm_update_vblank_count(dev, crtc);
+                spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+                return;
+        }
+
         dev->driver->disable_vblank(dev, crtc);
-        dev->vblank[crtc].enabled = false;
+        vblank->enabled = false;
 
         /* No further vblank irq's will be processed after
          * this point. Get current hardware vblank count and
@@ -100,9 +183,9 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc)
          * delayed gpu counter increment.
          */
         do {
-                dev->vblank[crtc].last = dev->driver->get_vblank_counter(dev, crtc);
+                vblank->last = dev->driver->get_vblank_counter(dev, crtc);
                 vblrc = drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0);
-        } while (dev->vblank[crtc].last != dev->driver->get_vblank_counter(dev, crtc) && (--count) && vblrc);
+        } while (vblank->last != dev->driver->get_vblank_counter(dev, crtc) && (--count) && vblrc);
 
         if (!count)
                 vblrc = 0;
@@ -110,7 +193,7 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc)
         /* Compute time difference to stored timestamp of last vblank
          * as updated by last invocation of drm_handle_vblank() in vblank irq.
          */
-        vblcount = atomic_read(&dev->vblank[crtc].count);
+        vblcount = atomic_read(&vblank->count);
         diff_ns = timeval_to_ns(&tvblank) -
                   timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount));
 
@@ -126,14 +209,18 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc)
          * available. In that case we can't account for this and just
          * hope for the best.
          */
-        if ((vblrc > 0) && (abs64(diff_ns) > 1000000)) {
-                atomic_inc(&dev->vblank[crtc].count);
+        if (vblrc && (abs64(diff_ns) > 1000000)) {
+                /* Store new timestamp in ringbuffer. */
+                vblanktimestamp(dev, crtc, vblcount + 1) = tvblank;
+
+                /* Increment cooked vblank count. This also atomically commits
+                 * the timestamp computed above.
+                 */
+                smp_mb__before_atomic();
+                atomic_inc(&vblank->count);
                 smp_mb__after_atomic();
         }
 
-        /* Invalidate all timestamps while vblank irq's are off. */
-        clear_vblank_timestamps(dev, crtc);
-
         spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
 }
 
@@ -164,14 +251,20 @@ static void vblank_disable_fn(unsigned long arg)
 void drm_vblank_cleanup(struct drm_device *dev)
 {
         int crtc;
+        unsigned long irqflags;
 
         /* Bail if the driver didn't call drm_vblank_init() */
         if (dev->num_crtcs == 0)
                 return;
 
         for (crtc = 0; crtc < dev->num_crtcs; crtc++) {
-                del_timer_sync(&dev->vblank[crtc].disable_timer);
-                vblank_disable_fn((unsigned long)&dev->vblank[crtc]);
+                struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+
+                del_timer_sync(&vblank->disable_timer);
+
+                spin_lock_irqsave(&dev->vbl_lock, irqflags);
+                vblank_disable_and_save(dev, crtc);
+                spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
         }
 
         kfree(dev->vblank);
@@ -204,11 +297,13 @@ int drm_vblank_init(struct drm_device *dev, int num_crtcs)
                 goto err;
 
         for (i = 0; i < num_crtcs; i++) {
-                dev->vblank[i].dev = dev;
-                dev->vblank[i].crtc = i;
-                init_waitqueue_head(&dev->vblank[i].queue);
-                setup_timer(&dev->vblank[i].disable_timer, vblank_disable_fn,
-                            (unsigned long)&dev->vblank[i]);
+                struct drm_vblank_crtc *vblank = &dev->vblank[i];
+
+                vblank->dev = dev;
+                vblank->crtc = i;
+                init_waitqueue_head(&vblank->queue);
+                setup_timer(&vblank->disable_timer, vblank_disable_fn,
+                            (unsigned long)vblank);
         }
 
         DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
@@ -224,7 +319,7 @@ int drm_vblank_init(struct drm_device *dev, int num_crtcs)
         return 0;
 
 err:
-        drm_vblank_cleanup(dev);
+        dev->num_crtcs = 0;
         return ret;
 }
 EXPORT_SYMBOL(drm_vblank_init);
@@ -360,9 +455,11 @@ int drm_irq_uninstall(struct drm_device *dev)
         if (dev->num_crtcs) {
                 spin_lock_irqsave(&dev->vbl_lock, irqflags);
                 for (i = 0; i < dev->num_crtcs; i++) {
-                        wake_up(&dev->vblank[i].queue);
-                        dev->vblank[i].enabled = false;
-                        dev->vblank[i].last =
+                        struct drm_vblank_crtc *vblank = &dev->vblank[i];
+
+                        wake_up(&vblank->queue);
+                        vblank->enabled = false;
+                        vblank->last =
                                 dev->driver->get_vblank_counter(dev, i);
                 }
                 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
@@ -617,7 +714,7 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
          * within vblank area, counting down the number of lines until
          * start of scanout.
          */
-        invbl = vbl_status & DRM_SCANOUTPOS_INVBL;
+        invbl = vbl_status & DRM_SCANOUTPOS_IN_VBLANK;
 
         /* Convert scanout position into elapsed time at raw_time query
          * since start of scanout at first display scanline. delta_ns
@@ -647,7 +744,7 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
 
         vbl_status = DRM_VBLANKTIME_SCANOUTPOS_METHOD;
         if (invbl)
-                vbl_status |= DRM_VBLANKTIME_INVBL;
+                vbl_status |= DRM_VBLANKTIME_IN_VBLANK;
 
         return vbl_status;
 }
@@ -679,10 +776,11 @@ static struct timeval get_drm_timestamp(void)
  * call, i.e., it isn't very precisely locked to the true vblank.
  *
  * Returns:
- * Non-zero if timestamp is considered to be very precise, zero otherwise.
+ * True if timestamp is considered to be very precise, false otherwise.
  */
-u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
-                              struct timeval *tvblank, unsigned flags)
+static bool
+drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
+                          struct timeval *tvblank, unsigned flags)
 {
         int ret;
 
@@ -694,7 +792,7 @@ u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
                 ret = dev->driver->get_vblank_timestamp(dev, crtc, &max_error,
                                                         tvblank, flags);
                 if (ret > 0)
-                        return (u32) ret;
+                        return true;
         }
 
         /* GPU high precision timestamp query unsupported or failed.
@@ -702,9 +800,8 @@ u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
          */
         *tvblank = get_drm_timestamp();
 
-        return 0;
+        return false;
 }
-EXPORT_SYMBOL(drm_get_last_vbltimestamp);
 
 /**
  * drm_vblank_count - retrieve "cooked" vblank counter value
@@ -720,9 +817,11 @@ EXPORT_SYMBOL(drm_get_last_vbltimestamp);
  */
 u32 drm_vblank_count(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+
         if (WARN_ON(crtc >= dev->num_crtcs))
                 return 0;
-        return atomic_read(&dev->vblank[crtc].count);
+        return atomic_read(&vblank->count);
 }
 EXPORT_SYMBOL(drm_vblank_count);
 
@@ -742,6 +841,7 @@ EXPORT_SYMBOL(drm_vblank_count);
 u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc,
                               struct timeval *vblanktime)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         u32 cur_vblank;
 
         if (WARN_ON(crtc >= dev->num_crtcs))
@@ -753,10 +853,10 @@ u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc,
          * a seqlock.
          */
         do {
-                cur_vblank = atomic_read(&dev->vblank[crtc].count);
+                cur_vblank = atomic_read(&vblank->count);
                 *vblanktime = vblanktimestamp(dev, crtc, cur_vblank);
                 smp_rmb();
-        } while (cur_vblank != atomic_read(&dev->vblank[crtc].count));
+        } while (cur_vblank != atomic_read(&vblank->count));
 
         return cur_vblank;
 }
@@ -805,83 +905,20 @@ void drm_send_vblank_event(struct drm_device *dev, int crtc,
 EXPORT_SYMBOL(drm_send_vblank_event);
 
 /**
- * drm_update_vblank_count - update the master vblank counter
- * @dev: DRM device
- * @crtc: counter to update
- *
- * Call back into the driver to update the appropriate vblank counter
- * (specified by @crtc).  Deal with wraparound, if it occurred, and
- * update the last read value so we can deal with wraparound on the next
- * call if necessary.
- *
- * Only necessary when going from off->on, to account for frames we
- * didn't get an interrupt for.
- *
- * Note: caller must hold dev->vbl_lock since this reads & writes
- * device vblank fields.
- */
-static void drm_update_vblank_count(struct drm_device *dev, int crtc)
-{
-        u32 cur_vblank, diff, tslot, rc;
-        struct timeval t_vblank;
-
-        /*
-         * Interrupts were disabled prior to this call, so deal with counter
-         * wrap if needed.
-         * NOTE!  It's possible we lost a full dev->max_vblank_count events
-         * here if the register is small or we had vblank interrupts off for
-         * a long time.
-         *
-         * We repeat the hardware vblank counter & timestamp query until
-         * we get consistent results. This to prevent races between gpu
-         * updating its hardware counter while we are retrieving the
-         * corresponding vblank timestamp.
-         */
-        do {
-                cur_vblank = dev->driver->get_vblank_counter(dev, crtc);
-                rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0);
-        } while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc));
-
-        /* Deal with counter wrap */
-        diff = cur_vblank - dev->vblank[crtc].last;
-        if (cur_vblank < dev->vblank[crtc].last) {
-                diff += dev->max_vblank_count;
-
-                DRM_DEBUG("last_vblank[%d]=0x%x, cur_vblank=0x%x => diff=0x%x\n",
-                          crtc, dev->vblank[crtc].last, cur_vblank, diff);
-        }
-
-        DRM_DEBUG("enabling vblank interrupts on crtc %d, missed %d\n",
-                  crtc, diff);
-
-        /* Reinitialize corresponding vblank timestamp if high-precision query
-         * available. Skip this step if query unsupported or failed. Will
-         * reinitialize delayed at next vblank interrupt in that case.
-         */
-        if (rc) {
-                tslot = atomic_read(&dev->vblank[crtc].count) + diff;
-                vblanktimestamp(dev, crtc, tslot) = t_vblank;
-        }
-
-        smp_mb__before_atomic();
-        atomic_add(diff, &dev->vblank[crtc].count);
-        smp_mb__after_atomic();
-}
-
-/**
  * drm_vblank_enable - enable the vblank interrupt on a CRTC
  * @dev: DRM device
  * @crtc: CRTC in question
  */
 static int drm_vblank_enable(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         int ret = 0;
 
         assert_spin_locked(&dev->vbl_lock);
 
         spin_lock(&dev->vblank_time_lock);
 
-        if (!dev->vblank[crtc].enabled) {
+        if (!vblank->enabled) {
                 /*
                  * Enable vblank irqs under vblank_time_lock protection.
                  * All vblank count & timestamp updates are held off
@@ -892,9 +929,9 @@ static int drm_vblank_enable(struct drm_device *dev, int crtc)
                 ret = dev->driver->enable_vblank(dev, crtc);
                 DRM_DEBUG("enabling vblank on crtc %d, ret: %d\n", crtc, ret);
                 if (ret)
-                        atomic_dec(&dev->vblank[crtc].refcount);
+                        atomic_dec(&vblank->refcount);
                 else {
-                        dev->vblank[crtc].enabled = true;
+                        vblank->enabled = true;
                         drm_update_vblank_count(dev, crtc);
                 }
         }
@@ -919,6 +956,7 @@ static int drm_vblank_enable(struct drm_device *dev, int crtc)
  */
 int drm_vblank_get(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         unsigned long irqflags;
         int ret = 0;
 
@@ -927,11 +965,11 @@ int drm_vblank_get(struct drm_device *dev, int crtc)
 
         spin_lock_irqsave(&dev->vbl_lock, irqflags);
         /* Going from 0->1 means we have to enable interrupts again */
-        if (atomic_add_return(1, &dev->vblank[crtc].refcount) == 1) {
+        if (atomic_add_return(1, &vblank->refcount) == 1) {
                 ret = drm_vblank_enable(dev, crtc);
         } else {
-                if (!dev->vblank[crtc].enabled) {
-                        atomic_dec(&dev->vblank[crtc].refcount);
+                if (!vblank->enabled) {
+                        atomic_dec(&vblank->refcount);
                         ret = -EINVAL;
                 }
         }
@@ -971,16 +1009,23 @@ EXPORT_SYMBOL(drm_crtc_vblank_get);
  */
 void drm_vblank_put(struct drm_device *dev, int crtc)
 {
-        BUG_ON(atomic_read(&dev->vblank[crtc].refcount) == 0);
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+
+        BUG_ON(atomic_read(&vblank->refcount) == 0);
 
         if (WARN_ON(crtc >= dev->num_crtcs))
                 return;
 
         /* Last user schedules interrupt disable */
-        if (atomic_dec_and_test(&dev->vblank[crtc].refcount) &&
-            (drm_vblank_offdelay > 0))
-                mod_timer(&dev->vblank[crtc].disable_timer,
-                          jiffies + ((drm_vblank_offdelay * HZ)/1000));
+        if (atomic_dec_and_test(&vblank->refcount)) {
+                if (drm_vblank_offdelay == 0)
+                        return;
+                else if (dev->vblank_disable_immediate || drm_vblank_offdelay < 0)
+                        vblank_disable_fn((unsigned long)vblank);
+                else
+                        mod_timer(&vblank->disable_timer,
+                                  jiffies + ((drm_vblank_offdelay * HZ)/1000));
+        }
 }
 EXPORT_SYMBOL(drm_vblank_put);
 
@@ -1059,6 +1104,7 @@ EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
  */
 void drm_vblank_off(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         struct drm_pending_vblank_event *e, *t;
         struct timeval now;
         unsigned long irqflags;
@@ -1067,14 +1113,25 @@ void drm_vblank_off(struct drm_device *dev, int crtc)
         if (WARN_ON(crtc >= dev->num_crtcs))
                 return;
 
-        spin_lock_irqsave(&dev->vbl_lock, irqflags);
+        spin_lock_irqsave(&dev->event_lock, irqflags);
+
+        spin_lock(&dev->vbl_lock);
         vblank_disable_and_save(dev, crtc);
-        wake_up(&dev->vblank[crtc].queue);
+        wake_up(&vblank->queue);
+
+        /*
+         * Prevent subsequent drm_vblank_get() from re-enabling
+         * the vblank interrupt by bumping the refcount.
+         */
+        if (!vblank->inmodeset) {
+                atomic_inc(&vblank->refcount);
+                vblank->inmodeset = 1;
+        }
+        spin_unlock(&dev->vbl_lock);
 
         /* Send any queued vblank events, lest the natives grow disquiet */
         seq = drm_vblank_count_and_time(dev, crtc, &now);
 
-        spin_lock(&dev->event_lock);
         list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
                 if (e->pipe != crtc)
                         continue;
@@ -1085,9 +1142,7 @@ void drm_vblank_off(struct drm_device *dev, int crtc)
                 drm_vblank_put(dev, e->pipe);
                 send_vblank_event(dev, e, seq, &now);
         }
-        spin_unlock(&dev->event_lock);
-
-        spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
+        spin_unlock_irqrestore(&dev->event_lock, irqflags);
 }
 EXPORT_SYMBOL(drm_vblank_off);
 
@@ -1124,14 +1179,35 @@ EXPORT_SYMBOL(drm_crtc_vblank_off);
  */
 void drm_vblank_on(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         unsigned long irqflags;
 
         if (WARN_ON(crtc >= dev->num_crtcs))
                 return;
 
         spin_lock_irqsave(&dev->vbl_lock, irqflags);
-        /* re-enable interrupts if there's are users left */
-        if (atomic_read(&dev->vblank[crtc].refcount) != 0)
+        /* Drop our private "prevent drm_vblank_get" refcount */
+        if (vblank->inmodeset) {
+                atomic_dec(&vblank->refcount);
+                vblank->inmodeset = 0;
+        }
+
+        /*
+         * sample the current counter to avoid random jumps
+         * when drm_vblank_enable() applies the diff
+         *
+         * -1 to make sure user will never see the same
+         * vblank counter value before and after a modeset
+         */
+        vblank->last =
+                (dev->driver->get_vblank_counter(dev, crtc) - 1) &
+                dev->max_vblank_count;
+        /*
+         * re-enable interrupts if there are users left, or the
+         * user wishes vblank interrupts to be enabled all the time.
+         */
+        if (atomic_read(&vblank->refcount) != 0 ||
+            (!dev->vblank_disable_immediate && drm_vblank_offdelay == 0))
                 WARN_ON(drm_vblank_enable(dev, crtc));
         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 }
@@ -1179,6 +1255,8 @@ EXPORT_SYMBOL(drm_crtc_vblank_on);
  */
 void drm_vblank_pre_modeset(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
+
         /* vblank is not initialized (IRQ not installed ?), or has been freed */
         if (!dev->num_crtcs)
                 return;
@@ -1193,10 +1271,10 @@ void drm_vblank_pre_modeset(struct drm_device *dev, int crtc)
          * to avoid corrupting the count if multiple, mismatch calls occur),
          * so that interrupts remain enabled in the interim.
          */
-        if (!dev->vblank[crtc].inmodeset) {
-                dev->vblank[crtc].inmodeset = 0x1;
+        if (!vblank->inmodeset) {
+                vblank->inmodeset = 0x1;
                 if (drm_vblank_get(dev, crtc) == 0)
-                        dev->vblank[crtc].inmodeset |= 0x2;
+                        vblank->inmodeset |= 0x2;
         }
 }
 EXPORT_SYMBOL(drm_vblank_pre_modeset);
@@ -1211,21 +1289,22 @@ EXPORT_SYMBOL(drm_vblank_pre_modeset);
  */
 void drm_vblank_post_modeset(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         unsigned long irqflags;
 
         /* vblank is not initialized (IRQ not installed ?), or has been freed */
         if (!dev->num_crtcs)
                 return;
 
-        if (dev->vblank[crtc].inmodeset) {
+        if (vblank->inmodeset) {
                 spin_lock_irqsave(&dev->vbl_lock, irqflags);
                 dev->vblank_disable_allowed = true;
                 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 
-                if (dev->vblank[crtc].inmodeset & 0x2)
+                if (vblank->inmodeset & 0x2)
                         drm_vblank_put(dev, crtc);
 
-                dev->vblank[crtc].inmodeset = 0;
+                vblank->inmodeset = 0;
         }
 }
 EXPORT_SYMBOL(drm_vblank_post_modeset);
@@ -1277,6 +1356,7 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
                                   union drm_wait_vblank *vblwait,
                                   struct drm_file *file_priv)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
         struct drm_pending_vblank_event *e;
         struct timeval now;
         unsigned long flags;
@@ -1300,6 +1380,18 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe,
 
         spin_lock_irqsave(&dev->event_lock, flags);
 
+        /*
+         * drm_vblank_off() might have been called after we called
+         * drm_vblank_get(). drm_vblank_off() holds event_lock
+         * around the vblank disable, so no need for further locking.
+         * The reference from drm_vblank_get() protects against
+         * vblank disable from another source.
+         */
+        if (!vblank->enabled) {
+                ret = -EINVAL;
+                goto err_unlock;
+        }
+
         if (file_priv->event_space < sizeof e->event) {
                 ret = -EBUSY;
                 goto err_unlock;
@@ -1360,6 +1452,7 @@ err_put:
 int drm_wait_vblank(struct drm_device *dev, void *data,
                     struct drm_file *file_priv)
 {
+        struct drm_vblank_crtc *vblank;
         union drm_wait_vblank *vblwait = data;
         int ret;
         unsigned int flags, seq, crtc, high_crtc;
@@ -1389,6 +1482,8 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
         if (crtc >= dev->num_crtcs)
                 return -EINVAL;
 
+        vblank = &dev->vblank[crtc];
+
         ret = drm_vblank_get(dev, crtc);
         if (ret) {
                 DRM_DEBUG("failed to acquire vblank counter, %d\n", ret);
@@ -1421,11 +1516,11 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
 
         DRM_DEBUG("waiting on vblank count %d, crtc %d\n",
                   vblwait->request.sequence, crtc);
-        dev->vblank[crtc].last_wait = vblwait->request.sequence;
-        DRM_WAIT_ON(ret, dev->vblank[crtc].queue, 3 * HZ,
+        vblank->last_wait = vblwait->request.sequence;
+        DRM_WAIT_ON(ret, vblank->queue, 3 * HZ,
                     (((drm_vblank_count(dev, crtc) -
                        vblwait->request.sequence) <= (1 << 23)) ||
-                     !dev->vblank[crtc].enabled ||
+                     !vblank->enabled ||
                      !dev->irq_enabled));
 
         if (ret != -EINTR) {
@@ -1450,12 +1545,11 @@ static void drm_handle_vblank_events(struct drm_device *dev, int crtc)
 {
         struct drm_pending_vblank_event *e, *t;
         struct timeval now;
-        unsigned long flags;
         unsigned int seq;
 
-        seq = drm_vblank_count_and_time(dev, crtc, &now);
+        assert_spin_locked(&dev->event_lock);
 
-        spin_lock_irqsave(&dev->event_lock, flags);
+        seq = drm_vblank_count_and_time(dev, crtc, &now);
 
         list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
                 if (e->pipe != crtc)
@@ -1471,8 +1565,6 @@ static void drm_handle_vblank_events(struct drm_device *dev, int crtc)
                 send_vblank_event(dev, e, seq, &now);
         }
 
-        spin_unlock_irqrestore(&dev->event_lock, flags);
-
         trace_drm_vblank_event(crtc, seq);
 }
 
@@ -1486,6 +1578,7 @@ static void drm_handle_vblank_events(struct drm_device *dev, int crtc)
  */
 bool drm_handle_vblank(struct drm_device *dev, int crtc)
 {
+        struct drm_vblank_crtc *vblank = &dev->vblank[crtc];
         u32 vblcount;
         s64 diff_ns;
         struct timeval tvblank;
@@ -1497,15 +1590,18 @@ bool drm_handle_vblank(struct drm_device *dev, int crtc)
         if (WARN_ON(crtc >= dev->num_crtcs))
                 return false;
 
+        spin_lock_irqsave(&dev->event_lock, irqflags);
+
         /* Need timestamp lock to prevent concurrent execution with
          * vblank enable/disable, as this would cause inconsistent
          * or corrupted timestamps and vblank counts.
          */
-        spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
+        spin_lock(&dev->vblank_time_lock);
 
         /* Vblank irq handling disabled. Nothing to do. */
-        if (!dev->vblank[crtc].enabled) {
-                spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+        if (!vblank->enabled) {
+                spin_unlock(&dev->vblank_time_lock);
+                spin_unlock_irqrestore(&dev->event_lock, irqflags);
                 return false;
         }
 
@@ -1514,7 +1610,7 @@ bool drm_handle_vblank(struct drm_device *dev, int crtc)
          */
 
         /* Get current timestamp and count. */
-        vblcount = atomic_read(&dev->vblank[crtc].count);
+        vblcount = atomic_read(&vblank->count);
         drm_get_last_vbltimestamp(dev, crtc, &tvblank, DRM_CALLED_FROM_VBLIRQ);
 
         /* Compute time difference to timestamp of last vblank */
@@ -1538,17 +1634,20 @@ bool drm_handle_vblank(struct drm_device *dev, int crtc)
                  * the timestamp computed above.
                  */
                 smp_mb__before_atomic();
-                atomic_inc(&dev->vblank[crtc].count);
+                atomic_inc(&vblank->count);
                 smp_mb__after_atomic();
         } else {
                 DRM_DEBUG("crtc %d: Redundant vblirq ignored. diff_ns = %d\n",
                           crtc, (int) diff_ns);
         }
 
-        wake_up(&dev->vblank[crtc].queue);
+        spin_unlock(&dev->vblank_time_lock);
+
+        wake_up(&vblank->queue);
         drm_handle_vblank_events(dev, crtc);
 
-        spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
+        spin_unlock_irqrestore(&dev->event_lock, irqflags);
+
         return true;
 }
 EXPORT_SYMBOL(drm_handle_vblank);