drm/i915: Eliminate lots of iterations over the execobjects array

The major scaling bottleneck in execbuffer is the processing of the
execobjects. Creating an auxiliary list is inefficient when compared to
using the execobject array we already have allocated.

Reservation is then split into phases. As we lookup up the VMA, we
try and bind it back into active location. Only if that fails, do we add
it to the unbound list for phase 2. In phase 2, we try and add all those
objects that could not fit into their previous location, with fallback
to retrying all objects and evicting the VM in case of severe
fragmentation. (This is the same as before, except that phase 1 is now
done inline with looking up the VMA to avoid an iteration over the
execobject array. In the ideal case, we eliminate the separate reservation
phase). During the reservation phase, we only evict from the VM between
passes (rather than currently as we try to fit every new VMA). In
testing with Unreal Engine's Atlantis demo which stresses the eviction
logic on gen7 class hardware, this speed up the framerate by a factor of
2.

The second loop amalgamation is between move_to_gpu and move_to_active.
As we always submit the request, even if incomplete, we can use the
current request to track active VMA as we perform the flushes and
synchronisation required.

The next big advancement is to avoid copying back to the user any
execobjects and relocations that are not changed.

v2: Add a Theory of Operation spiel.
v3: Fall back to slow relocations in preparation for flushing userptrs.
v4: Document struct members, factor out eb_validate_vma(), add a few
more comments to explain some magic and hide other magic behind macros.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>

7 files changed, 1239 insertions, 916 deletions

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index af2a54672396..7e182dd7e356 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -3581,7 +3581,7 @@ int __must_check i915_gem_evict_something(struct i915_address_space *vm,
 int __must_check i915_gem_evict_for_node(struct i915_address_space *vm,
                                          struct drm_mm_node *node,
                                          unsigned int flags);
-int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle);
+int i915_gem_evict_vm(struct i915_address_space *vm);
 
 /* belongs in i915_gem_gtt.h */
 static inline void i915_gem_chipset_flush(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 204a2d9288ae..a193f1b36c67 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -50,6 +50,29 @@ static bool ggtt_is_idle(struct drm_i915_private *dev_priv)
         return true;
 }
 
+static int ggtt_flush(struct drm_i915_private *i915)
+{
+        int err;
+
+        /* Not everything in the GGTT is tracked via vma (otherwise we
+         * could evict as required with minimal stalling) so we are forced
+         * to idle the GPU and explicitly retire outstanding requests in
+         * the hopes that we can then remove contexts and the like only
+         * bound by their active reference.
+         */
+        err = i915_gem_switch_to_kernel_context(i915);
+        if (err)
+                return err;
+
+        err = i915_gem_wait_for_idle(i915,
+                                     I915_WAIT_INTERRUPTIBLE |
+                                     I915_WAIT_LOCKED);
+        if (err)
+                return err;
+
+        return 0;
+}
+
 static bool
 mark_free(struct drm_mm_scan *scan,
           struct i915_vma *vma,
@@ -175,19 +198,7 @@ search_again:
                 return intel_has_pending_fb_unpin(dev_priv) ? -EAGAIN : -ENOSPC;
         }
 
-        /* Not everything in the GGTT is tracked via vma (otherwise we
-         * could evict as required with minimal stalling) so we are forced
-         * to idle the GPU and explicitly retire outstanding requests in
-         * the hopes that we can then remove contexts and the like only
-         * bound by their active reference.
-         */
-        ret = i915_gem_switch_to_kernel_context(dev_priv);
-        if (ret)
-                return ret;
-
-        ret = i915_gem_wait_for_idle(dev_priv,
-                                     I915_WAIT_INTERRUPTIBLE |
-                                     I915_WAIT_LOCKED);
+        ret = ggtt_flush(dev_priv);
         if (ret)
                 return ret;
 
@@ -337,10 +348,8 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
 /**
  * i915_gem_evict_vm - Evict all idle vmas from a vm
  * @vm: Address space to cleanse
- * @do_idle: Boolean directing whether to idle first.
  *
- * This function evicts all idles vmas from a vm. If all unpinned vmas should be
- * evicted the @do_idle needs to be set to true.
+ * This function evicts all vmas from a vm.
  *
  * This is used by the execbuf code as a last-ditch effort to defragment the
  * address space.
@@ -348,37 +357,50 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
  * To clarify: This is for freeing up virtual address space, not for freeing
  * memory in e.g. the shrinker.
  */
-int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
+int i915_gem_evict_vm(struct i915_address_space *vm)
 {
+        struct list_head *phases[] = {
+                &vm->inactive_list,
+                &vm->active_list,
+                NULL
+        }, **phase;
+        struct list_head eviction_list;
         struct i915_vma *vma, *next;
         int ret;
 
         lockdep_assert_held(&vm->i915->drm.struct_mutex);
         trace_i915_gem_evict_vm(vm);
 
-        if (do_idle) {
-                struct drm_i915_private *dev_priv = vm->i915;
-
-                if (i915_is_ggtt(vm)) {
-                        ret = i915_gem_switch_to_kernel_context(dev_priv);
-                        if (ret)
-                                return ret;
-                }
-
-                ret = i915_gem_wait_for_idle(dev_priv,
-                                             I915_WAIT_INTERRUPTIBLE |
-                                             I915_WAIT_LOCKED);
+        /* Switch back to the default context in order to unpin
+         * the existing context objects. However, such objects only
+         * pin themselves inside the global GTT and performing the
+         * switch otherwise is ineffective.
+         */
+        if (i915_is_ggtt(vm)) {
+                ret = ggtt_flush(vm->i915);
                 if (ret)
                         return ret;
-
-                WARN_ON(!list_empty(&vm->active_list));
         }
 
-        list_for_each_entry_safe(vma, next, &vm->inactive_list, vm_link)
-                if (!i915_vma_is_pinned(vma))
-                        WARN_ON(i915_vma_unbind(vma));
+        INIT_LIST_HEAD(&eviction_list);
+        phase = phases;
+        do {
+                list_for_each_entry(vma, *phase, vm_link) {
+                        if (i915_vma_is_pinned(vma))
+                                continue;
 
-        return 0;
+                        __i915_vma_pin(vma);
+                        list_add(&vma->evict_link, &eviction_list);
+                }
+        } while (*++phase);
+
+        ret = 0;
+        list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
+                __i915_vma_unpin(vma);
+                if (ret == 0)
+                        ret = i915_vma_unbind(vma);
+        }
+        return ret;
 }
 
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 9c3f6c40270f..a052072fe8b3 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -42,41 +42,195 @@
 
 #define DBG_USE_CPU_RELOC 0 /* -1 force GTT relocs; 1 force CPU relocs */
 
-#define  __EXEC_OBJECT_HAS_PIN          (1<<31)
-#define  __EXEC_OBJECT_HAS_FENCE        (1<<30)
-#define  __EXEC_OBJECT_NEEDS_MAP        (1<<29)
-#define  __EXEC_OBJECT_NEEDS_BIAS       (1<<28)
-#define  __EXEC_OBJECT_INTERNAL_FLAGS (0xf<<28) /* all of the above */
+#define __EXEC_OBJECT_HAS_PIN           BIT(31)
+#define __EXEC_OBJECT_HAS_FENCE         BIT(30)
+#define __EXEC_OBJECT_NEEDS_MAP         BIT(29)
+#define __EXEC_OBJECT_NEEDS_BIAS        BIT(28)
+#define __EXEC_OBJECT_INTERNAL_FLAGS    (~0u << 28) /* all of the above */
+#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
+
+#define __EXEC_HAS_RELOC        BIT(31)
+#define __EXEC_VALIDATED        BIT(30)
+#define UPDATE                  PIN_OFFSET_FIXED
 
 #define BATCH_OFFSET_BIAS (256*1024)
 
 #define __I915_EXEC_ILLEGAL_FLAGS \
         (__I915_EXEC_UNKNOWN_FLAGS | I915_EXEC_CONSTANTS_MASK)
 
+/**
+ * DOC: User command execution
+ *
+ * Userspace submits commands to be executed on the GPU as an instruction
+ * stream within a GEM object we call a batchbuffer. This instructions may
+ * refer to other GEM objects containing auxiliary state such as kernels,
+ * samplers, render targets and even secondary batchbuffers. Userspace does
+ * not know where in the GPU memory these objects reside and so before the
+ * batchbuffer is passed to the GPU for execution, those addresses in the
+ * batchbuffer and auxiliary objects are updated. This is known as relocation,
+ * or patching. To try and avoid having to relocate each object on the next
+ * execution, userspace is told the location of those objects in this pass,
+ * but this remains just a hint as the kernel may choose a new location for
+ * any object in the future.
+ *
+ * Processing an execbuf ioctl is conceptually split up into a few phases.
+ *
+ * 1. Validation - Ensure all the pointers, handles and flags are valid.
+ * 2. Reservation - Assign GPU address space for every object
+ * 3. Relocation - Update any addresses to point to the final locations
+ * 4. Serialisation - Order the request with respect to its dependencies
+ * 5. Construction - Construct a request to execute the batchbuffer
+ * 6. Submission (at some point in the future execution)
+ *
+ * Reserving resources for the execbuf is the most complicated phase. We
+ * neither want to have to migrate the object in the address space, nor do
+ * we want to have to update any relocations pointing to this object. Ideally,
+ * we want to leave the object where it is and for all the existing relocations
+ * to match. If the object is given a new address, or if userspace thinks the
+ * object is elsewhere, we have to parse all the relocation entries and update
+ * the addresses. Userspace can set the I915_EXEC_NORELOC flag to hint that
+ * all the target addresses in all of its objects match the value in the
+ * relocation entries and that they all match the presumed offsets given by the
+ * list of execbuffer objects. Using this knowledge, we know that if we haven't
+ * moved any buffers, all the relocation entries are valid and we can skip
+ * the update. (If userspace is wrong, the likely outcome is an impromptu GPU
+ * hang.) The requirement for using I915_EXEC_NO_RELOC are:
+ *
+ *      The addresses written in the objects must match the corresponding
+ *      reloc.presumed_offset which in turn must match the corresponding
+ *      execobject.offset.
+ *
+ *      Any render targets written to in the batch must be flagged with
+ *      EXEC_OBJECT_WRITE.
+ *
+ *      To avoid stalling, execobject.offset should match the current
+ *      address of that object within the active context.
+ *
+ * The reservation is done is multiple phases. First we try and keep any
+ * object already bound in its current location - so as long as meets the
+ * constraints imposed by the new execbuffer. Any object left unbound after the
+ * first pass is then fitted into any available idle space. If an object does
+ * not fit, all objects are removed from the reservation and the process rerun
+ * after sorting the objects into a priority order (more difficult to fit
+ * objects are tried first). Failing that, the entire VM is cleared and we try
+ * to fit the execbuf once last time before concluding that it simply will not
+ * fit.
+ *
+ * A small complication to all of this is that we allow userspace not only to
+ * specify an alignment and a size for the object in the address space, but
+ * we also allow userspace to specify the exact offset. This objects are
+ * simpler to place (the location is known a priori) all we have to do is make
+ * sure the space is available.
+ *
+ * Once all the objects are in place, patching up the buried pointers to point
+ * to the final locations is a fairly simple job of walking over the relocation
+ * entry arrays, looking up the right address and rewriting the value into
+ * the object. Simple! ... The relocation entries are stored in user memory
+ * and so to access them we have to copy them into a local buffer. That copy
+ * has to avoid taking any pagefaults as they may lead back to a GEM object
+ * requiring the struct_mutex (i.e. recursive deadlock). So once again we split
+ * the relocation into multiple passes. First we try to do everything within an
+ * atomic context (avoid the pagefaults) which requires that we never wait. If
+ * we detect that we may wait, or if we need to fault, then we have to fallback
+ * to a slower path. The slowpath has to drop the mutex. (Can you hear alarm
+ * bells yet?) Dropping the mutex means that we lose all the state we have
+ * built up so far for the execbuf and we must reset any global data. However,
+ * we do leave the objects pinned in their final locations - which is a
+ * potential issue for concurrent execbufs. Once we have left the mutex, we can
+ * allocate and copy all the relocation entries into a large array at our
+ * leisure, reacquire the mutex, reclaim all the objects and other state and
+ * then proceed to update any incorrect addresses with the objects.
+ *
+ * As we process the relocation entries, we maintain a record of whether the
+ * object is being written to. Using NORELOC, we expect userspace to provide
+ * this information instead. We also check whether we can skip the relocation
+ * by comparing the expected value inside the relocation entry with the target's
+ * final address. If they differ, we have to map the current object and rewrite
+ * the 4 or 8 byte pointer within.
+ *
+ * Serialising an execbuf is quite simple according to the rules of the GEM
+ * ABI. Execution within each context is ordered by the order of submission.
+ * Writes to any GEM object are in order of submission and are exclusive. Reads
+ * from a GEM object are unordered with respect to other reads, but ordered by
+ * writes. A write submitted after a read cannot occur before the read, and
+ * similarly any read submitted after a write cannot occur before the write.
+ * Writes are ordered between engines such that only one write occurs at any
+ * time (completing any reads beforehand) - using semaphores where available
+ * and CPU serialisation otherwise. Other GEM access obey the same rules, any
+ * write (either via mmaps using set-domain, or via pwrite) must flush all GPU
+ * reads before starting, and any read (either using set-domain or pread) must
+ * flush all GPU writes before starting. (Note we only employ a barrier before,
+ * we currently rely on userspace not concurrently starting a new execution
+ * whilst reading or writing to an object. This may be an advantage or not
+ * depending on how much you trust userspace not to shoot themselves in the
+ * foot.) Serialisation may just result in the request being inserted into
+ * a DAG awaiting its turn, but most simple is to wait on the CPU until
+ * all dependencies are resolved.
+ *
+ * After all of that, is just a matter of closing the request and handing it to
+ * the hardware (well, leaving it in a queue to be executed). However, we also
+ * offer the ability for batchbuffers to be run with elevated privileges so
+ * that they access otherwise hidden registers. (Used to adjust L3 cache etc.)
+ * Before any batch is given extra privileges we first must check that it
+ * contains no nefarious instructions, we check that each instruction is from
+ * our whitelist and all registers are also from an allowed list. We first
+ * copy the user's batchbuffer to a shadow (so that the user doesn't have
+ * access to it, either by the CPU or GPU as we scan it) and then parse each
+ * instruction. If everything is ok, we set a flag telling the hardware to run
+ * the batchbuffer in trusted mode, otherwise the ioctl is rejected.
+ */
+
 struct i915_execbuffer {
-        struct drm_i915_private *i915;
-        struct drm_file *file;
-        struct drm_i915_gem_execbuffer2 *args;
-        struct drm_i915_gem_exec_object2 *exec;
-        struct intel_engine_cs *engine;
-        struct i915_gem_context *ctx;
-        struct i915_address_space *vm;
-        struct i915_vma *batch;
-        struct drm_i915_gem_request *request;
-        u32 batch_start_offset;
-        u32 batch_len;
-        unsigned int dispatch_flags;
-        struct drm_i915_gem_exec_object2 shadow_exec_entry;
-        bool need_relocs;
-        struct list_head vmas;
+        struct drm_i915_private *i915; /** i915 backpointer */
+        struct drm_file *file; /** per-file lookup tables and limits */
+        struct drm_i915_gem_execbuffer2 *args; /** ioctl parameters */
+        struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */
+
+        struct intel_engine_cs *engine; /** engine to queue the request to */
+        struct i915_gem_context *ctx; /** context for building the request */
+        struct i915_address_space *vm; /** GTT and vma for the request */
+
+        struct drm_i915_gem_request *request; /** our request to build */
+        struct i915_vma *batch; /** identity of the batch obj/vma */
+
+        /** actual size of execobj[] as we may extend it for the cmdparser */
+        unsigned int buffer_count;
+
+        /** list of vma not yet bound during reservation phase */
+        struct list_head unbound;
+
+        /** list of vma that have execobj.relocation_count */
+        struct list_head relocs;
+
+        /**
+         * Track the most recently used object for relocations, as we
+         * frequently have to perform multiple relocations within the same
+         * obj/page
+         */
         struct reloc_cache {
-                struct drm_mm_node node;
-                unsigned long vaddr;
-                unsigned int page;
+                struct drm_mm_node node; /** temporary GTT binding */
+                unsigned long vaddr; /** Current kmap address */
+                unsigned long page; /** Currently mapped page index */
                 bool use_64bit_reloc : 1;
+                bool has_llc : 1;
+                bool has_fence : 1;
+                bool needs_unfenced : 1;
         } reloc_cache;
-        int lut_mask;
-        struct hlist_head *buckets;
+
+        u64 invalid_flags; /** Set of execobj.flags that are invalid */
+        u32 context_flags; /** Set of execobj.flags to insert from the ctx */
+
+        u32 batch_start_offset; /** Location within object of batch */
+        u32 batch_len; /** Length of batch within object */
+        u32 batch_flags; /** Flags composed for emit_bb_start() */
+
+        /**
+         * Indicate either the size of the hastable used to resolve
+         * relocation handles, or if negative that we are using a direct
+         * index into the execobj[].
+         */
+        int lut_size;
+        struct hlist_head *buckets; /** ht for relocation handles */
 };
 
 /*
@@ -87,11 +241,41 @@ struct i915_execbuffer {
 #define __exec_to_vma(ee) (ee)->rsvd2
 #define exec_to_vma(ee) u64_to_ptr(struct i915_vma, __exec_to_vma(ee))
 
+/*
+ * Used to convert any address to canonical form.
+ * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
+ * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
+ * addresses to be in a canonical form:
+ * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
+ * canonical form [63:48] == [47]."
+ */
+#define GEN8_HIGH_ADDRESS_BIT 47
+static inline u64 gen8_canonical_addr(u64 address)
+{
+        return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
+}
+
+static inline u64 gen8_noncanonical_addr(u64 address)
+{
+        return address & GENMASK_ULL(GEN8_HIGH_ADDRESS_BIT, 0);
+}
+
 static int eb_create(struct i915_execbuffer *eb)
 {
-        if ((eb->args->flags & I915_EXEC_HANDLE_LUT) == 0) {
-                unsigned int size = 1 + ilog2(eb->args->buffer_count);
+        if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) {
+                unsigned int size = 1 + ilog2(eb->buffer_count);
 
+                /*
+                 * Without a 1:1 association between relocation handles and
+                 * the execobject[] index, we instead create a hashtable.
+                 * We size it dynamically based on available memory, starting
+                 * first with 1:1 assocative hash and scaling back until
+                 * the allocation succeeds.
+                 *
+                 * Later on we use a positive lut_size to indicate we are
+                 * using this hashtable, and a negative value to indicate a
+                 * direct lookup.
+                 */
                 do {
                         eb->buckets = kzalloc(sizeof(struct hlist_head) << size,
                                               GFP_TEMPORARY |
@@ -108,112 +292,411 @@ static int eb_create(struct i915_execbuffer *eb)
                                 return -ENOMEM;
                 }
 
-                eb->lut_mask = size;
+                eb->lut_size = size;
         } else {
-                eb->lut_mask = -eb->args->buffer_count;
+                eb->lut_size = -eb->buffer_count;
         }
 
         return 0;
 }
 
+static bool
+eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry,
+                 const struct i915_vma *vma)
+{
+        if (!(entry->flags & __EXEC_OBJECT_HAS_PIN))
+                return true;
+
+        if (vma->node.size < entry->pad_to_size)
+                return true;
+
+        if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
+                return true;
+
+        if (entry->flags & EXEC_OBJECT_PINNED &&
+            vma->node.start != entry->offset)
+                return true;
+
+        if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
+            vma->node.start < BATCH_OFFSET_BIAS)
+                return true;
+
+        if (!(entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) &&
+            (vma->node.start + vma->node.size - 1) >> 32)
+                return true;
+
+        return false;
+}
+
+static inline void
+eb_pin_vma(struct i915_execbuffer *eb,
+           struct drm_i915_gem_exec_object2 *entry,
+           struct i915_vma *vma)
+{
+        u64 flags;
+
+        flags = vma->node.start;
+        flags |= PIN_USER | PIN_NONBLOCK | PIN_OFFSET_FIXED;
+        if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_GTT))
+                flags |= PIN_GLOBAL;
+        if (unlikely(i915_vma_pin(vma, 0, 0, flags)))
+                return;
+
+        if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_FENCE)) {
+                if (unlikely(i915_vma_get_fence(vma))) {
+                        i915_vma_unpin(vma);
+                        return;
+                }
+
+                if (i915_vma_pin_fence(vma))
+                        entry->flags |= __EXEC_OBJECT_HAS_FENCE;
+        }
+
+        entry->flags |= __EXEC_OBJECT_HAS_PIN;
+}
+
 static inline void
 __eb_unreserve_vma(struct i915_vma *vma,
                    const struct drm_i915_gem_exec_object2 *entry)
 {
+        GEM_BUG_ON(!(entry->flags & __EXEC_OBJECT_HAS_PIN));
+
         if (unlikely(entry->flags & __EXEC_OBJECT_HAS_FENCE))
                 i915_vma_unpin_fence(vma);
 
-        if (entry->flags & __EXEC_OBJECT_HAS_PIN)
-                __i915_vma_unpin(vma);
+        __i915_vma_unpin(vma);
 }
 
-static void
-eb_unreserve_vma(struct i915_vma *vma)
+static inline void
+eb_unreserve_vma(struct i915_vma *vma,
+                 struct drm_i915_gem_exec_object2 *entry)
 {
-        struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+        if (!(entry->flags & __EXEC_OBJECT_HAS_PIN))
+                return;
 
         __eb_unreserve_vma(vma, entry);
-        entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
+        entry->flags &= ~__EXEC_OBJECT_RESERVED;
 }
 
-static void
-eb_reset(struct i915_execbuffer *eb)
+static int
+eb_validate_vma(struct i915_execbuffer *eb,
+                struct drm_i915_gem_exec_object2 *entry,
+                struct i915_vma *vma)
 {
-        struct i915_vma *vma;
+        if (unlikely(entry->flags & eb->invalid_flags))
+                return -EINVAL;
 
-        list_for_each_entry(vma, &eb->vmas, exec_link) {
-                eb_unreserve_vma(vma);
-                i915_vma_put(vma);
-                vma->exec_entry = NULL;
+        if (unlikely(entry->alignment && !is_power_of_2(entry->alignment)))
+                return -EINVAL;
+
+        /*
+         * Offset can be used as input (EXEC_OBJECT_PINNED), reject
+         * any non-page-aligned or non-canonical addresses.
+         */
+        if (unlikely(entry->flags & EXEC_OBJECT_PINNED &&
+                     entry->offset != gen8_canonical_addr(entry->offset & PAGE_MASK)))
+                return -EINVAL;
+
+        /* pad_to_size was once a reserved field, so sanitize it */
+        if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) {
+                if (unlikely(offset_in_page(entry->pad_to_size)))
+                        return -EINVAL;
+        } else {
+                entry->pad_to_size = 0;
         }
 
-        if (eb->lut_mask >= 0)
-                memset(eb->buckets, 0,
-                       sizeof(struct hlist_head) << eb->lut_mask);
+        if (unlikely(vma->exec_entry)) {
+                DRM_DEBUG("Object [handle %d, index %d] appears more than once in object list\n",
+                          entry->handle, (int)(entry - eb->exec));
+                return -EINVAL;
+        }
+
+        /*
+         * From drm_mm perspective address space is continuous,
+         * so from this point we're always using non-canonical
+         * form internally.
+         */
+        entry->offset = gen8_noncanonical_addr(entry->offset);
+
+        return 0;
 }
 
-static bool
-eb_add_vma(struct i915_execbuffer *eb, struct i915_vma *vma, int i)
+static int
+eb_add_vma(struct i915_execbuffer *eb,
+           struct drm_i915_gem_exec_object2 *entry,
+           struct i915_vma *vma)
 {
-        if (unlikely(vma->exec_entry)) {
-                DRM_DEBUG("Object [handle %d, index %d] appears more than once in object list\n",
-                          eb->exec[i].handle, i);
-                return false;
+        int err;
+
+        GEM_BUG_ON(i915_vma_is_closed(vma));
+
+        if (!(eb->args->flags & __EXEC_VALIDATED)) {
+                err = eb_validate_vma(eb, entry, vma);
+                if (unlikely(err))
+                        return err;
         }
-        list_add_tail(&vma->exec_link, &eb->vmas);
 
-        vma->exec_entry = &eb->exec[i];
-        if (eb->lut_mask >= 0) {
-                vma->exec_handle = eb->exec[i].handle;
+        if (eb->lut_size >= 0) {
+                vma->exec_handle = entry->handle;
                 hlist_add_head(&vma->exec_node,
-                               &eb->buckets[hash_32(vma->exec_handle,
-                                                    eb->lut_mask)]);
+                               &eb->buckets[hash_32(entry->handle,
+                                                    eb->lut_size)]);
         }
 
-        i915_vma_get(vma);
-        __exec_to_vma(&eb->exec[i]) = (uintptr_t)vma;
-        return true;
+        if (entry->relocation_count)
+                list_add_tail(&vma->reloc_link, &eb->relocs);
+
+        if (!eb->reloc_cache.has_fence) {
+                entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
+        } else {
+                if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
+                     eb->reloc_cache.needs_unfenced) &&
+                    i915_gem_object_is_tiled(vma->obj))
+                        entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP;
+        }
+
+        if (!(entry->flags & EXEC_OBJECT_PINNED))
+                entry->flags |= eb->context_flags;
+
+        /*
+         * Stash a pointer from the vma to execobj, so we can query its flags,
+         * size, alignment etc as provided by the user. Also we stash a pointer
+         * to the vma inside the execobj so that we can use a direct lookup
+         * to find the right target VMA when doing relocations.
+         */
+        vma->exec_entry = entry;
+        __exec_to_vma(entry) = (uintptr_t)i915_vma_get(vma);
+
+        err = 0;
+        if (vma->node.size)
+                eb_pin_vma(eb, entry, vma);
+        if (eb_vma_misplaced(entry, vma)) {
+                eb_unreserve_vma(vma, entry);
+
+                list_add_tail(&vma->exec_link, &eb->unbound);
+                if (drm_mm_node_allocated(&vma->node))
+                        err = i915_vma_unbind(vma);
+        } else {
+                if (entry->offset != vma->node.start) {
+                        entry->offset = vma->node.start | UPDATE;
+                        eb->args->flags |= __EXEC_HAS_RELOC;
+                }
+        }
+        return err;
+}
+
+static inline int use_cpu_reloc(const struct reloc_cache *cache,
+                                const struct drm_i915_gem_object *obj)
+{
+        if (!i915_gem_object_has_struct_page(obj))
+                return false;
+
+        if (DBG_USE_CPU_RELOC)
+                return DBG_USE_CPU_RELOC > 0;
+
+        return (cache->has_llc ||
+                obj->cache_dirty ||
+                obj->cache_level != I915_CACHE_NONE);
+}
+
+static int eb_reserve_vma(const struct i915_execbuffer *eb,
+                          struct i915_vma *vma)
+{
+        struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+        u64 flags;
+        int err;
+
+        flags = PIN_USER | PIN_NONBLOCK;
+        if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
+                flags |= PIN_GLOBAL;
+
+        /*
+         * Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
+         * limit address to the first 4GBs for unflagged objects.
+         */
+        if (!(entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
+                flags |= PIN_ZONE_4G;
+
+        if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
+                flags |= PIN_MAPPABLE;
+
+        if (entry->flags & EXEC_OBJECT_PINNED) {
+                flags |= entry->offset | PIN_OFFSET_FIXED;
+                flags &= ~PIN_NONBLOCK; /* force overlapping PINNED checks */
+        } else if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS) {
+                flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
+        }
+
+        err = i915_vma_pin(vma, entry->pad_to_size, entry->alignment, flags);
+        if (err)
+                return err;
+
+        if (entry->offset != vma->node.start) {
+                entry->offset = vma->node.start | UPDATE;
+                eb->args->flags |= __EXEC_HAS_RELOC;
+        }
+
+        entry->flags |= __EXEC_OBJECT_HAS_PIN;
+        GEM_BUG_ON(eb_vma_misplaced(entry, vma));
+
+        if (unlikely(entry->flags & EXEC_OBJECT_NEEDS_FENCE)) {
+                err = i915_vma_get_fence(vma);
+                if (unlikely(err)) {
+                        i915_vma_unpin(vma);
+                        return err;
+                }
+
+                if (i915_vma_pin_fence(vma))
+                        entry->flags |= __EXEC_OBJECT_HAS_FENCE;
+        }
+
+        return 0;
+}
+
+static int eb_reserve(struct i915_execbuffer *eb)
+{
+        const unsigned int count = eb->buffer_count;
+        struct list_head last;
+        struct i915_vma *vma;
+        unsigned int i, pass;
+        int err;
+
+        /*
+         * Attempt to pin all of the buffers into the GTT.
+         * This is done in 3 phases:
+         *
+         * 1a. Unbind all objects that do not match the GTT constraints for
+         *     the execbuffer (fenceable, mappable, alignment etc).
+         * 1b. Increment pin count for already bound objects.
+         * 2.  Bind new objects.
+         * 3.  Decrement pin count.
+         *
+         * This avoid unnecessary unbinding of later objects in order to make
+         * room for the earlier objects *unless* we need to defragment.
+         */
+
+        pass = 0;
+        err = 0;
+        do {
+                list_for_each_entry(vma, &eb->unbound, exec_link) {
+                        err = eb_reserve_vma(eb, vma);
+                        if (err)
+                                break;
+                }
+                if (err != -ENOSPC)
+                        return err;
+
+                /* Resort *all* the objects into priority order */
+                INIT_LIST_HEAD(&eb->unbound);
+                INIT_LIST_HEAD(&last);
+                for (i = 0; i < count; i++) {
+                        struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+
+                        if (entry->flags & EXEC_OBJECT_PINNED &&
+                            entry->flags & __EXEC_OBJECT_HAS_PIN)
+                                continue;
+
+                        vma = exec_to_vma(entry);
+                        eb_unreserve_vma(vma, entry);
+
+                        if (entry->flags & EXEC_OBJECT_PINNED)
+                                list_add(&vma->exec_link, &eb->unbound);
+                        else if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
+                                list_add_tail(&vma->exec_link, &eb->unbound);
+                        else
+                                list_add_tail(&vma->exec_link, &last);
+                }
+                list_splice_tail(&last, &eb->unbound);
+
+                switch (pass++) {
+                case 0:
+                        break;
+
+                case 1:
+                        /* Too fragmented, unbind everything and retry */
+                        err = i915_gem_evict_vm(eb->vm);
+                        if (err)
+                                return err;
+                        break;
+
+                default:
+                        return -ENOSPC;
+                }
+        } while (1);
 }
 
 static inline struct hlist_head *
-ht_head(const struct i915_gem_context *ctx, u32 handle)
+ht_head(const  struct i915_gem_context_vma_lut *lut, u32 handle)
 {
-        return &ctx->vma_lut.ht[hash_32(handle, ctx->vma_lut.ht_bits)];
+        return &lut->ht[hash_32(handle, lut->ht_bits)];
 }
 
 static inline bool
-ht_needs_resize(const struct i915_gem_context *ctx)
+ht_needs_resize(const struct i915_gem_context_vma_lut *lut)
 {
-        return (4*ctx->vma_lut.ht_count > 3*ctx->vma_lut.ht_size ||
-                4*ctx->vma_lut.ht_count + 1 < ctx->vma_lut.ht_size);
+        return (4*lut->ht_count > 3*lut->ht_size ||
+                4*lut->ht_count + 1 < lut->ht_size);
 }
 
-static int
-eb_lookup_vmas(struct i915_execbuffer *eb)
+static unsigned int eb_batch_index(const struct i915_execbuffer *eb)
+{
+        return eb->buffer_count - 1;
+}
+
+static int eb_select_context(struct i915_execbuffer *eb)
+{
+        struct i915_gem_context *ctx;
+
+        ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1);
+        if (unlikely(IS_ERR(ctx)))
+                return PTR_ERR(ctx);
+
+        if (unlikely(i915_gem_context_is_banned(ctx))) {
+                DRM_DEBUG("Context %u tried to submit while banned\n",
+                          ctx->user_handle);
+                return -EIO;
+        }
+
+        eb->ctx = i915_gem_context_get(ctx);
+        eb->vm = ctx->ppgtt ? &ctx->ppgtt->base : &eb->i915->ggtt.base;
+
+        eb->context_flags = 0;
+        if (ctx->flags & CONTEXT_NO_ZEROMAP)
+                eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS;
+
+        return 0;
+}
+
+static int eb_lookup_vmas(struct i915_execbuffer *eb)
 {
 #define INTERMEDIATE BIT(0)
-        const int count = eb->args->buffer_count;
+        const unsigned int count = eb->buffer_count;
+        struct i915_gem_context_vma_lut *lut = &eb->ctx->vma_lut;
         struct i915_vma *vma;
+        struct idr *idr;
+        unsigned int i;
         int slow_pass = -1;
-        int i;
+        int err;
 
-        INIT_LIST_HEAD(&eb->vmas);
+        INIT_LIST_HEAD(&eb->relocs);
+        INIT_LIST_HEAD(&eb->unbound);
 
-        if (unlikely(eb->ctx->vma_lut.ht_size & I915_CTX_RESIZE_IN_PROGRESS))
-                flush_work(&eb->ctx->vma_lut.resize);
-        GEM_BUG_ON(eb->ctx->vma_lut.ht_size & I915_CTX_RESIZE_IN_PROGRESS);
+        if (unlikely(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS))
+                flush_work(&lut->resize);
+        GEM_BUG_ON(lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS);
 
         for (i = 0; i < count; i++) {
                 __exec_to_vma(&eb->exec[i]) = 0;
 
                 hlist_for_each_entry(vma,
-                                     ht_head(eb->ctx, eb->exec[i].handle),
+                                     ht_head(lut, eb->exec[i].handle),
                                      ctx_node) {
                         if (vma->ctx_handle != eb->exec[i].handle)
                                 continue;
 
-                        if (!eb_add_vma(eb, vma, i))
-                                return -EINVAL;
+                        err = eb_add_vma(eb, &eb->exec[i], vma);
+                        if (unlikely(err))
+                                return err;
 
                         goto next_vma;
                 }
@@ -224,24 +707,27 @@ next_vma: ;
         }
 
         if (slow_pass < 0)
-                return 0;
+                goto out;
 
         spin_lock(&eb->file->table_lock);
-        /* Grab a reference to the object and release the lock so we can lookup
-         * or create the VMA without using GFP_ATOMIC */
+        /*
+         * Grab a reference to the object and release the lock so we can lookup
+         * or create the VMA without using GFP_ATOMIC
+         */
+        idr = &eb->file->object_idr;
         for (i = slow_pass; i < count; i++) {
                 struct drm_i915_gem_object *obj;
 
                 if (__exec_to_vma(&eb->exec[i]))
                         continue;
 
-                obj = to_intel_bo(idr_find(&eb->file->object_idr,
-                                           eb->exec[i].handle));
+                obj = to_intel_bo(idr_find(idr, eb->exec[i].handle));
                 if (unlikely(!obj)) {
                         spin_unlock(&eb->file->table_lock);
                         DRM_DEBUG("Invalid object handle %d at index %d\n",
                                   eb->exec[i].handle, i);
-                        return -ENOENT;
+                        err = -ENOENT;
+                        goto err;
                 }
 
                 __exec_to_vma(&eb->exec[i]) = INTERMEDIATE | (uintptr_t)obj;
@@ -251,7 +737,7 @@ next_vma: ;
         for (i = slow_pass; i < count; i++) {
                 struct drm_i915_gem_object *obj;
 
-                if ((__exec_to_vma(&eb->exec[i]) & INTERMEDIATE) == 0)
+                if (!(__exec_to_vma(&eb->exec[i]) & INTERMEDIATE))
                         continue;
 
                 /*
@@ -262,12 +748,13 @@ next_vma: ;
                  * from the (obj, vm) we don't run the risk of creating
                  * duplicated vmas for the same vm.
                  */
-                obj = u64_to_ptr(struct drm_i915_gem_object,
+                obj = u64_to_ptr(typeof(*obj),
                                  __exec_to_vma(&eb->exec[i]) & ~INTERMEDIATE);
                 vma = i915_vma_instance(obj, eb->vm, NULL);
                 if (unlikely(IS_ERR(vma))) {
                         DRM_DEBUG("Failed to lookup VMA\n");
-                        return PTR_ERR(vma);
+                        err = PTR_ERR(vma);
+                        goto err;
                 }
 
                 /* First come, first served */
@@ -275,32 +762,31 @@ next_vma: ;
                         vma->ctx = eb->ctx;
                         vma->ctx_handle = eb->exec[i].handle;
                         hlist_add_head(&vma->ctx_node,
-                                       ht_head(eb->ctx, eb->exec[i].handle));
-                        eb->ctx->vma_lut.ht_count++;
+                                       ht_head(lut, eb->exec[i].handle));
+                        lut->ht_count++;
+                        lut->ht_size |= I915_CTX_RESIZE_IN_PROGRESS;
                         if (i915_vma_is_ggtt(vma)) {
                                 GEM_BUG_ON(obj->vma_hashed);
                                 obj->vma_hashed = vma;
                         }
                 }
 
-                if (!eb_add_vma(eb, vma, i))
-                        return -EINVAL;
+                err = eb_add_vma(eb, &eb->exec[i], vma);
+                if (unlikely(err))
+                        goto err;
         }
 
-        if (ht_needs_resize(eb->ctx)) {
-                eb->ctx->vma_lut.ht_size |= I915_CTX_RESIZE_IN_PROGRESS;
-                queue_work(system_highpri_wq, &eb->ctx->vma_lut.resize);
+        if (lut->ht_size & I915_CTX_RESIZE_IN_PROGRESS) {
+                if (ht_needs_resize(lut))
+                        queue_work(system_highpri_wq, &lut->resize);
+                else
+                        lut->ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
         }
 
-        return 0;
-#undef INTERMEDIATE
-}
-
-static struct i915_vma *
-eb_get_batch(struct i915_execbuffer *eb)
-{
-        struct i915_vma *vma =
-                exec_to_vma(&eb->exec[eb->args->buffer_count - 1]);
+out:
+        /* take note of the batch buffer before we might reorder the lists */
+        i = eb_batch_index(eb);
+        eb->batch = exec_to_vma(&eb->exec[i]);
 
         /*
          * SNA is doing fancy tricks with compressing batch buffers, which leads
@@ -311,24 +797,36 @@ eb_get_batch(struct i915_execbuffer *eb)
          * Note that actual hangs have only been observed on gen7, but for
          * paranoia do it everywhere.
          */
-        if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
-                vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
+        if (!(eb->exec[i].flags & EXEC_OBJECT_PINNED))
+                eb->exec[i].flags |= __EXEC_OBJECT_NEEDS_BIAS;
+        if (eb->reloc_cache.has_fence)
+                eb->exec[i].flags |= EXEC_OBJECT_NEEDS_FENCE;
 
-        return vma;
+        eb->args->flags |= __EXEC_VALIDATED;
+        return eb_reserve(eb);
+
+err:
+        for (i = slow_pass; i < count; i++) {
+                if (__exec_to_vma(&eb->exec[i]) & INTERMEDIATE)
+                        __exec_to_vma(&eb->exec[i]) = 0;
+        }
+        lut->ht_size &= ~I915_CTX_RESIZE_IN_PROGRESS;
+        return err;
+#undef INTERMEDIATE
 }
 
 static struct i915_vma *
-eb_get_vma(struct i915_execbuffer *eb, unsigned long handle)
+eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle)
 {
-        if (eb->lut_mask < 0) {
-                if (handle >= -eb->lut_mask)
+        if (eb->lut_size < 0) {
+                if (handle >= -eb->lut_size)
                         return NULL;
                 return exec_to_vma(&eb->exec[handle]);
         } else {
                 struct hlist_head *head;
                 struct i915_vma *vma;
 
-                head = &eb->buckets[hash_32(handle, eb->lut_mask)];
+                head = &eb->buckets[hash_32(handle, eb->lut_size)];
                 hlist_for_each_entry(vma, head, exec_node) {
                         if (vma->exec_handle == handle)
                                 return vma;
@@ -337,61 +835,46 @@ eb_get_vma(struct i915_execbuffer *eb, unsigned long handle)
         }
 }
 
-static void eb_destroy(struct i915_execbuffer *eb)
+static void eb_release_vmas(const struct i915_execbuffer *eb)
 {
-        struct i915_vma *vma;
+        const unsigned int count = eb->buffer_count;
+        unsigned int i;
 
-        list_for_each_entry(vma, &eb->vmas, exec_link) {
-                if (!vma->exec_entry)
+        for (i = 0; i < count; i++) {
+                struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+                struct i915_vma *vma = exec_to_vma(entry);
+
+                if (!vma)
                         continue;
 
-                __eb_unreserve_vma(vma, vma->exec_entry);
+                GEM_BUG_ON(vma->exec_entry != entry);
                 vma->exec_entry = NULL;
-                i915_vma_put(vma);
-        }
-
-        i915_gem_context_put(eb->ctx);
 
-        if (eb->lut_mask >= 0)
-                kfree(eb->buckets);
-}
-
-static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
-{
-        if (!i915_gem_object_has_struct_page(obj))
-                return false;
+                eb_unreserve_vma(vma, entry);
 
-        if (DBG_USE_CPU_RELOC)
-                return DBG_USE_CPU_RELOC > 0;
-
-        return (HAS_LLC(to_i915(obj->base.dev)) ||
-                obj->cache_dirty ||
-                obj->cache_level != I915_CACHE_NONE);
+                i915_vma_put(vma);
+        }
 }
 
-/* Used to convert any address to canonical form.
- * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
- * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
- * addresses to be in a canonical form:
- * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
- * canonical form [63:48] == [47]."
- */
-#define GEN8_HIGH_ADDRESS_BIT 47
-static inline uint64_t gen8_canonical_addr(uint64_t address)
+static void eb_reset_vmas(const struct i915_execbuffer *eb)
 {
-        return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
+        eb_release_vmas(eb);
+        if (eb->lut_size >= 0)
+                memset(eb->buckets, 0,
+                       sizeof(struct hlist_head) << eb->lut_size);
 }
 
-static inline uint64_t gen8_noncanonical_addr(uint64_t address)
+static void eb_destroy(const struct i915_execbuffer *eb)
 {
-        return address & ((1ULL << (GEN8_HIGH_ADDRESS_BIT + 1)) - 1);
+        if (eb->lut_size >= 0)
+                kfree(eb->buckets);
 }
 
-static inline uint64_t
+static inline u64
 relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
-                  uint64_t target_offset)
+                  const struct i915_vma *target)
 {
-        return gen8_canonical_addr((int)reloc->delta + target_offset);
+        return gen8_canonical_addr((int)reloc->delta + target->node.start);
 }
 
 static void reloc_cache_init(struct reloc_cache *cache,
@@ -400,6 +883,9 @@ static void reloc_cache_init(struct reloc_cache *cache,
         cache->page = -1;
         cache->vaddr = 0;
         /* Must be a variable in the struct to allow GCC to unroll. */
+        cache->has_llc = HAS_LLC(i915);
+        cache->has_fence = INTEL_GEN(i915) < 4;
+        cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment;
         cache->use_64bit_reloc = HAS_64BIT_RELOC(i915);
         cache->node.allocated = false;
 }
@@ -458,7 +944,7 @@ static void reloc_cache_reset(struct reloc_cache *cache)
 
 static void *reloc_kmap(struct drm_i915_gem_object *obj,
                         struct reloc_cache *cache,
-                        int page)
+                        unsigned long page)
 {
         void *vaddr;
 
@@ -466,11 +952,11 @@ static void *reloc_kmap(struct drm_i915_gem_object *obj,
                 kunmap_atomic(unmask_page(cache->vaddr));
         } else {
                 unsigned int flushes;
-                int ret;
+                int err;
 
-                ret = i915_gem_obj_prepare_shmem_write(obj, &flushes);
-                if (ret)
-                        return ERR_PTR(ret);
+                err = i915_gem_obj_prepare_shmem_write(obj, &flushes);
+                if (err)
+                        return ERR_PTR(err);
 
                 BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS);
                 BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK);
@@ -490,7 +976,7 @@ static void *reloc_kmap(struct drm_i915_gem_object *obj,
 
 static void *reloc_iomap(struct drm_i915_gem_object *obj,
                          struct reloc_cache *cache,
-                         int page)
+                         unsigned long page)
 {
         struct i915_ggtt *ggtt = cache_to_ggtt(cache);
         unsigned long offset;
@@ -500,31 +986,31 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
                 io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));
         } else {
                 struct i915_vma *vma;
-                int ret;
+                int err;
 
-                if (use_cpu_reloc(obj))
+                if (use_cpu_reloc(cache, obj))
                         return NULL;
 
-                ret = i915_gem_object_set_to_gtt_domain(obj, true);
-                if (ret)
-                        return ERR_PTR(ret);
+                err = i915_gem_object_set_to_gtt_domain(obj, true);
+                if (err)
+                        return ERR_PTR(err);
 
                 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
                                                PIN_MAPPABLE | PIN_NONBLOCK);
                 if (IS_ERR(vma)) {
                         memset(&cache->node, 0, sizeof(cache->node));
-                        ret = drm_mm_insert_node_in_range
+                        err = drm_mm_insert_node_in_range
                                 (&ggtt->base.mm, &cache->node,
                                  PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
                                  0, ggtt->mappable_end,
                                  DRM_MM_INSERT_LOW);
-                        if (ret) /* no inactive aperture space, use cpu reloc */
+                        if (err) /* no inactive aperture space, use cpu reloc */
                                 return NULL;
                 } else {
-                        ret = i915_vma_put_fence(vma);
-                        if (ret) {
+                        err = i915_vma_put_fence(vma);
+                        if (err) {
                                 i915_vma_unpin(vma);
-                                return ERR_PTR(ret);
+                                return ERR_PTR(err);
                         }
 
                         cache->node.start = vma->node.start;
@@ -552,7 +1038,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj,
 
 static void *reloc_vaddr(struct drm_i915_gem_object *obj,
                          struct reloc_cache *cache,
-                         int page)
+                         unsigned long page)
 {
         void *vaddr;
 
@@ -579,7 +1065,8 @@ static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
 
                 *addr = value;
 
-                /* Writes to the same cacheline are serialised by the CPU
+                /*
+                 * Writes to the same cacheline are serialised by the CPU
                  * (including clflush). On the write path, we only require
                  * that it hits memory in an orderly fashion and place
                  * mb barriers at the start and end of the relocation phase
@@ -591,25 +1078,26 @@ static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
                 *addr = value;
 }
 
-static int
-relocate_entry(struct drm_i915_gem_object *obj,
+static u64
+relocate_entry(struct i915_vma *vma,
                const struct drm_i915_gem_relocation_entry *reloc,
-               struct reloc_cache *cache,
-               u64 target_offset)
+               struct i915_execbuffer *eb,
+               const struct i915_vma *target)
 {
+        struct drm_i915_gem_object *obj = vma->obj;
         u64 offset = reloc->offset;
-        bool wide = cache->use_64bit_reloc;
+        u64 target_offset = relocation_target(reloc, target);
+        bool wide = eb->reloc_cache.use_64bit_reloc;
         void *vaddr;
 
-        target_offset = relocation_target(reloc, target_offset);
 repeat:
-        vaddr = reloc_vaddr(obj, cache, offset >> PAGE_SHIFT);
+        vaddr = reloc_vaddr(obj, &eb->reloc_cache, offset >> PAGE_SHIFT);
         if (IS_ERR(vaddr))
                 return PTR_ERR(vaddr);
 
         clflush_write32(vaddr + offset_in_page(offset),
                         lower_32_bits(target_offset),
-                        cache->vaddr);
+                        eb->reloc_cache.vaddr);
 
         if (wide) {
                 offset += sizeof(u32);
@@ -618,17 +1106,16 @@ repeat:
                 goto repeat;
         }
 
-        return 0;
+        return target->node.start | UPDATE;
 }
 
-static int
-eb_relocate_entry(struct i915_vma *vma,
-                  struct i915_execbuffer *eb,
-                  struct drm_i915_gem_relocation_entry *reloc)
+static u64
+eb_relocate_entry(struct i915_execbuffer *eb,
+                  struct i915_vma *vma,
+                  const struct drm_i915_gem_relocation_entry *reloc)
 {
         struct i915_vma *target;
-        u64 target_offset;
-        int ret;
+        int err;
 
         /* we've already hold a reference to all valid objects */
         target = eb_get_vma(eb, reloc->target_handle);
@@ -658,27 +1145,30 @@ eb_relocate_entry(struct i915_vma *vma,
                 return -EINVAL;
         }
 
-        if (reloc->write_domain)
+        if (reloc->write_domain) {
                 target->exec_entry->flags |= EXEC_OBJECT_WRITE;
 
-        /*
-         * Sandybridge PPGTT errata: We need a global gtt mapping for MI and
-         * pipe_control writes because the gpu doesn't properly redirect them
-         * through the ppgtt for non_secure batchbuffers.
-         */
-        if (unlikely(IS_GEN6(eb->i915) &&
-                     reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
-                ret = i915_vma_bind(target, target->obj->cache_level,
-                                    PIN_GLOBAL);
-                if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!"))
-                        return ret;
+                /*
+                 * Sandybridge PPGTT errata: We need a global gtt mapping
+                 * for MI and pipe_control writes because the gpu doesn't
+                 * properly redirect them through the ppgtt for non_secure
+                 * batchbuffers.
+                 */
+                if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
+                    IS_GEN6(eb->i915)) {
+                        err = i915_vma_bind(target, target->obj->cache_level,
+                                            PIN_GLOBAL);
+                        if (WARN_ONCE(err,
+                                      "Unexpected failure to bind target VMA!"))
+                                return err;
+                }
         }
 
-        /* If the relocation already has the right value in it, no
+        /*
+         * If the relocation already has the right value in it, no
          * more work needs to be done.
          */
-        target_offset = gen8_canonical_addr(target->node.start);
-        if (target_offset == reloc->presumed_offset)
+        if (gen8_canonical_addr(target->node.start) == reloc->presumed_offset)
                 return 0;
 
         /* Check that the relocation address is valid... */
@@ -709,35 +1199,39 @@ eb_relocate_entry(struct i915_vma *vma,
          */
         vma->exec_entry->flags &= ~EXEC_OBJECT_ASYNC;
 
-        ret = relocate_entry(vma->obj, reloc, &eb->reloc_cache, target_offset);
-        if (ret)
-                return ret;
-
         /* and update the user's relocation entry */
-        reloc->presumed_offset = target_offset;
-        return 0;
+        return relocate_entry(vma, reloc, eb, target);
 }
 
-static int eb_relocate_vma(struct i915_vma *vma, struct i915_execbuffer *eb)
+static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma)
 {
 #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
-        struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
-        struct drm_i915_gem_relocation_entry __user *user_relocs;
-        struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-        int remain, ret = 0;
-
-        user_relocs = u64_to_user_ptr(entry->relocs_ptr);
+        struct drm_i915_gem_relocation_entry stack[N_RELOC(512)];
+        struct drm_i915_gem_relocation_entry __user *urelocs;
+        const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+        unsigned int remain;
 
+        urelocs = u64_to_user_ptr(entry->relocs_ptr);
         remain = entry->relocation_count;
-        while (remain) {
-                struct drm_i915_gem_relocation_entry *r = stack_reloc;
-                unsigned long unwritten;
-                unsigned int count;
+        if (unlikely(remain > N_RELOC(ULONG_MAX)))
+                return -EINVAL;
 
-                count = min_t(unsigned int, remain, ARRAY_SIZE(stack_reloc));
-                remain -= count;
+        /*
+         * We must check that the entire relocation array is safe
+         * to read. However, if the array is not writable the user loses
+         * the updated relocation values.
+         */
+        if (unlikely(!access_ok(VERIFY_READ, urelocs, remain*sizeof(urelocs))))
+                return -EFAULT;
+
+        do {
+                struct drm_i915_gem_relocation_entry *r = stack;
+                unsigned int count =
+                        min_t(unsigned int, remain, ARRAY_SIZE(stack));
+                unsigned int copied;
 
-                /* This is the fast path and we cannot handle a pagefault
+                /*
+                 * This is the fast path and we cannot handle a pagefault
                  * whilst holding the struct mutex lest the user pass in the
                  * relocations contained within a mmaped bo. For in such a case
                  * we, the page fault handler would call i915_gem_fault() and
@@ -745,409 +1239,357 @@ static int eb_relocate_vma(struct i915_vma *vma, struct i915_execbuffer *eb)
                  * this is bad and so lockdep complains vehemently.
                  */
                 pagefault_disable();
-                unwritten = __copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0]));
+                copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0]));
                 pagefault_enable();
-                if (unlikely(unwritten)) {
-                        ret = -EFAULT;
+                if (unlikely(copied)) {
+                        remain = -EFAULT;
                         goto out;
                 }
 
+                remain -= count;
                 do {
-                        u64 offset = r->presumed_offset;
+                        u64 offset = eb_relocate_entry(eb, vma, r);
 
-                        ret = eb_relocate_entry(vma, eb, r);
-                        if (ret)
+                        if (likely(offset == 0)) {
+                        } else if ((s64)offset < 0) {
+                                remain = (int)offset;
                                 goto out;
-
-                        if (r->presumed_offset != offset) {
-                                pagefault_disable();
-                                unwritten = __put_user(r->presumed_offset,
-                                                       &user_relocs->presumed_offset);
-                                pagefault_enable();
-                                if (unlikely(unwritten)) {
-                                        /* Note that reporting an error now
-                                         * leaves everything in an inconsistent
-                                         * state as we have *already* changed
-                                         * the relocation value inside the
-                                         * object. As we have not changed the
-                                         * reloc.presumed_offset or will not
-                                         * change the execobject.offset, on the
-                                         * call we may not rewrite the value
-                                         * inside the object, leaving it
-                                         * dangling and causing a GPU hang.
-                                         */
-                                        ret = -EFAULT;
-                                        goto out;
-                                }
+                        } else {
+                                /*
+                                 * Note that reporting an error now
+                                 * leaves everything in an inconsistent
+                                 * state as we have *already* changed
+                                 * the relocation value inside the
+                                 * object. As we have not changed the
+                                 * reloc.presumed_offset or will not
+                                 * change the execobject.offset, on the
+                                 * call we may not rewrite the value
+                                 * inside the object, leaving it
+                                 * dangling and causing a GPU hang. Unless
+                                 * userspace dynamically rebuilds the
+                                 * relocations on each execbuf rather than
+                                 * presume a static tree.
+                                 *
+                                 * We did previously check if the relocations
+                                 * were writable (access_ok), an error now
+                                 * would be a strange race with mprotect,
+                                 * having already demonstrated that we
+                                 * can read from this userspace address.
+                                 */
+                                offset = gen8_canonical_addr(offset & ~UPDATE);
+                                __put_user(offset,
+                                           &urelocs[r-stack].presumed_offset);
                         }
-
-                        user_relocs++;
-                        r++;
-                } while (--count);
-        }
-
+                } while (r++, --count);
+                urelocs += ARRAY_SIZE(stack);
+        } while (remain);
 out:
         reloc_cache_reset(&eb->reloc_cache);
-        return ret;
-#undef N_RELOC
+        return remain;
 }
 
 static int
-eb_relocate_vma_slow(struct i915_vma *vma,
-                     struct i915_execbuffer *eb,
-                     struct drm_i915_gem_relocation_entry *relocs)
+eb_relocate_vma_slow(struct i915_execbuffer *eb, struct i915_vma *vma)
 {
         const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-        int i, ret = 0;
+        struct drm_i915_gem_relocation_entry *relocs =
+                u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
+        unsigned int i;
+        int err;
 
         for (i = 0; i < entry->relocation_count; i++) {
-                ret = eb_relocate_entry(vma, eb, &relocs[i]);
-                if (ret)
-                        break;
+                u64 offset = eb_relocate_entry(eb, vma, &relocs[i]);
+
+                if ((s64)offset < 0) {
+                        err = (int)offset;
+                        goto err;
+                }
         }
+        err = 0;
+err:
         reloc_cache_reset(&eb->reloc_cache);
-        return ret;
+        return err;
 }
 
-static int eb_relocate(struct i915_execbuffer *eb)
+static int check_relocations(const struct drm_i915_gem_exec_object2 *entry)
 {
-        struct i915_vma *vma;
-        int ret = 0;
+        const char __user *addr, *end;
+        unsigned long size;
+        char __maybe_unused c;
 
-        list_for_each_entry(vma, &eb->vmas, exec_link) {
-                ret = eb_relocate_vma(vma, eb);
-                if (ret)
-                        break;
-        }
+        size = entry->relocation_count;
+        if (size == 0)
+                return 0;
 
-        return ret;
-}
+        if (size > N_RELOC(ULONG_MAX))
+                return -EINVAL;
 
-static bool only_mappable_for_reloc(unsigned int flags)
-{
-        return (flags & (EXEC_OBJECT_NEEDS_FENCE | __EXEC_OBJECT_NEEDS_MAP)) ==
-                __EXEC_OBJECT_NEEDS_MAP;
+        addr = u64_to_user_ptr(entry->relocs_ptr);
+        size *= sizeof(struct drm_i915_gem_relocation_entry);
+        if (!access_ok(VERIFY_READ, addr, size))
+                return -EFAULT;
+
+        end = addr + size;
+        for (; addr < end; addr += PAGE_SIZE) {
+                int err = __get_user(c, addr);
+                if (err)
+                        return err;
+        }
+        return __get_user(c, end - 1);
 }
 
-static int
-eb_reserve_vma(struct i915_vma *vma,
-               struct intel_engine_cs *engine,
-               bool *need_reloc)
+static int eb_copy_relocations(const struct i915_execbuffer *eb)
 {
-        struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-        uint64_t flags;
-        int ret;
-
-        flags = PIN_USER;
-        if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
-                flags |= PIN_GLOBAL;
-
-        if (!drm_mm_node_allocated(&vma->node)) {
-                /* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
-                 * limit address to the first 4GBs for unflagged objects.
-                 */
-                if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0)
-                        flags |= PIN_ZONE_4G;
-                if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
-                        flags |= PIN_GLOBAL | PIN_MAPPABLE;
-                if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
-                        flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
-                if (entry->flags & EXEC_OBJECT_PINNED)
-                        flags |= entry->offset | PIN_OFFSET_FIXED;
-                if ((flags & PIN_MAPPABLE) == 0)
-                        flags |= PIN_HIGH;
-        }
-
-        ret = i915_vma_pin(vma,
-                           entry->pad_to_size,
-                           entry->alignment,
-                           flags);
-        if ((ret == -ENOSPC || ret == -E2BIG) &&
-            only_mappable_for_reloc(entry->flags))
-                ret = i915_vma_pin(vma,
-                                   entry->pad_to_size,
-                                   entry->alignment,
-                                   flags & ~PIN_MAPPABLE);
-        if (ret)
-                return ret;
+        const unsigned int count = eb->buffer_count;
+        unsigned int i;
+        int err;
 
-        entry->flags |= __EXEC_OBJECT_HAS_PIN;
+        for (i = 0; i < count; i++) {
+                const unsigned int nreloc = eb->exec[i].relocation_count;
+                struct drm_i915_gem_relocation_entry __user *urelocs;
+                struct drm_i915_gem_relocation_entry *relocs;
+                unsigned long size;
+                unsigned long copied;
 
-        if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
-                ret = i915_vma_get_fence(vma);
-                if (ret)
-                        return ret;
+                if (nreloc == 0)
+                        continue;
 
-                if (i915_vma_pin_fence(vma))
-                        entry->flags |= __EXEC_OBJECT_HAS_FENCE;
-        }
+                err = check_relocations(&eb->exec[i]);
+                if (err)
+                        goto err;
 
-        if (entry->offset != vma->node.start) {
-                entry->offset = vma->node.start;
-                *need_reloc = true;
-        }
+                urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr);
+                size = nreloc * sizeof(*relocs);
 
-        return 0;
-}
+                relocs = kvmalloc_array(size, 1, GFP_TEMPORARY);
+                if (!relocs) {
+                        kvfree(relocs);
+                        err = -ENOMEM;
+                        goto err;
+                }
 
-static bool
-need_reloc_mappable(struct i915_vma *vma)
-{
-        struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+                /* copy_from_user is limited to < 4GiB */
+                copied = 0;
+                do {
+                        unsigned int len =
+                                min_t(u64, BIT_ULL(31), size - copied);
+
+                        if (__copy_from_user((char *)relocs + copied,
+                                             (char *)urelocs + copied,
+                                             len)) {
+                                kvfree(relocs);
+                                err = -EFAULT;
+                                goto err;
+                        }
 
-        if (entry->relocation_count == 0)
-                return false;
+                        copied += len;
+                } while (copied < size);
 
-        if (!i915_vma_is_ggtt(vma))
-                return false;
+                /*
+                 * As we do not update the known relocation offsets after
+                 * relocating (due to the complexities in lock handling),
+                 * we need to mark them as invalid now so that we force the
+                 * relocation processing next time. Just in case the target
+                 * object is evicted and then rebound into its old
+                 * presumed_offset before the next execbuffer - if that
+                 * happened we would make the mistake of assuming that the
+                 * relocations were valid.
+                 */
+                user_access_begin();
+                for (copied = 0; copied < nreloc; copied++)
+                        unsafe_put_user(-1,
+                                        &urelocs[copied].presumed_offset,
+                                        end_user);
+end_user:
+                user_access_end();
 
-        /* See also use_cpu_reloc() */
-        if (HAS_LLC(to_i915(vma->obj->base.dev)))
-                return false;
+                eb->exec[i].relocs_ptr = (uintptr_t)relocs;
+        }
 
-        if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
-                return false;
+        return 0;
 
-        return true;
+err:
+        while (i--) {
+                struct drm_i915_gem_relocation_entry *relocs =
+                        u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr);
+                if (eb->exec[i].relocation_count)
+                        kvfree(relocs);
+        }
+        return err;
 }
 
-static bool
-eb_vma_misplaced(struct i915_vma *vma)
+static int eb_prefault_relocations(const struct i915_execbuffer *eb)
 {
-        struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-
-        WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
-                !i915_vma_is_ggtt(vma));
-
-        if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
-                return true;
+        const unsigned int count = eb->buffer_count;
+        unsigned int i;
 
-        if (vma->node.size < entry->pad_to_size)
-                return true;
-
-        if (entry->flags & EXEC_OBJECT_PINNED &&
-            vma->node.start != entry->offset)
-                return true;
-
-        if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
-            vma->node.start < BATCH_OFFSET_BIAS)
-                return true;
+        if (unlikely(i915.prefault_disable))
+                return 0;
 
-        /* avoid costly ping-pong once a batch bo ended up non-mappable */
-        if (entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
-            !i915_vma_is_map_and_fenceable(vma))
-                return !only_mappable_for_reloc(entry->flags);
+        for (i = 0; i < count; i++) {
+                int err;
 
-        if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
-            (vma->node.start + vma->node.size - 1) >> 32)
-                return true;
+                err = check_relocations(&eb->exec[i]);
+                if (err)
+                        return err;
+        }
 
-        return false;
+        return 0;
 }
 
-static int eb_reserve(struct i915_execbuffer *eb)
+static noinline int eb_relocate_slow(struct i915_execbuffer *eb)
 {
-        const bool has_fenced_gpu_access = INTEL_GEN(eb->i915) < 4;
-        const bool needs_unfenced_map = INTEL_INFO(eb->i915)->unfenced_needs_alignment;
+        struct drm_device *dev = &eb->i915->drm;
+        bool have_copy = false;
         struct i915_vma *vma;
-        struct list_head ordered_vmas;
-        struct list_head pinned_vmas;
-        int retry;
-
-        INIT_LIST_HEAD(&ordered_vmas);
-        INIT_LIST_HEAD(&pinned_vmas);
-        while (!list_empty(&eb->vmas)) {
-                struct drm_i915_gem_exec_object2 *entry;
-                bool need_fence, need_mappable;
-
-                vma = list_first_entry(&eb->vmas, struct i915_vma, exec_link);
-                entry = vma->exec_entry;
-
-                if (eb->ctx->flags & CONTEXT_NO_ZEROMAP)
-                        entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;
-
-                if (!has_fenced_gpu_access)
-                        entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
-                need_fence =
-                        (entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
-                         needs_unfenced_map) &&
-                        i915_gem_object_is_tiled(vma->obj);
-                need_mappable = need_fence || need_reloc_mappable(vma);
-
-                if (entry->flags & EXEC_OBJECT_PINNED)
-                        list_move_tail(&vma->exec_link, &pinned_vmas);
-                else if (need_mappable) {
-                        entry->flags |= __EXEC_OBJECT_NEEDS_MAP;
-                        list_move(&vma->exec_link, &ordered_vmas);
-                } else
-                        list_move_tail(&vma->exec_link, &ordered_vmas);
-        }
-        list_splice(&ordered_vmas, &eb->vmas);
-        list_splice(&pinned_vmas, &eb->vmas);
-
-        /* Attempt to pin all of the buffers into the GTT.
-         * This is done in 3 phases:
+        int err = 0;
+
+repeat:
+        if (signal_pending(current)) {
+                err = -ERESTARTSYS;
+                goto out;
+        }
+
+        /* We may process another execbuffer during the unlock... */
+        eb_reset_vmas(eb);
+        mutex_unlock(&dev->struct_mutex);
+
+        /*
+         * We take 3 passes through the slowpatch.
          *
-         * 1a. Unbind all objects that do not match the GTT constraints for
-         *     the execbuffer (fenceable, mappable, alignment etc).
-         * 1b. Increment pin count for already bound objects.
-         * 2.  Bind new objects.
-         * 3.  Decrement pin count.
+         * 1 - we try to just prefault all the user relocation entries and
+         * then attempt to reuse the atomic pagefault disabled fast path again.
          *
-         * This avoid unnecessary unbinding of later objects in order to make
-         * room for the earlier objects *unless* we need to defragment.
+         * 2 - we copy the user entries to a local buffer here outside of the
+         * local and allow ourselves to wait upon any rendering before
+         * relocations
+         *
+         * 3 - we already have a local copy of the relocation entries, but
+         * were interrupted (EAGAIN) whilst waiting for the objects, try again.
          */
-        retry = 0;
-        do {
-                int ret = 0;
-
-                /* Unbind any ill-fitting objects or pin. */
-                list_for_each_entry(vma, &eb->vmas, exec_link) {
-                        if (!drm_mm_node_allocated(&vma->node))
-                                continue;
+        if (!err) {
+                err = eb_prefault_relocations(eb);
+        } else if (!have_copy) {
+                err = eb_copy_relocations(eb);
+                have_copy = err == 0;
+        } else {
+                cond_resched();
+                err = 0;
+        }
+        if (err) {
+                mutex_lock(&dev->struct_mutex);
+                goto out;
+        }
 
-                        if (eb_vma_misplaced(vma))
-                                ret = i915_vma_unbind(vma);
-                        else
-                                ret = eb_reserve_vma(vma, eb->engine, &eb->need_relocs);
-                        if (ret)
-                                goto err;
-                }
+        err = i915_mutex_lock_interruptible(dev);
+        if (err) {
+                mutex_lock(&dev->struct_mutex);
+                goto out;
+        }
 
-                /* Bind fresh objects */
-                list_for_each_entry(vma, &eb->vmas, exec_link) {
-                        if (drm_mm_node_allocated(&vma->node))
-                                continue;
+        /* reacquire the objects */
+        err = eb_lookup_vmas(eb);
+        if (err)
+                goto err;
 
-                        ret = eb_reserve_vma(vma, eb->engine, &eb->need_relocs);
-                        if (ret)
+        list_for_each_entry(vma, &eb->relocs, reloc_link) {
+                if (!have_copy) {
+                        pagefault_disable();
+                        err = eb_relocate_vma(eb, vma);
+                        pagefault_enable();
+                        if (err)
+                                goto repeat;
+                } else {
+                        err = eb_relocate_vma_slow(eb, vma);
+                        if (err)
                                 goto err;
                 }
+        }
 
-err:
-                if (ret != -ENOSPC || retry++)
-                        return ret;
-
-                /* Decrement pin count for bound objects */
-                list_for_each_entry(vma, &eb->vmas, exec_link)
-                        eb_unreserve_vma(vma);
+        /*
+         * Leave the user relocations as are, this is the painfully slow path,
+         * and we want to avoid the complication of dropping the lock whilst
+         * having buffers reserved in the aperture and so causing spurious
+         * ENOSPC for random operations.
+         */
 
-                ret = i915_gem_evict_vm(eb->vm, true);
-                if (ret)
-                        return ret;
-        } while (1);
-}
+err:
+        if (err == -EAGAIN)
+                goto repeat;
 
-static int
-eb_relocate_slow(struct i915_execbuffer *eb)
-{
-        const unsigned int count = eb->args->buffer_count;
-        struct drm_device *dev = &eb->i915->drm;
-        struct drm_i915_gem_relocation_entry *reloc;
-        struct i915_vma *vma;
-        int *reloc_offset;
-        int i, total, ret;
+out:
+        if (have_copy) {
+                const unsigned int count = eb->buffer_count;
+                unsigned int i;
 
-        /* We may process another execbuffer during the unlock... */
-        eb_reset(eb);
-        mutex_unlock(&dev->struct_mutex);
+                for (i = 0; i < count; i++) {
+                        const struct drm_i915_gem_exec_object2 *entry =
+                                &eb->exec[i];
+                        struct drm_i915_gem_relocation_entry *relocs;
 
-        total = 0;
-        for (i = 0; i < count; i++)
-                total += eb->exec[i].relocation_count;
+                        if (!entry->relocation_count)
+                                continue;
 
-        reloc_offset = kvmalloc_array(count, sizeof(*reloc_offset), GFP_KERNEL);
-        reloc = kvmalloc_array(total, sizeof(*reloc), GFP_KERNEL);
-        if (reloc == NULL || reloc_offset == NULL) {
-                kvfree(reloc);
-                kvfree(reloc_offset);
-                mutex_lock(&dev->struct_mutex);
-                return -ENOMEM;
+                        relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
+                        kvfree(relocs);
+                }
         }
 
-        total = 0;
-        for (i = 0; i < count; i++) {
-                struct drm_i915_gem_relocation_entry __user *user_relocs;
-                u64 invalid_offset = (u64)-1;
-                int j;
+        return err ?: have_copy;
+}
 
-                user_relocs = u64_to_user_ptr(eb->exec[i].relocs_ptr);
+static int eb_relocate(struct i915_execbuffer *eb)
+{
+        if (eb_lookup_vmas(eb))
+                goto slow;
 
-                if (copy_from_user(reloc+total, user_relocs,
-                                   eb->exec[i].relocation_count * sizeof(*reloc))) {
-                        ret = -EFAULT;
-                        mutex_lock(&dev->struct_mutex);
-                        goto err;
-                }
+        /* The objects are in their final locations, apply the relocations. */
+        if (eb->args->flags & __EXEC_HAS_RELOC) {
+                struct i915_vma *vma;
 
-                /* As we do not update the known relocation offsets after
-                 * relocating (due to the complexities in lock handling),
-                 * we need to mark them as invalid now so that we force the
-                 * relocation processing next time. Just in case the target
-                 * object is evicted and then rebound into its old
-                 * presumed_offset before the next execbuffer - if that
-                 * happened we would make the mistake of assuming that the
-                 * relocations were valid.
-                 */
-                for (j = 0; j < eb->exec[i].relocation_count; j++) {
-                        if (__copy_to_user(&user_relocs[j].presumed_offset,
-                                           &invalid_offset,
-                                           sizeof(invalid_offset))) {
-                                ret = -EFAULT;
-                                mutex_lock(&dev->struct_mutex);
-                                goto err;
-                        }
+                list_for_each_entry(vma, &eb->relocs, reloc_link) {
+                        if (eb_relocate_vma(eb, vma))
+                                goto slow;
                 }
-
-                reloc_offset[i] = total;
-                total += eb->exec[i].relocation_count;
         }
 
-        ret = i915_mutex_lock_interruptible(dev);
-        if (ret) {
-                mutex_lock(&dev->struct_mutex);
-                goto err;
-        }
-
-        /* reacquire the objects */
-        ret = eb_lookup_vmas(eb);
-        if (ret)
-                goto err;
-
-        ret = eb_reserve(eb);
-        if (ret)
-                goto err;
+        return 0;
 
-        list_for_each_entry(vma, &eb->vmas, exec_link) {
-                int idx = vma->exec_entry - eb->exec;
+slow:
+        return eb_relocate_slow(eb);
+}
 
-                ret = eb_relocate_vma_slow(vma, eb, reloc + reloc_offset[idx]);
-                if (ret)
-                        goto err;
-        }
+static void eb_export_fence(struct drm_i915_gem_object *obj,
+                            struct drm_i915_gem_request *req,
+                            unsigned int flags)
+{
+        struct reservation_object *resv = obj->resv;
 
-        /* Leave the user relocations as are, this is the painfully slow path,
-         * and we want to avoid the complication of dropping the lock whilst
-         * having buffers reserved in the aperture and so causing spurious
-         * ENOSPC for random operations.
+        /*
+         * Ignore errors from failing to allocate the new fence, we can't
+         * handle an error right now. Worst case should be missed
+         * synchronisation leading to rendering corruption.
          */
-
-err:
-        kvfree(reloc);
-        kvfree(reloc_offset);
-        return ret;
+        reservation_object_lock(resv, NULL);
+        if (flags & EXEC_OBJECT_WRITE)
+                reservation_object_add_excl_fence(resv, &req->fence);
+        else if (reservation_object_reserve_shared(resv) == 0)
+                reservation_object_add_shared_fence(resv, &req->fence);
+        reservation_object_unlock(resv);
 }
 
-static int
-eb_move_to_gpu(struct i915_execbuffer *eb)
+static int eb_move_to_gpu(struct i915_execbuffer *eb)
 {
-        struct i915_vma *vma;
-        int ret;
+        const unsigned int count = eb->buffer_count;
+        unsigned int i;
+        int err;
 
-        list_for_each_entry(vma, &eb->vmas, exec_link) {
+        for (i = 0; i < count; i++) {
+                const struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+                struct i915_vma *vma = exec_to_vma(entry);
                 struct drm_i915_gem_object *obj = vma->obj;
 
-                if (vma->exec_entry->flags & EXEC_OBJECT_CAPTURE) {
+                if (entry->flags & EXEC_OBJECT_CAPTURE) {
                         struct i915_gem_capture_list *capture;
 
                         capture = kmalloc(sizeof(*capture), GFP_KERNEL);
@@ -1159,18 +1601,32 @@ eb_move_to_gpu(struct i915_execbuffer *eb)
                         eb->request->capture_list = capture;
                 }
 
-                if (vma->exec_entry->flags & EXEC_OBJECT_ASYNC)
-                        continue;
+                if (entry->flags & EXEC_OBJECT_ASYNC)
+                        goto skip_flushes;
 
                 if (unlikely(obj->cache_dirty && !obj->cache_coherent))
                         i915_gem_clflush_object(obj, 0);
 
-                ret = i915_gem_request_await_object
-                        (eb->request, obj, vma->exec_entry->flags & EXEC_OBJECT_WRITE);
-                if (ret)
-                        return ret;
+                err = i915_gem_request_await_object
+                        (eb->request, obj, entry->flags & EXEC_OBJECT_WRITE);
+                if (err)
+                        return err;
+
+skip_flushes:
+                i915_vma_move_to_active(vma, eb->request, entry->flags);
+                __eb_unreserve_vma(vma, entry);
+                vma->exec_entry = NULL;
         }
 
+        for (i = 0; i < count; i++) {
+                const struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
+                struct i915_vma *vma = exec_to_vma(entry);
+
+                eb_export_fence(vma->obj, eb->request, entry->flags);
+                i915_vma_put(vma);
+        }
+        eb->exec = NULL;
+
         /* Unconditionally flush any chipset caches (for streaming writes). */
         i915_gem_chipset_flush(eb->i915);
 
@@ -1178,8 +1634,7 @@ eb_move_to_gpu(struct i915_execbuffer *eb)
         return eb->engine->emit_flush(eb->request, EMIT_INVALIDATE);
 }
 
-static bool
-i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
+static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
 {
         if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS)
                 return false;
@@ -1201,103 +1656,6 @@ i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
         return true;
 }
 
-static int
-validate_exec_list(struct drm_device *dev,
-                   struct drm_i915_gem_exec_object2 *exec,
-                   int count)
-{
-        unsigned relocs_total = 0;
-        unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
-        unsigned invalid_flags;
-        int i;
-
-        /* INTERNAL flags must not overlap with external ones */
-        BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & ~__EXEC_OBJECT_UNKNOWN_FLAGS);
-
-        invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
-        if (USES_FULL_PPGTT(dev))
-                invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
-
-        for (i = 0; i < count; i++) {
-                char __user *ptr = u64_to_user_ptr(exec[i].relocs_ptr);
-                int length; /* limited by fault_in_pages_readable() */
-
-                if (exec[i].flags & invalid_flags)
-                        return -EINVAL;
-
-                /* Offset can be used as input (EXEC_OBJECT_PINNED), reject
-                 * any non-page-aligned or non-canonical addresses.
-                 */
-                if (exec[i].flags & EXEC_OBJECT_PINNED) {
-                        if (exec[i].offset !=
-                            gen8_canonical_addr(exec[i].offset & PAGE_MASK))
-                                return -EINVAL;
-                }
-
-                /* From drm_mm perspective address space is continuous,
-                 * so from this point we're always using non-canonical
-                 * form internally.
-                 */
-                exec[i].offset = gen8_noncanonical_addr(exec[i].offset);
-
-                if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
-                        return -EINVAL;
-
-                /* pad_to_size was once a reserved field, so sanitize it */
-                if (exec[i].flags & EXEC_OBJECT_PAD_TO_SIZE) {
-                        if (offset_in_page(exec[i].pad_to_size))
-                                return -EINVAL;
-                } else {
-                        exec[i].pad_to_size = 0;
-                }
-
-                /* First check for malicious input causing overflow in
-                 * the worst case where we need to allocate the entire
-                 * relocation tree as a single array.
-                 */
-                if (exec[i].relocation_count > relocs_max - relocs_total)
-                        return -EINVAL;
-                relocs_total += exec[i].relocation_count;
-
-                length = exec[i].relocation_count *
-                        sizeof(struct drm_i915_gem_relocation_entry);
-                /*
-                 * We must check that the entire relocation array is safe
-                 * to read, but since we may need to update the presumed
-                 * offsets during execution, check for full write access.
-                 */
-                if (!access_ok(VERIFY_WRITE, ptr, length))
-                        return -EFAULT;
-
-                if (likely(!i915.prefault_disable)) {
-                        if (fault_in_pages_readable(ptr, length))
-                                return -EFAULT;
-                }
-        }
-
-        return 0;
-}
-
-static int eb_select_context(struct i915_execbuffer *eb)
-{
-        unsigned int ctx_id = i915_execbuffer2_get_context_id(*eb->args);
-        struct i915_gem_context *ctx;
-
-        ctx = i915_gem_context_lookup(eb->file->driver_priv, ctx_id);
-        if (unlikely(IS_ERR(ctx)))
-                return PTR_ERR(ctx);
-
-        if (unlikely(i915_gem_context_is_banned(ctx))) {
-                DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
-                return -EIO;
-        }
-
-        eb->ctx = i915_gem_context_get(ctx);
-        eb->vm = ctx->ppgtt ? &ctx->ppgtt->base : &eb->i915->ggtt.base;
-
-        return 0;
-}
-
 void i915_vma_move_to_active(struct i915_vma *vma,
                              struct drm_i915_gem_request *req,
                              unsigned int flags)
@@ -1308,7 +1666,8 @@ void i915_vma_move_to_active(struct i915_vma *vma,
         lockdep_assert_held(&req->i915->drm.struct_mutex);
         GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 
-        /* Add a reference if we're newly entering the active list.
+        /*
+         * Add a reference if we're newly entering the active list.
          * The order in which we add operations to the retirement queue is
          * vital here: mark_active adds to the start of the callback list,
          * such that subsequent callbacks are called first. Therefore we
@@ -1336,44 +1695,7 @@ void i915_vma_move_to_active(struct i915_vma *vma,
                 i915_gem_active_set(&vma->last_fence, req);
 }
 
-static void eb_export_fence(struct drm_i915_gem_object *obj,
-                            struct drm_i915_gem_request *req,
-                            unsigned int flags)
-{
-        struct reservation_object *resv = obj->resv;
-
-        /* Ignore errors from failing to allocate the new fence, we can't
-         * handle an error right now. Worst case should be missed
-         * synchronisation leading to rendering corruption.
-         */
-        reservation_object_lock(resv, NULL);
-        if (flags & EXEC_OBJECT_WRITE)
-                reservation_object_add_excl_fence(resv, &req->fence);
-        else if (reservation_object_reserve_shared(resv) == 0)
-                reservation_object_add_shared_fence(resv, &req->fence);
-        reservation_object_unlock(resv);
-}
-
-static void
-eb_move_to_active(struct i915_execbuffer *eb)
-{
-        struct i915_vma *vma;
-
-        list_for_each_entry(vma, &eb->vmas, exec_link) {
-                struct drm_i915_gem_object *obj = vma->obj;
-
-                obj->base.write_domain = 0;
-                if (vma->exec_entry->flags & EXEC_OBJECT_WRITE)
-                        obj->base.read_domains = 0;
-                obj->base.read_domains |= I915_GEM_GPU_DOMAINS;
-
-                i915_vma_move_to_active(vma, eb->request, vma->exec_entry->flags);
-                eb_export_fence(obj, eb->request, vma->exec_entry->flags);
-        }
-}
-
-static int
-i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
+static int i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
 {
         u32 *cs;
         int i;
@@ -1383,16 +1705,16 @@ i915_reset_gen7_sol_offsets(struct drm_i915_gem_request *req)
                 return -EINVAL;
         }
 
-        cs = intel_ring_begin(req, 4 * 3);
+        cs = intel_ring_begin(req, 4 * 2 + 2);
         if (IS_ERR(cs))
                 return PTR_ERR(cs);
 
+        *cs++ = MI_LOAD_REGISTER_IMM(4);
         for (i = 0; i < 4; i++) {
-                *cs++ = MI_LOAD_REGISTER_IMM(1);
                 *cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i));
                 *cs++ = 0;
         }
-
+        *cs++ = MI_NOOP;
         intel_ring_advance(req, cs);
 
         return 0;
@@ -1402,24 +1724,24 @@ static struct i915_vma *eb_parse(struct i915_execbuffer *eb, bool is_master)
 {
         struct drm_i915_gem_object *shadow_batch_obj;
         struct i915_vma *vma;
-        int ret;
+        int err;
 
         shadow_batch_obj = i915_gem_batch_pool_get(&eb->engine->batch_pool,
                                                    PAGE_ALIGN(eb->batch_len));
         if (IS_ERR(shadow_batch_obj))
                 return ERR_CAST(shadow_batch_obj);
 
-        ret = intel_engine_cmd_parser(eb->engine,
+        err = intel_engine_cmd_parser(eb->engine,
                                       eb->batch->obj,
                                       shadow_batch_obj,
                                       eb->batch_start_offset,
                                       eb->batch_len,
                                       is_master);
-        if (ret) {
-                if (ret == -EACCES) /* unhandled chained batch */
+        if (err) {
+                if (err == -EACCES) /* unhandled chained batch */
                         vma = NULL;
                 else
-                        vma = ERR_PTR(ret);
+                        vma = ERR_PTR(err);
                 goto out;
         }
 
@@ -1428,10 +1750,10 @@ static struct i915_vma *eb_parse(struct i915_execbuffer *eb, bool is_master)
                 goto out;
 
         vma->exec_entry =
-                memset(&eb->shadow_exec_entry, 0, sizeof(*vma->exec_entry));
+                memset(&eb->exec[eb->buffer_count++],
+                       0, sizeof(*vma->exec_entry));
         vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
-        i915_gem_object_get(shadow_batch_obj);
-        list_add_tail(&vma->exec_link, &eb->vmas);
+        __exec_to_vma(vma->exec_entry) = (uintptr_t)i915_vma_get(vma);
 
 out:
         i915_gem_object_unpin_pages(shadow_batch_obj);
@@ -1439,41 +1761,37 @@ out:
 }
 
 static void
-add_to_client(struct drm_i915_gem_request *req,
-              struct drm_file *file)
+add_to_client(struct drm_i915_gem_request *req, struct drm_file *file)
 {
         req->file_priv = file->driver_priv;
         list_add_tail(&req->client_link, &req->file_priv->mm.request_list);
 }
 
-static int
-execbuf_submit(struct i915_execbuffer *eb)
+static int eb_submit(struct i915_execbuffer *eb)
 {
-        int ret;
+        int err;
 
-        ret = eb_move_to_gpu(eb);
-        if (ret)
-                return ret;
+        err = eb_move_to_gpu(eb);
+        if (err)
+                return err;
 
-        ret = i915_switch_context(eb->request);
-        if (ret)
-                return ret;
+        err = i915_switch_context(eb->request);
+        if (err)
+                return err;
 
         if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) {
-                ret = i915_reset_gen7_sol_offsets(eb->request);
-                if (ret)
-                        return ret;
+                err = i915_reset_gen7_sol_offsets(eb->request);
+                if (err)
+                        return err;
         }
 
-        ret = eb->engine->emit_bb_start(eb->request,
+        err = eb->engine->emit_bb_start(eb->request,
                                         eb->batch->node.start +
                                         eb->batch_start_offset,
                                         eb->batch_len,
-                                        eb->dispatch_flags);
-        if (ret)
-                return ret;
-
-        eb_move_to_active(eb);
+                                        eb->batch_flags);
+        if (err)
+                return err;
 
         return 0;
 }
@@ -1564,34 +1882,36 @@ i915_gem_do_execbuffer(struct drm_device *dev,
         struct dma_fence *in_fence = NULL;
         struct sync_file *out_fence = NULL;
         int out_fence_fd = -1;
-        int ret;
+        int err;
 
-        if (!i915_gem_check_execbuffer(args))
-                return -EINVAL;
-
-        ret = validate_exec_list(dev, exec, args->buffer_count);
-        if (ret)
-                return ret;
+        BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &
+                     ~__EXEC_OBJECT_UNKNOWN_FLAGS);
 
         eb.i915 = to_i915(dev);
         eb.file = file;
         eb.args = args;
+        if (!(args->flags & I915_EXEC_NO_RELOC))
+                args->flags |= __EXEC_HAS_RELOC;
         eb.exec = exec;
-        eb.need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
+        eb.ctx = NULL;
+        eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
+        if (USES_FULL_PPGTT(eb.i915))
+                eb.invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
         reloc_cache_init(&eb.reloc_cache, eb.i915);
 
+        eb.buffer_count = args->buffer_count;
         eb.batch_start_offset = args->batch_start_offset;
         eb.batch_len = args->batch_len;
 
-        eb.dispatch_flags = 0;
+        eb.batch_flags = 0;
         if (args->flags & I915_EXEC_SECURE) {
                 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
                     return -EPERM;
 
-                eb.dispatch_flags |= I915_DISPATCH_SECURE;
+                eb.batch_flags |= I915_DISPATCH_SECURE;
         }
         if (args->flags & I915_EXEC_IS_PINNED)
-                eb.dispatch_flags |= I915_DISPATCH_PINNED;
+                eb.batch_flags |= I915_DISPATCH_PINNED;
 
         eb.engine = eb_select_engine(eb.i915, file, args);
         if (!eb.engine)
@@ -1608,7 +1928,7 @@ i915_gem_do_execbuffer(struct drm_device *dev,
                         return -EINVAL;
                 }
 
-                eb.dispatch_flags |= I915_DISPATCH_RS;
+                eb.batch_flags |= I915_DISPATCH_RS;
         }
 
         if (args->flags & I915_EXEC_FENCE_IN) {
@@ -1620,71 +1940,53 @@ i915_gem_do_execbuffer(struct drm_device *dev,
         if (args->flags & I915_EXEC_FENCE_OUT) {
                 out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
                 if (out_fence_fd < 0) {
-                        ret = out_fence_fd;
+                        err = out_fence_fd;
                         goto err_in_fence;
                 }
         }
 
-        /* Take a local wakeref for preparing to dispatch the execbuf as
+        if (eb_create(&eb))
+                return -ENOMEM;
+
+        /*
+         * Take a local wakeref for preparing to dispatch the execbuf as
          * we expect to access the hardware fairly frequently in the
          * process. Upon first dispatch, we acquire another prolonged
          * wakeref that we hold until the GPU has been idle for at least
          * 100ms.
          */
         intel_runtime_pm_get(eb.i915);
+        err = i915_mutex_lock_interruptible(dev);
+        if (err)
+                goto err_rpm;
 
-        ret = i915_mutex_lock_interruptible(dev);
-        if (ret)
-                goto pre_mutex_err;
-
-        ret = eb_select_context(&eb);
-        if (ret) {
-                mutex_unlock(&dev->struct_mutex);
-                goto pre_mutex_err;
-        }
-
-        if (eb_create(&eb)) {
-                i915_gem_context_put(eb.ctx);
-                mutex_unlock(&dev->struct_mutex);
-                ret = -ENOMEM;
-                goto pre_mutex_err;
-        }
-
-        /* Look up object handles */
-        ret = eb_lookup_vmas(&eb);
-        if (ret)
-                goto err;
-
-        /* take note of the batch buffer before we might reorder the lists */
-        eb.batch = eb_get_batch(&eb);
-
-        /* Move the objects en-masse into the GTT, evicting if necessary. */
-        ret = eb_reserve(&eb);
-        if (ret)
-                goto err;
+        err = eb_select_context(&eb);
+        if (unlikely(err))
+                goto err_unlock;
 
-        /* The objects are in their final locations, apply the relocations. */
-        if (eb.need_relocs)
-                ret = eb_relocate(&eb);
-        if (ret) {
-                if (ret == -EFAULT) {
-                        ret = eb_relocate_slow(&eb);
-                        BUG_ON(!mutex_is_locked(&dev->struct_mutex));
-                }
-                if (ret)
-                        goto err;
-        }
+        err = eb_relocate(&eb);
+        if (err)
+                /*
+                 * If the user expects the execobject.offset and
+                 * reloc.presumed_offset to be an exact match,
+                 * as for using NO_RELOC, then we cannot update
+                 * the execobject.offset until we have completed
+                 * relocation.
+                 */
+                args->flags &= ~__EXEC_HAS_RELOC;
+        if (err < 0)
+                goto err_vma;
 
-        if (eb.batch->exec_entry->flags & EXEC_OBJECT_WRITE) {
+        if (unlikely(eb.batch->exec_entry->flags & EXEC_OBJECT_WRITE)) {
                 DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
-                ret = -EINVAL;
-                goto err;
+                err = -EINVAL;
+                goto err_vma;
         }
         if (eb.batch_start_offset > eb.batch->size ||
             eb.batch_len > eb.batch->size - eb.batch_start_offset) {
                 DRM_DEBUG("Attempting to use out-of-bounds batch\n");
-                ret = -EINVAL;
-                goto err;
+                err = -EINVAL;
+                goto err_vma;
         }
 
         if (eb.engine->needs_cmd_parser && eb.batch_len) {
@@ -1692,8 +1994,8 @@ i915_gem_do_execbuffer(struct drm_device *dev,
 
                 vma = eb_parse(&eb, drm_is_current_master(file));
                 if (IS_ERR(vma)) {
-                        ret = PTR_ERR(vma);
-                        goto err;
+                        err = PTR_ERR(vma);
+                        goto err_vma;
                 }
 
                 if (vma) {
@@ -1706,7 +2008,7 @@ i915_gem_do_execbuffer(struct drm_device *dev,
                          * specifically don't want that set on batches the
                          * command parser has accepted.
                          */
-                        eb.dispatch_flags |= I915_DISPATCH_SECURE;
+                        eb.batch_flags |= I915_DISPATCH_SECURE;
                         eb.batch_start_offset = 0;
                         eb.batch = vma;
                 }
@@ -1715,11 +2017,11 @@ i915_gem_do_execbuffer(struct drm_device *dev,
         if (eb.batch_len == 0)
                 eb.batch_len = eb.batch->size - eb.batch_start_offset;
 
-        /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
+        /*
+         * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
          * batch" bit. Hence we need to pin secure batches into the global gtt.
          * hsw should have this fixed, but bdw mucks it up again. */
-        if (eb.dispatch_flags & I915_DISPATCH_SECURE) {
-                struct drm_i915_gem_object *obj = eb.batch->obj;
+        if (eb.batch_flags & I915_DISPATCH_SECURE) {
                 struct i915_vma *vma;
 
                 /*
@@ -1732,10 +2034,10 @@ i915_gem_do_execbuffer(struct drm_device *dev,
                  *   fitting due to fragmentation.
                  * So this is actually safe.
                  */
-                vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
+                vma = i915_gem_object_ggtt_pin(eb.batch->obj, NULL, 0, 0, 0);
                 if (IS_ERR(vma)) {
-                        ret = PTR_ERR(vma);
-                        goto err;
+                        err = PTR_ERR(vma);
+                        goto err_vma;
                 }
 
                 eb.batch = vma;
@@ -1744,25 +2046,26 @@ i915_gem_do_execbuffer(struct drm_device *dev,
         /* Allocate a request for this batch buffer nice and early. */
         eb.request = i915_gem_request_alloc(eb.engine, eb.ctx);
         if (IS_ERR(eb.request)) {
-                ret = PTR_ERR(eb.request);
+                err = PTR_ERR(eb.request);
                 goto err_batch_unpin;
         }
 
         if (in_fence) {
-                ret = i915_gem_request_await_dma_fence(eb.request, in_fence);
-                if (ret < 0)
+                err = i915_gem_request_await_dma_fence(eb.request, in_fence);
+                if (err < 0)
                         goto err_request;
         }
 
         if (out_fence_fd != -1) {
                 out_fence = sync_file_create(&eb.request->fence);
                 if (!out_fence) {
-                        ret = -ENOMEM;
+                        err = -ENOMEM;
                         goto err_request;
                 }
         }
 
-        /* Whilst this request exists, batch_obj will be on the
+        /*
+         * Whilst this request exists, batch_obj will be on the
          * active_list, and so will hold the active reference. Only when this
          * request is retired will the the batch_obj be moved onto the
          * inactive_list and lose its active reference. Hence we do not need
@@ -1770,14 +2073,14 @@ i915_gem_do_execbuffer(struct drm_device *dev,
          */
         eb.request->batch = eb.batch;
 
-        trace_i915_gem_request_queue(eb.request, eb.dispatch_flags);
-        ret = execbuf_submit(&eb);
+        trace_i915_gem_request_queue(eb.request, eb.batch_flags);
+        err = eb_submit(&eb);
 err_request:
-        __i915_add_request(eb.request, ret == 0);
+        __i915_add_request(eb.request, err == 0);
         add_to_client(eb.request, file);
 
         if (out_fence) {
-                if (ret == 0) {
+                if (err == 0) {
                         fd_install(out_fence_fd, out_fence->file);
                         args->rsvd2 &= GENMASK_ULL(0, 31); /* keep in-fence */
                         args->rsvd2 |= (u64)out_fence_fd << 32;
@@ -1788,28 +2091,22 @@ err_request:
         }
 
 err_batch_unpin:
-        /*
-         * FIXME: We crucially rely upon the active tracking for the (ppgtt)
-         * batch vma for correctness. For less ugly and less fragility this
-         * needs to be adjusted to also track the ggtt batch vma properly as
-         * active.
-         */
-        if (eb.dispatch_flags & I915_DISPATCH_SECURE)
+        if (eb.batch_flags & I915_DISPATCH_SECURE)
                 i915_vma_unpin(eb.batch);
-err:
-        /* the request owns the ref now */
-        eb_destroy(&eb);
+err_vma:
+        if (eb.exec)
+                eb_release_vmas(&eb);
+        i915_gem_context_put(eb.ctx);
+err_unlock:
         mutex_unlock(&dev->struct_mutex);
-
-pre_mutex_err:
-        /* intel_gpu_busy should also get a ref, so it will free when the device
-         * is really idle. */
+err_rpm:
         intel_runtime_pm_put(eb.i915);
+        eb_destroy(&eb);
         if (out_fence_fd != -1)
                 put_unused_fd(out_fence_fd);
 err_in_fence:
         dma_fence_put(in_fence);
-        return ret;
+        return err;
 }
 
 /*
@@ -1820,20 +2117,38 @@ int
 i915_gem_execbuffer(struct drm_device *dev, void *data,
                     struct drm_file *file)
 {
+        const size_t sz = sizeof(struct drm_i915_gem_exec_object2);
         struct drm_i915_gem_execbuffer *args = data;
         struct drm_i915_gem_execbuffer2 exec2;
         struct drm_i915_gem_exec_object *exec_list = NULL;
         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
-        int ret, i;
+        unsigned int i;
+        int err;
 
-        if (args->buffer_count < 1) {
-                DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
+        if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
+                DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
                 return -EINVAL;
         }
 
+        exec2.buffers_ptr = args->buffers_ptr;
+        exec2.buffer_count = args->buffer_count;
+        exec2.batch_start_offset = args->batch_start_offset;
+        exec2.batch_len = args->batch_len;
+        exec2.DR1 = args->DR1;
+        exec2.DR4 = args->DR4;
+        exec2.num_cliprects = args->num_cliprects;
+        exec2.cliprects_ptr = args->cliprects_ptr;
+        exec2.flags = I915_EXEC_RENDER;
+        i915_execbuffer2_set_context_id(exec2, 0);
+
+        if (!i915_gem_check_execbuffer(&exec2))
+                return -EINVAL;
+
         /* Copy in the exec list from userland */
-        exec_list = kvmalloc_array(sizeof(*exec_list), args->buffer_count, GFP_KERNEL);
-        exec2_list = kvmalloc_array(sizeof(*exec2_list), args->buffer_count, GFP_KERNEL);
+        exec_list = kvmalloc_array(args->buffer_count, sizeof(*exec_list),
+                                   __GFP_NOWARN | GFP_TEMPORARY);
+        exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
+                                    __GFP_NOWARN | GFP_TEMPORARY);
         if (exec_list == NULL || exec2_list == NULL) {
                 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
                           args->buffer_count);
@@ -1841,12 +2156,12 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
                 kvfree(exec2_list);
                 return -ENOMEM;
         }
-        ret = copy_from_user(exec_list,
+        err = copy_from_user(exec_list,
                              u64_to_user_ptr(args->buffers_ptr),
                              sizeof(*exec_list) * args->buffer_count);
-        if (ret != 0) {
+        if (err) {
                 DRM_DEBUG("copy %d exec entries failed %d\n",
-                          args->buffer_count, ret);
+                          args->buffer_count, err);
                 kvfree(exec_list);
                 kvfree(exec2_list);
                 return -EFAULT;
@@ -1864,99 +2179,94 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
                         exec2_list[i].flags = 0;
         }
 
-        exec2.buffers_ptr = args->buffers_ptr;
-        exec2.buffer_count = args->buffer_count;
-        exec2.batch_start_offset = args->batch_start_offset;
-        exec2.batch_len = args->batch_len;
-        exec2.DR1 = args->DR1;
-        exec2.DR4 = args->DR4;
-        exec2.num_cliprects = args->num_cliprects;
-        exec2.cliprects_ptr = args->cliprects_ptr;
-        exec2.flags = I915_EXEC_RENDER;
-        i915_execbuffer2_set_context_id(exec2, 0);
-
-        ret = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list);
-        if (!ret) {
+        err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list);
+        if (exec2.flags & __EXEC_HAS_RELOC) {
                 struct drm_i915_gem_exec_object __user *user_exec_list =
                         u64_to_user_ptr(args->buffers_ptr);
 
                 /* Copy the new buffer offsets back to the user's exec list. */
                 for (i = 0; i < args->buffer_count; i++) {
+                        if (!(exec2_list[i].offset & UPDATE))
+                                continue;
+
                         exec2_list[i].offset =
-                                gen8_canonical_addr(exec2_list[i].offset);
-                        ret = __copy_to_user(&user_exec_list[i].offset,
-                                             &exec2_list[i].offset,
-                                             sizeof(user_exec_list[i].offset));
-                        if (ret) {
-                                ret = -EFAULT;
-                                DRM_DEBUG("failed to copy %d exec entries "
-                                          "back to user (%d)\n",
-                                          args->buffer_count, ret);
+                                gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
+                        exec2_list[i].offset &= PIN_OFFSET_MASK;
+                        if (__copy_to_user(&user_exec_list[i].offset,
+                                           &exec2_list[i].offset,
+                                           sizeof(user_exec_list[i].offset)))
                                 break;
-                        }
                 }
         }
 
         kvfree(exec_list);
         kvfree(exec2_list);
-        return ret;
+        return err;
 }
 
 int
 i915_gem_execbuffer2(struct drm_device *dev, void *data,
                      struct drm_file *file)
 {
+        const size_t sz = sizeof(struct drm_i915_gem_exec_object2);
         struct drm_i915_gem_execbuffer2 *args = data;
-        struct drm_i915_gem_exec_object2 *exec2_list = NULL;
-        int ret;
+        struct drm_i915_gem_exec_object2 *exec2_list;
+        int err;
 
-        if (args->buffer_count < 1 ||
-            args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
+        if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
                 DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
                 return -EINVAL;
         }
 
-        exec2_list = kvmalloc_array(args->buffer_count,
-                                    sizeof(*exec2_list),
-                                    GFP_TEMPORARY);
+        if (!i915_gem_check_execbuffer(args))
+                return -EINVAL;
+
+        /* Allocate an extra slot for use by the command parser */
+        exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
+                                    __GFP_NOWARN | GFP_TEMPORARY);
         if (exec2_list == NULL) {
                 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
                           args->buffer_count);
                 return -ENOMEM;
         }
-        ret = copy_from_user(exec2_list,
-                             u64_to_user_ptr(args->buffers_ptr),
-                             sizeof(*exec2_list) * args->buffer_count);
-        if (ret != 0) {
-                DRM_DEBUG("copy %d exec entries failed %d\n",
-                          args->buffer_count, ret);
+        if (copy_from_user(exec2_list,
+                           u64_to_user_ptr(args->buffers_ptr),
+                           sizeof(*exec2_list) * args->buffer_count)) {
+                DRM_DEBUG("copy %d exec entries failed\n", args->buffer_count);
                 kvfree(exec2_list);
                 return -EFAULT;
         }
 
-        ret = i915_gem_do_execbuffer(dev, file, args, exec2_list);
-        if (!ret) {
-                /* Copy the new buffer offsets back to the user's exec list. */
+        err = i915_gem_do_execbuffer(dev, file, args, exec2_list);
+
+        /*
+         * Now that we have begun execution of the batchbuffer, we ignore
+         * any new error after this point. Also given that we have already
+         * updated the associated relocations, we try to write out the current
+         * object locations irrespective of any error.
+         */
+        if (args->flags & __EXEC_HAS_RELOC) {
                 struct drm_i915_gem_exec_object2 __user *user_exec_list =
-                                   u64_to_user_ptr(args->buffers_ptr);
-                int i;
+                        u64_to_user_ptr(args->buffers_ptr);
+                unsigned int i;
 
+                /* Copy the new buffer offsets back to the user's exec list. */
+                user_access_begin();
                 for (i = 0; i < args->buffer_count; i++) {
+                        if (!(exec2_list[i].offset & UPDATE))
+                                continue;
+
                         exec2_list[i].offset =
-                                gen8_canonical_addr(exec2_list[i].offset);
-                        ret = __copy_to_user(&user_exec_list[i].offset,
-                                             &exec2_list[i].offset,
-                                             sizeof(user_exec_list[i].offset));
-                        if (ret) {
-                                ret = -EFAULT;
-                                DRM_DEBUG("failed to copy %d exec entries "
-                                          "back to user\n",
-                                          args->buffer_count);
-                                break;
-                        }
+                                gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
+                        unsafe_put_user(exec2_list[i].offset,
+                                        &user_exec_list[i].offset,
+                                        end_user);
                 }
+end_user:
+                user_access_end();
         }
 
+        args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS;
         kvfree(exec2_list);
-        return ret;
+        return err;
 }
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index ce68194ebff6..9e6a47323362 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -463,7 +463,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
                           size, obj->base.size,
                           flags & PIN_MAPPABLE ? "mappable" : "total",
                           end);
-                return -E2BIG;
+                return -ENOSPC;
         }
 
         ret = i915_gem_object_pin_pages(obj);
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index ea98e6e4262f..04d7a5da70fd 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -103,6 +103,7 @@ struct i915_vma {
 
         /** This vma's place in the execbuf reservation list */
         struct list_head exec_link;
+        struct list_head reloc_link;
 
         /** This vma's place in the eviction list */
         struct list_head evict_link;
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
index 14e9c2fbc4e6..5ea373221f49 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
@@ -304,7 +304,7 @@ static int igt_evict_vm(void *arg)
                 goto cleanup;
 
         /* Everything is pinned, nothing should happen */
-        err = i915_gem_evict_vm(&ggtt->base, false);
+        err = i915_gem_evict_vm(&ggtt->base);
         if (err) {
                 pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",
                        err);
@@ -313,7 +313,7 @@ static int igt_evict_vm(void *arg)
 
         unpin_ggtt(i915);
 
-        err = i915_gem_evict_vm(&ggtt->base, false);
+        err = i915_gem_evict_vm(&ggtt->base);
         if (err) {
                 pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",
                        err);
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index ad56566e24db..fb9072d5877f 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -225,14 +225,6 @@ static bool assert_pin_valid(const struct i915_vma *vma,
 }
 
 __maybe_unused
-static bool assert_pin_e2big(const struct i915_vma *vma,
-                             const struct pin_mode *mode,
-                             int result)
-{
-        return result == -E2BIG;
-}
-
-__maybe_unused
 static bool assert_pin_enospc(const struct i915_vma *vma,
                               const struct pin_mode *mode,
                               int result)
@@ -255,7 +247,6 @@ static int igt_vma_pin1(void *arg)
 #define VALID(sz, fl) { .size = (sz), .flags = (fl), .assert = assert_pin_valid, .string = #sz ", " #fl ", (valid) " }
 #define __INVALID(sz, fl, check, eval) { .size = (sz), .flags = (fl), .assert = (check), .string = #sz ", " #fl ", (invalid " #eval ")" }
 #define INVALID(sz, fl) __INVALID(sz, fl, assert_pin_einval, EINVAL)
-#define TOOBIG(sz, fl) __INVALID(sz, fl, assert_pin_e2big, E2BIG)
 #define NOSPACE(sz, fl) __INVALID(sz, fl, assert_pin_enospc, ENOSPC)
                 VALID(0, PIN_GLOBAL),
                 VALID(0, PIN_GLOBAL | PIN_MAPPABLE),
@@ -276,11 +267,11 @@ static int igt_vma_pin1(void *arg)
                 VALID(8192, PIN_GLOBAL),
                 VALID(i915->ggtt.mappable_end - 4096, PIN_GLOBAL | PIN_MAPPABLE),
                 VALID(i915->ggtt.mappable_end, PIN_GLOBAL | PIN_MAPPABLE),
-                TOOBIG(i915->ggtt.mappable_end + 4096, PIN_GLOBAL | PIN_MAPPABLE),
+                NOSPACE(i915->ggtt.mappable_end + 4096, PIN_GLOBAL | PIN_MAPPABLE),
                 VALID(i915->ggtt.base.total - 4096, PIN_GLOBAL),
                 VALID(i915->ggtt.base.total, PIN_GLOBAL),
-                TOOBIG(i915->ggtt.base.total + 4096, PIN_GLOBAL),
-                TOOBIG(round_down(U64_MAX, PAGE_SIZE), PIN_GLOBAL),
+                NOSPACE(i915->ggtt.base.total + 4096, PIN_GLOBAL),
+                NOSPACE(round_down(U64_MAX, PAGE_SIZE), PIN_GLOBAL),
                 INVALID(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (i915->ggtt.mappable_end - 4096)),
                 INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (i915->ggtt.base.total - 4096)),
                 INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (round_down(U64_MAX, PAGE_SIZE) - 4096)),
@@ -300,7 +291,6 @@ static int igt_vma_pin1(void *arg)
 #endif
                 { },
 #undef NOSPACE
-#undef TOOBIG
 #undef INVALID
 #undef __INVALID
 #undef VALID