Merge branch 'amdkfd-v6' of git://people.freedesktop.org/~gabbayo/linux into drm-next

Merge AMDKFD it seems clean enough.
* 'amdkfd-v6' of git://people.freedesktop.org/~gabbayo/linux: (29 commits)
  amdkfd: Implement the Get Version IOCTL
  amdkfd: Implement the Get Process Aperture IOCTL
  amdkfd: Implement the Get Clock Counters IOCTL
  amdkfd: Implement the Set Memory Policy IOCTL
  amdkfd: Implement the create/destroy/update queue IOCTLs
  amdkfd: Add interrupt handling module
  amdkfd: Add device queue manager module
  amdkfd: Add process queue manager module
  amdkfd: Add packet manager module
  amdkfd: Add module parameter of scheduling policy
  amdkfd: Add kernel queue module
  amdkfd: Add mqd_manager module
  amdkfd: Add queue module
  amdkfd: Add binding/unbinding calls to amd_iommu driver
  amdkfd: Add basic modules to amdkfd
  amdkfd: Add topology module to amdkfd
  amdkfd: Add amdkfd skeleton driver
  amdkfd: Add IOCTL set definitions of amdkfd
  Update MAINTAINERS and CREDITS files with amdkfd info
  drm/radeon: Add radeon <--> amdkfd interface
  ...

1 files changed, 83 insertions, 5 deletions

diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index 88787bb4b3b9..94d19f64cecf 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -98,11 +98,11 @@ struct mmu_notifier_ops {
         /*
          * invalidate_range_start() and invalidate_range_end() must be
          * paired and are called only when the mmap_sem and/or the
-         * locks protecting the reverse maps are held. The subsystem
-         * must guarantee that no additional references are taken to
-         * the pages in the range established between the call to
-         * invalidate_range_start() and the matching call to
-         * invalidate_range_end().
+         * locks protecting the reverse maps are held. If the subsystem
+         * can't guarantee that no additional references are taken to
+         * the pages in the range, it has to implement the
+         * invalidate_range() notifier to remove any references taken
+         * after invalidate_range_start().
          *
          * Invalidation of multiple concurrent ranges may be
          * optionally permitted by the driver. Either way the
@@ -144,6 +144,29 @@ struct mmu_notifier_ops {
         void (*invalidate_range_end)(struct mmu_notifier *mn,
                                      struct mm_struct *mm,
                                      unsigned long start, unsigned long end);
+
+        /*
+         * invalidate_range() is either called between
+         * invalidate_range_start() and invalidate_range_end() when the
+         * VM has to free pages that where unmapped, but before the
+         * pages are actually freed, or outside of _start()/_end() when
+         * a (remote) TLB is necessary.
+         *
+         * If invalidate_range() is used to manage a non-CPU TLB with
+         * shared page-tables, it not necessary to implement the
+         * invalidate_range_start()/end() notifiers, as
+         * invalidate_range() alread catches the points in time when an
+         * external TLB range needs to be flushed.
+         *
+         * The invalidate_range() function is called under the ptl
+         * spin-lock and not allowed to sleep.
+         *
+         * Note that this function might be called with just a sub-range
+         * of what was passed to invalidate_range_start()/end(), if
+         * called between those functions.
+         */
+        void (*invalidate_range)(struct mmu_notifier *mn, struct mm_struct *mm,
+                                 unsigned long start, unsigned long end);
 };
 
 /*
@@ -190,6 +213,8 @@ extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
                                   unsigned long start, unsigned long end);
 extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
                                   unsigned long start, unsigned long end);
+extern void __mmu_notifier_invalidate_range(struct mm_struct *mm,
+                                  unsigned long start, unsigned long end);
 
 static inline void mmu_notifier_release(struct mm_struct *mm)
 {
@@ -242,6 +267,13 @@ static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
                 __mmu_notifier_invalidate_range_end(mm, start, end);
 }
 
+static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
+                                  unsigned long start, unsigned long end)
+{
+        if (mm_has_notifiers(mm))
+                __mmu_notifier_invalidate_range(mm, start, end);
+}
+
 static inline void mmu_notifier_mm_init(struct mm_struct *mm)
 {
         mm->mmu_notifier_mm = NULL;
@@ -279,6 +311,44 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
         __young;                                                        \
 })
 
+#define ptep_clear_flush_notify(__vma, __address, __ptep)               \
+({                                                                      \
+        unsigned long ___addr = __address & PAGE_MASK;                  \
+        struct mm_struct *___mm = (__vma)->vm_mm;                       \
+        pte_t ___pte;                                                   \
+                                                                        \
+        ___pte = ptep_clear_flush(__vma, __address, __ptep);            \
+        mmu_notifier_invalidate_range(___mm, ___addr,                   \
+                                        ___addr + PAGE_SIZE);           \
+                                                                        \
+        ___pte;                                                         \
+})
+
+#define pmdp_clear_flush_notify(__vma, __haddr, __pmd)                  \
+({                                                                      \
+        unsigned long ___haddr = __haddr & HPAGE_PMD_MASK;              \
+        struct mm_struct *___mm = (__vma)->vm_mm;                       \
+        pmd_t ___pmd;                                                   \
+                                                                        \
+        ___pmd = pmdp_clear_flush(__vma, __haddr, __pmd);               \
+        mmu_notifier_invalidate_range(___mm, ___haddr,                  \
+                                      ___haddr + HPAGE_PMD_SIZE);       \
+                                                                        \
+        ___pmd;                                                         \
+})
+
+#define pmdp_get_and_clear_notify(__mm, __haddr, __pmd)                 \
+({                                                                      \
+        unsigned long ___haddr = __haddr & HPAGE_PMD_MASK;              \
+        pmd_t ___pmd;                                                   \
+                                                                        \
+        ___pmd = pmdp_get_and_clear(__mm, __haddr, __pmd);              \
+        mmu_notifier_invalidate_range(__mm, ___haddr,                   \
+                                      ___haddr + HPAGE_PMD_SIZE);       \
+                                                                        \
+        ___pmd;                                                         \
+})
+
 /*
  * set_pte_at_notify() sets the pte _after_ running the notifier.
  * This is safe to start by updating the secondary MMUs, because the primary MMU
@@ -342,6 +412,11 @@ static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
 {
 }
 
+static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
+                                  unsigned long start, unsigned long end)
+{
+}
+
 static inline void mmu_notifier_mm_init(struct mm_struct *mm)
 {
 }
@@ -352,6 +427,9 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
 
 #define ptep_clear_flush_young_notify ptep_clear_flush_young
 #define pmdp_clear_flush_young_notify pmdp_clear_flush_young
+#define ptep_clear_flush_notify ptep_clear_flush
+#define pmdp_clear_flush_notify pmdp_clear_flush
+#define pmdp_get_and_clear_notify pmdp_get_and_clear
 #define set_pte_at_notify set_pte_at
 
 #endif /* CONFIG_MMU_NOTIFIER */