1 files changed, 98 insertions, 64 deletions

diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index a9717b3fbf1b..54529b837afa 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -150,9 +150,6 @@ extern int radeon_backlight;
 /* number of hw syncs before falling back on blocking */
 #define RADEON_NUM_SYNCS                        4
 
-/* number of hw syncs before falling back on blocking */
-#define RADEON_NUM_SYNCS                        4
-
 /* hardcode those limit for now */
 #define RADEON_VA_IB_OFFSET                     (1 << 20)
 #define RADEON_VA_RESERVED_SIZE                 (8 << 20)
@@ -363,14 +360,15 @@ struct radeon_fence_driver {
 };
 
 struct radeon_fence {
-        struct fence base;
+        struct fence            base;
 
-        struct radeon_device            *rdev;
-        uint64_t                        seq;
+        struct radeon_device    *rdev;
+        uint64_t                seq;
         /* RB, DMA, etc. */
-        unsigned                        ring;
+        unsigned                ring;
+        bool                    is_vm_update;
 
-        wait_queue_t                    fence_wake;
+        wait_queue_t            fence_wake;
 };
 
 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring);
@@ -452,12 +450,22 @@ struct radeon_mman {
 #endif
 };
 
+struct radeon_bo_list {
+        struct radeon_bo                *robj;
+        struct ttm_validate_buffer      tv;
+        uint64_t                        gpu_offset;
+        unsigned                        prefered_domains;
+        unsigned                        allowed_domains;
+        uint32_t                        tiling_flags;
+};
+
 /* bo virtual address in a specific vm */
 struct radeon_bo_va {
         /* protected by bo being reserved */
         struct list_head                bo_list;
         uint32_t                        flags;
         uint64_t                        addr;
+        struct radeon_fence             *last_pt_update;
         unsigned                        ref_count;
 
         /* protected by vm mutex */
@@ -474,7 +482,7 @@ struct radeon_bo {
         struct list_head                list;
         /* Protected by tbo.reserved */
         u32                             initial_domain;
-        struct ttm_place                placements[3];
+        struct ttm_place                placements[4];
         struct ttm_placement            placement;
         struct ttm_buffer_object        tbo;
         struct ttm_bo_kmap_obj          kmap;
@@ -576,10 +584,9 @@ int radeon_mode_dumb_mmap(struct drm_file *filp,
  * Semaphores.
  */
 struct radeon_semaphore {
-        struct radeon_sa_bo             *sa_bo;
-        signed                          waiters;
-        uint64_t                        gpu_addr;
-        struct radeon_fence             *sync_to[RADEON_NUM_RINGS];
+        struct radeon_sa_bo     *sa_bo;
+        signed                  waiters;
+        uint64_t                gpu_addr;
 };
 
 int radeon_semaphore_create(struct radeon_device *rdev,
@@ -588,20 +595,33 @@ bool radeon_semaphore_emit_signal(struct radeon_device *rdev, int ring,
                                   struct radeon_semaphore *semaphore);
 bool radeon_semaphore_emit_wait(struct radeon_device *rdev, int ring,
                                 struct radeon_semaphore *semaphore);
-void radeon_semaphore_sync_fence(struct radeon_semaphore *semaphore,
-                                 struct radeon_fence *fence);
-int radeon_semaphore_sync_resv(struct radeon_device *rdev,
-                               struct radeon_semaphore *semaphore,
-                               struct reservation_object *resv,
-                               bool shared);
-int radeon_semaphore_sync_rings(struct radeon_device *rdev,
-                                struct radeon_semaphore *semaphore,
-                                int waiting_ring);
 void radeon_semaphore_free(struct radeon_device *rdev,
                            struct radeon_semaphore **semaphore,
                            struct radeon_fence *fence);
 
 /*
+ * Synchronization
+ */
+struct radeon_sync {
+        struct radeon_semaphore *semaphores[RADEON_NUM_SYNCS];
+        struct radeon_fence     *sync_to[RADEON_NUM_RINGS];
+        struct radeon_fence     *last_vm_update;
+};
+
+void radeon_sync_create(struct radeon_sync *sync);
+void radeon_sync_fence(struct radeon_sync *sync,
+                       struct radeon_fence *fence);
+int radeon_sync_resv(struct radeon_device *rdev,
+                     struct radeon_sync *sync,
+                     struct reservation_object *resv,
+                     bool shared);
+int radeon_sync_rings(struct radeon_device *rdev,
+                      struct radeon_sync *sync,
+                      int waiting_ring);
+void radeon_sync_free(struct radeon_device *rdev, struct radeon_sync *sync,
+                      struct radeon_fence *fence);
+
+/*
  * GART structures, functions & helpers
  */
 struct radeon_mc;
@@ -701,6 +721,10 @@ struct radeon_doorbell {
 
 int radeon_doorbell_get(struct radeon_device *rdev, u32 *page);
 void radeon_doorbell_free(struct radeon_device *rdev, u32 doorbell);
+void radeon_doorbell_get_kfd_info(struct radeon_device *rdev,
+                                  phys_addr_t *aperture_base,
+                                  size_t *aperture_size,
+                                  size_t *start_offset);
 
 /*
  * IRQS.
@@ -814,7 +838,7 @@ struct radeon_ib {
         struct radeon_fence             *fence;
         struct radeon_vm                *vm;
         bool                            is_const_ib;
-        struct radeon_semaphore         *semaphore;
+        struct radeon_sync              sync;
 };
 
 struct radeon_ring {
@@ -891,33 +915,40 @@ struct radeon_vm_pt {
         uint64_t                        addr;
 };
 
+struct radeon_vm_id {
+        unsigned                id;
+        uint64_t                pd_gpu_addr;
+        /* last flushed PD/PT update */
+        struct radeon_fence     *flushed_updates;
+        /* last use of vmid */
+        struct radeon_fence     *last_id_use;
+};
+
 struct radeon_vm {
-        struct rb_root                  va;
-        unsigned                        id;
+        struct mutex            mutex;
+
+        struct rb_root          va;
+
+        /* protecting invalidated and freed */
+        spinlock_t              status_lock;
 
         /* BOs moved, but not yet updated in the PT */
-        struct list_head                invalidated;
+        struct list_head        invalidated;
 
         /* BOs freed, but not yet updated in the PT */
-        struct list_head                freed;
+        struct list_head        freed;
 
         /* contains the page directory */
-        struct radeon_bo                *page_directory;
-        uint64_t                        pd_gpu_addr;
-        unsigned                        max_pde_used;
+        struct radeon_bo        *page_directory;
+        unsigned                max_pde_used;
 
         /* array of page tables, one for each page directory entry */
-        struct radeon_vm_pt             *page_tables;
+        struct radeon_vm_pt     *page_tables;
 
-        struct radeon_bo_va             *ib_bo_va;
+        struct radeon_bo_va     *ib_bo_va;
 
-        struct mutex                    mutex;
-        /* last fence for cs using this vm */
-        struct radeon_fence             *fence;
-        /* last flush or NULL if we still need to flush */
-        struct radeon_fence             *last_flush;
-        /* last use of vmid */
-        struct radeon_fence             *last_id_use;
+        /* for id and flush management per ring */
+        struct radeon_vm_id     ids[RADEON_NUM_RINGS];
 };
 
 struct radeon_vm_manager {
@@ -1025,19 +1056,7 @@ void cayman_dma_fini(struct radeon_device *rdev);
 /*
  * CS.
  */
-struct radeon_cs_reloc {
-        struct drm_gem_object           *gobj;
-        struct radeon_bo                *robj;
-        struct ttm_validate_buffer      tv;
-        uint64_t                        gpu_offset;
-        unsigned                        prefered_domains;
-        unsigned                        allowed_domains;
-        uint32_t                        tiling_flags;
-        uint32_t                        handle;
-};
-
 struct radeon_cs_chunk {
-        uint32_t                chunk_id;
         uint32_t                length_dw;
         uint32_t                *kdata;
         void __user             *user_ptr;
@@ -1055,16 +1074,15 @@ struct radeon_cs_parser {
         unsigned                idx;
         /* relocations */
         unsigned                nrelocs;
-        struct radeon_cs_reloc  *relocs;
-        struct radeon_cs_reloc  **relocs_ptr;
-        struct radeon_cs_reloc  *vm_bos;
+        struct radeon_bo_list   *relocs;
+        struct radeon_bo_list   *vm_bos;
         struct list_head        validated;
         unsigned                dma_reloc_idx;
         /* indices of various chunks */
-        int                     chunk_ib_idx;
-        int                     chunk_relocs_idx;
-        int                     chunk_flags_idx;
-        int                     chunk_const_ib_idx;
+        struct radeon_cs_chunk  *chunk_ib;
+        struct radeon_cs_chunk  *chunk_relocs;
+        struct radeon_cs_chunk  *chunk_flags;
+        struct radeon_cs_chunk  *chunk_const_ib;
         struct radeon_ib        ib;
         struct radeon_ib        const_ib;
         void                    *track;
@@ -1078,7 +1096,7 @@ struct radeon_cs_parser {
 
 static inline u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
 {
-        struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
+        struct radeon_cs_chunk *ibc = p->chunk_ib;
 
         if (ibc->kdata)
                 return ibc->kdata[idx];
@@ -1490,6 +1508,10 @@ struct radeon_dpm_fan {
         u8 t_hyst;
         u32 cycle_delay;
         u16 t_max;
+        u8 control_mode;
+        u16 default_max_fan_pwm;
+        u16 default_fan_output_sensitivity;
+        u16 fan_output_sensitivity;
         bool ucode_fan_control;
 };
 
@@ -1623,6 +1645,11 @@ struct radeon_pm {
         /* internal thermal controller on rv6xx+ */
         enum radeon_int_thermal_type int_thermal_type;
         struct device           *int_hwmon_dev;
+        /* fan control parameters */
+        bool                    no_fan;
+        u8                      fan_pulses_per_revolution;
+        u8                      fan_min_rpm;
+        u8                      fan_max_rpm;
         /* dpm */
         bool                    dpm_enabled;
         struct radeon_dpm       dpm;
@@ -1785,7 +1812,8 @@ struct radeon_asic_ring {
         void (*hdp_flush)(struct radeon_device *rdev, struct radeon_ring *ring);
         bool (*emit_semaphore)(struct radeon_device *rdev, struct radeon_ring *cp,
                                struct radeon_semaphore *semaphore, bool emit_wait);
-        void (*vm_flush)(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
+        void (*vm_flush)(struct radeon_device *rdev, struct radeon_ring *ring,
+                         unsigned vm_id, uint64_t pd_addr);
 
         /* testing functions */
         int (*ring_test)(struct radeon_device *rdev, struct radeon_ring *cp);
@@ -2388,6 +2416,8 @@ struct radeon_device {
         struct radeon_atcs              atcs;
         /* srbm instance registers */
         struct mutex                    srbm_mutex;
+        /* GRBM index mutex. Protects concurrents access to GRBM index */
+        struct mutex                    grbm_idx_mutex;
         /* clock, powergating flags */
         u32 cg_flags;
         u32 pg_flags;
@@ -2400,6 +2430,10 @@ struct radeon_device {
         u64 vram_pin_size;
         u64 gart_pin_size;
 
+        /* amdkfd interface */
+        struct kfd_dev          *kfd;
+        struct radeon_sa_manager        kfd_bo;
+
         struct mutex    mn_lock;
         DECLARE_HASHTABLE(mn_hash, 7);
 };
@@ -2831,7 +2865,7 @@ static inline void radeon_ring_write(struct radeon_ring *ring, uint32_t v)
 #define radeon_ring_ib_execute(rdev, r, ib) (rdev)->asic->ring[(r)]->ib_execute((rdev), (ib))
 #define radeon_ring_ib_parse(rdev, r, ib) (rdev)->asic->ring[(r)]->ib_parse((rdev), (ib))
 #define radeon_ring_is_lockup(rdev, r, cp) (rdev)->asic->ring[(r)]->is_lockup((rdev), (cp))
-#define radeon_ring_vm_flush(rdev, r, vm) (rdev)->asic->ring[(r)]->vm_flush((rdev), (r), (vm))
+#define radeon_ring_vm_flush(rdev, r, vm_id, pd_addr) (rdev)->asic->ring[(r)->idx]->vm_flush((rdev), (r), (vm_id), (pd_addr))
 #define radeon_ring_get_rptr(rdev, r) (rdev)->asic->ring[(r)->idx]->get_rptr((rdev), (r))
 #define radeon_ring_get_wptr(rdev, r) (rdev)->asic->ring[(r)->idx]->get_wptr((rdev), (r))
 #define radeon_ring_set_wptr(rdev, r) (rdev)->asic->ring[(r)->idx]->set_wptr((rdev), (r))
@@ -2940,14 +2974,14 @@ int radeon_vm_manager_init(struct radeon_device *rdev);
 void radeon_vm_manager_fini(struct radeon_device *rdev);
 int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm);
 void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm);
-struct radeon_cs_reloc *radeon_vm_get_bos(struct radeon_device *rdev,
+struct radeon_bo_list *radeon_vm_get_bos(struct radeon_device *rdev,
                                           struct radeon_vm *vm,
                                           struct list_head *head);
 struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,
                                        struct radeon_vm *vm, int ring);
 void radeon_vm_flush(struct radeon_device *rdev,
                      struct radeon_vm *vm,
-                     int ring);
+                     int ring, struct radeon_fence *fence);
 void radeon_vm_fence(struct radeon_device *rdev,
                      struct radeon_vm *vm,
                      struct radeon_fence *fence);
@@ -3054,7 +3088,7 @@ bool radeon_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p);
 void radeon_cs_dump_packet(struct radeon_cs_parser *p,
                            struct radeon_cs_packet *pkt);
 int radeon_cs_packet_next_reloc(struct radeon_cs_parser *p,
-                                struct radeon_cs_reloc **cs_reloc,
+                                struct radeon_bo_list **cs_reloc,
                                 int nomm);
 int r600_cs_common_vline_parse(struct radeon_cs_parser *p,
                                uint32_t *vline_start_end,