aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/amd/amdkfd
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd')
-rw-r--r--drivers/gpu/drm/amd/amdkfd/Makefile3
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device.c20
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c2
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c176
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_priv.h15
5 files changed, 3 insertions, 213 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
index be6246de5091..307a309110e6 100644
--- a/drivers/gpu/drm/amd/amdkfd/Makefile
+++ b/drivers/gpu/drm/amd/amdkfd/Makefile
@@ -8,7 +8,6 @@ amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \
8 kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \ 8 kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \
9 kfd_process.o kfd_queue.o kfd_mqd_manager.o \ 9 kfd_process.o kfd_queue.o kfd_mqd_manager.o \
10 kfd_kernel_queue.o kfd_packet_manager.o \ 10 kfd_kernel_queue.o kfd_packet_manager.o \
11 kfd_process_queue_manager.o kfd_device_queue_manager.o \ 11 kfd_process_queue_manager.o kfd_device_queue_manager.o
12 kfd_interrupt.o
13 12
14obj-$(CONFIG_HSA_AMD) += amdkfd.o 13obj-$(CONFIG_HSA_AMD) += amdkfd.o
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 43884ebd4303..633532a2e7ec 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -192,13 +192,6 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
192 goto kfd_topology_add_device_error; 192 goto kfd_topology_add_device_error;
193 } 193 }
194 194
195 if (kfd_interrupt_init(kfd)) {
196 dev_err(kfd_device,
197 "Error initializing interrupts for device (%x:%x)\n",
198 kfd->pdev->vendor, kfd->pdev->device);
199 goto kfd_interrupt_error;
200 }
201
202 if (!device_iommu_pasid_init(kfd)) { 195 if (!device_iommu_pasid_init(kfd)) {
203 dev_err(kfd_device, 196 dev_err(kfd_device,
204 "Error initializing iommuv2 for device (%x:%x)\n", 197 "Error initializing iommuv2 for device (%x:%x)\n",
@@ -237,8 +230,6 @@ dqm_start_error:
237device_queue_manager_error: 230device_queue_manager_error:
238 amd_iommu_free_device(kfd->pdev); 231 amd_iommu_free_device(kfd->pdev);
239device_iommu_pasid_error: 232device_iommu_pasid_error:
240 kfd_interrupt_exit(kfd);
241kfd_interrupt_error:
242 kfd_topology_remove_device(kfd); 233 kfd_topology_remove_device(kfd);
243kfd_topology_add_device_error: 234kfd_topology_add_device_error:
244 kfd2kgd->fini_sa_manager(kfd->kgd); 235 kfd2kgd->fini_sa_manager(kfd->kgd);
@@ -254,7 +245,6 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
254 if (kfd->init_complete) { 245 if (kfd->init_complete) {
255 device_queue_manager_uninit(kfd->dqm); 246 device_queue_manager_uninit(kfd->dqm);
256 amd_iommu_free_device(kfd->pdev); 247 amd_iommu_free_device(kfd->pdev);
257 kfd_interrupt_exit(kfd);
258 kfd_topology_remove_device(kfd); 248 kfd_topology_remove_device(kfd);
259 } 249 }
260 250
@@ -296,13 +286,5 @@ int kgd2kfd_resume(struct kfd_dev *kfd)
296/* This is called directly from KGD at ISR. */ 286/* This is called directly from KGD at ISR. */
297void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) 287void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
298{ 288{
299 if (kfd->init_complete) { 289 /* Process interrupts / schedule work as necessary */
300 spin_lock(&kfd->interrupt_lock);
301
302 if (kfd->interrupts_active
303 && enqueue_ih_ring_entry(kfd, ih_ring_entry))
304 schedule_work(&kfd->interrupt_work);
305
306 spin_unlock(&kfd->interrupt_lock);
307 }
308} 290}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 9c8961d22360..30c8fda9622e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -280,7 +280,7 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
280 q->queue); 280 q->queue);
281 281
282 retval = mqd->load_mqd(mqd, q->mqd, q->pipe, 282 retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
283 q->queue, q->properties.write_ptr); 283 q->queue, (uint32_t __user *) q->properties.write_ptr);
284 if (retval != 0) { 284 if (retval != 0) {
285 deallocate_hqd(dqm, q); 285 deallocate_hqd(dqm, q);
286 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); 286 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
deleted file mode 100644
index 5b999095a1f7..000000000000
--- a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c
+++ /dev/null
@@ -1,176 +0,0 @@
1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23/*
24 * KFD Interrupts.
25 *
26 * AMD GPUs deliver interrupts by pushing an interrupt description onto the
27 * interrupt ring and then sending an interrupt. KGD receives the interrupt
28 * in ISR and sends us a pointer to each new entry on the interrupt ring.
29 *
30 * We generally can't process interrupt-signaled events from ISR, so we call
31 * out to each interrupt client module (currently only the scheduler) to ask if
32 * each interrupt is interesting. If they return true, then it requires further
33 * processing so we copy it to an internal interrupt ring and call each
34 * interrupt client again from a work-queue.
35 *
36 * There's no acknowledgment for the interrupts we use. The hardware simply
37 * queues a new interrupt each time without waiting.
38 *
39 * The fixed-size internal queue means that it's possible for us to lose
40 * interrupts because we have no back-pressure to the hardware.
41 */
42
43#include <linux/slab.h>
44#include <linux/device.h>
45#include "kfd_priv.h"
46
47#define KFD_INTERRUPT_RING_SIZE 256
48
49static void interrupt_wq(struct work_struct *);
50
51int kfd_interrupt_init(struct kfd_dev *kfd)
52{
53 void *interrupt_ring = kmalloc_array(KFD_INTERRUPT_RING_SIZE,
54 kfd->device_info->ih_ring_entry_size,
55 GFP_KERNEL);
56 if (!interrupt_ring)
57 return -ENOMEM;
58
59 kfd->interrupt_ring = interrupt_ring;
60 kfd->interrupt_ring_size =
61 KFD_INTERRUPT_RING_SIZE * kfd->device_info->ih_ring_entry_size;
62 atomic_set(&kfd->interrupt_ring_wptr, 0);
63 atomic_set(&kfd->interrupt_ring_rptr, 0);
64
65 spin_lock_init(&kfd->interrupt_lock);
66
67 INIT_WORK(&kfd->interrupt_work, interrupt_wq);
68
69 kfd->interrupts_active = true;
70
71 /*
72 * After this function returns, the interrupt will be enabled. This
73 * barrier ensures that the interrupt running on a different processor
74 * sees all the above writes.
75 */
76 smp_wmb();
77
78 return 0;
79}
80
81void kfd_interrupt_exit(struct kfd_dev *kfd)
82{
83 /*
84 * Stop the interrupt handler from writing to the ring and scheduling
85 * workqueue items. The spinlock ensures that any interrupt running
86 * after we have unlocked sees interrupts_active = false.
87 */
88 unsigned long flags;
89
90 spin_lock_irqsave(&kfd->interrupt_lock, flags);
91 kfd->interrupts_active = false;
92 spin_unlock_irqrestore(&kfd->interrupt_lock, flags);
93
94 /*
95 * Flush_scheduled_work ensures that there are no outstanding
96 * work-queue items that will access interrupt_ring. New work items
97 * can't be created because we stopped interrupt handling above.
98 */
99 flush_scheduled_work();
100
101 kfree(kfd->interrupt_ring);
102}
103
104/*
105 * This assumes that it can't be called concurrently with itself
106 * but only with dequeue_ih_ring_entry.
107 */
108bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry)
109{
110 unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr);
111 unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr);
112
113 if ((rptr - wptr) % kfd->interrupt_ring_size ==
114 kfd->device_info->ih_ring_entry_size) {
115 /* This is very bad, the system is likely to hang. */
116 dev_err_ratelimited(kfd_chardev(),
117 "Interrupt ring overflow, dropping interrupt.\n");
118 return false;
119 }
120
121 memcpy(kfd->interrupt_ring + wptr, ih_ring_entry,
122 kfd->device_info->ih_ring_entry_size);
123
124 wptr = (wptr + kfd->device_info->ih_ring_entry_size) %
125 kfd->interrupt_ring_size;
126 smp_wmb(); /* Ensure memcpy'd data is visible before wptr update. */
127 atomic_set(&kfd->interrupt_ring_wptr, wptr);
128
129 return true;
130}
131
132/*
133 * This assumes that it can't be called concurrently with itself
134 * but only with enqueue_ih_ring_entry.
135 */
136static bool dequeue_ih_ring_entry(struct kfd_dev *kfd, void *ih_ring_entry)
137{
138 /*
139 * Assume that wait queues have an implicit barrier, i.e. anything that
140 * happened in the ISR before it queued work is visible.
141 */
142
143 unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr);
144 unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr);
145
146 if (rptr == wptr)
147 return false;
148
149 memcpy(ih_ring_entry, kfd->interrupt_ring + rptr,
150 kfd->device_info->ih_ring_entry_size);
151
152 rptr = (rptr + kfd->device_info->ih_ring_entry_size) %
153 kfd->interrupt_ring_size;
154
155 /*
156 * Ensure the rptr write update is not visible until
157 * memcpy has finished reading.
158 */
159 smp_mb();
160 atomic_set(&kfd->interrupt_ring_rptr, rptr);
161
162 return true;
163}
164
165static void interrupt_wq(struct work_struct *work)
166{
167 struct kfd_dev *dev = container_of(work, struct kfd_dev,
168 interrupt_work);
169
170 uint32_t ih_ring_entry[DIV_ROUND_UP(
171 dev->device_info->ih_ring_entry_size,
172 sizeof(uint32_t))];
173
174 while (dequeue_ih_ring_entry(dev, ih_ring_entry))
175 ;
176}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index a5edb29507e3..b3dc13c83169 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -135,22 +135,10 @@ struct kfd_dev {
135 135
136 struct kgd2kfd_shared_resources shared_resources; 136 struct kgd2kfd_shared_resources shared_resources;
137 137
138 void *interrupt_ring;
139 size_t interrupt_ring_size;
140 atomic_t interrupt_ring_rptr;
141 atomic_t interrupt_ring_wptr;
142 struct work_struct interrupt_work;
143 spinlock_t interrupt_lock;
144
145 /* QCM Device instance */ 138 /* QCM Device instance */
146 struct device_queue_manager *dqm; 139 struct device_queue_manager *dqm;
147 140
148 bool init_complete; 141 bool init_complete;
149 /*
150 * Interrupts of interest to KFD are copied
151 * from the HW ring into a SW ring.
152 */
153 bool interrupts_active;
154}; 142};
155 143
156/* KGD2KFD callbacks */ 144/* KGD2KFD callbacks */
@@ -531,10 +519,7 @@ struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
531struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx); 519struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx);
532 520
533/* Interrupts */ 521/* Interrupts */
534int kfd_interrupt_init(struct kfd_dev *dev);
535void kfd_interrupt_exit(struct kfd_dev *dev);
536void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); 522void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
537bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry);
538 523
539/* Power Management */ 524/* Power Management */
540void kgd2kfd_suspend(struct kfd_dev *kfd); 525void kgd2kfd_suspend(struct kfd_dev *kfd);