diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd')
-rw-r--r-- | drivers/gpu/drm/amd/amdkfd/Makefile | 3 | ||||
-rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device.c | 20 | ||||
-rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 2 | ||||
-rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c | 176 | ||||
-rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 15 |
5 files changed, 3 insertions, 213 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile index be6246de5091..307a309110e6 100644 --- a/drivers/gpu/drm/amd/amdkfd/Makefile +++ b/drivers/gpu/drm/amd/amdkfd/Makefile | |||
@@ -8,7 +8,6 @@ amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \ | |||
8 | kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \ | 8 | kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \ |
9 | kfd_process.o kfd_queue.o kfd_mqd_manager.o \ | 9 | kfd_process.o kfd_queue.o kfd_mqd_manager.o \ |
10 | kfd_kernel_queue.o kfd_packet_manager.o \ | 10 | kfd_kernel_queue.o kfd_packet_manager.o \ |
11 | kfd_process_queue_manager.o kfd_device_queue_manager.o \ | 11 | kfd_process_queue_manager.o kfd_device_queue_manager.o |
12 | kfd_interrupt.o | ||
13 | 12 | ||
14 | obj-$(CONFIG_HSA_AMD) += amdkfd.o | 13 | obj-$(CONFIG_HSA_AMD) += amdkfd.o |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 43884ebd4303..633532a2e7ec 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c | |||
@@ -192,13 +192,6 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, | |||
192 | goto kfd_topology_add_device_error; | 192 | goto kfd_topology_add_device_error; |
193 | } | 193 | } |
194 | 194 | ||
195 | if (kfd_interrupt_init(kfd)) { | ||
196 | dev_err(kfd_device, | ||
197 | "Error initializing interrupts for device (%x:%x)\n", | ||
198 | kfd->pdev->vendor, kfd->pdev->device); | ||
199 | goto kfd_interrupt_error; | ||
200 | } | ||
201 | |||
202 | if (!device_iommu_pasid_init(kfd)) { | 195 | if (!device_iommu_pasid_init(kfd)) { |
203 | dev_err(kfd_device, | 196 | dev_err(kfd_device, |
204 | "Error initializing iommuv2 for device (%x:%x)\n", | 197 | "Error initializing iommuv2 for device (%x:%x)\n", |
@@ -237,8 +230,6 @@ dqm_start_error: | |||
237 | device_queue_manager_error: | 230 | device_queue_manager_error: |
238 | amd_iommu_free_device(kfd->pdev); | 231 | amd_iommu_free_device(kfd->pdev); |
239 | device_iommu_pasid_error: | 232 | device_iommu_pasid_error: |
240 | kfd_interrupt_exit(kfd); | ||
241 | kfd_interrupt_error: | ||
242 | kfd_topology_remove_device(kfd); | 233 | kfd_topology_remove_device(kfd); |
243 | kfd_topology_add_device_error: | 234 | kfd_topology_add_device_error: |
244 | kfd2kgd->fini_sa_manager(kfd->kgd); | 235 | kfd2kgd->fini_sa_manager(kfd->kgd); |
@@ -254,7 +245,6 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd) | |||
254 | if (kfd->init_complete) { | 245 | if (kfd->init_complete) { |
255 | device_queue_manager_uninit(kfd->dqm); | 246 | device_queue_manager_uninit(kfd->dqm); |
256 | amd_iommu_free_device(kfd->pdev); | 247 | amd_iommu_free_device(kfd->pdev); |
257 | kfd_interrupt_exit(kfd); | ||
258 | kfd_topology_remove_device(kfd); | 248 | kfd_topology_remove_device(kfd); |
259 | } | 249 | } |
260 | 250 | ||
@@ -296,13 +286,5 @@ int kgd2kfd_resume(struct kfd_dev *kfd) | |||
296 | /* This is called directly from KGD at ISR. */ | 286 | /* This is called directly from KGD at ISR. */ |
297 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) | 287 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) |
298 | { | 288 | { |
299 | if (kfd->init_complete) { | 289 | /* Process interrupts / schedule work as necessary */ |
300 | spin_lock(&kfd->interrupt_lock); | ||
301 | |||
302 | if (kfd->interrupts_active | ||
303 | && enqueue_ih_ring_entry(kfd, ih_ring_entry)) | ||
304 | schedule_work(&kfd->interrupt_work); | ||
305 | |||
306 | spin_unlock(&kfd->interrupt_lock); | ||
307 | } | ||
308 | } | 290 | } |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 9c8961d22360..30c8fda9622e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | |||
@@ -280,7 +280,7 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, | |||
280 | q->queue); | 280 | q->queue); |
281 | 281 | ||
282 | retval = mqd->load_mqd(mqd, q->mqd, q->pipe, | 282 | retval = mqd->load_mqd(mqd, q->mqd, q->pipe, |
283 | q->queue, q->properties.write_ptr); | 283 | q->queue, (uint32_t __user *) q->properties.write_ptr); |
284 | if (retval != 0) { | 284 | if (retval != 0) { |
285 | deallocate_hqd(dqm, q); | 285 | deallocate_hqd(dqm, q); |
286 | mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); | 286 | mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c deleted file mode 100644 index 5b999095a1f7..000000000000 --- a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c +++ /dev/null | |||
@@ -1,176 +0,0 @@ | |||
1 | /* | ||
2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | /* | ||
24 | * KFD Interrupts. | ||
25 | * | ||
26 | * AMD GPUs deliver interrupts by pushing an interrupt description onto the | ||
27 | * interrupt ring and then sending an interrupt. KGD receives the interrupt | ||
28 | * in ISR and sends us a pointer to each new entry on the interrupt ring. | ||
29 | * | ||
30 | * We generally can't process interrupt-signaled events from ISR, so we call | ||
31 | * out to each interrupt client module (currently only the scheduler) to ask if | ||
32 | * each interrupt is interesting. If they return true, then it requires further | ||
33 | * processing so we copy it to an internal interrupt ring and call each | ||
34 | * interrupt client again from a work-queue. | ||
35 | * | ||
36 | * There's no acknowledgment for the interrupts we use. The hardware simply | ||
37 | * queues a new interrupt each time without waiting. | ||
38 | * | ||
39 | * The fixed-size internal queue means that it's possible for us to lose | ||
40 | * interrupts because we have no back-pressure to the hardware. | ||
41 | */ | ||
42 | |||
43 | #include <linux/slab.h> | ||
44 | #include <linux/device.h> | ||
45 | #include "kfd_priv.h" | ||
46 | |||
47 | #define KFD_INTERRUPT_RING_SIZE 256 | ||
48 | |||
49 | static void interrupt_wq(struct work_struct *); | ||
50 | |||
51 | int kfd_interrupt_init(struct kfd_dev *kfd) | ||
52 | { | ||
53 | void *interrupt_ring = kmalloc_array(KFD_INTERRUPT_RING_SIZE, | ||
54 | kfd->device_info->ih_ring_entry_size, | ||
55 | GFP_KERNEL); | ||
56 | if (!interrupt_ring) | ||
57 | return -ENOMEM; | ||
58 | |||
59 | kfd->interrupt_ring = interrupt_ring; | ||
60 | kfd->interrupt_ring_size = | ||
61 | KFD_INTERRUPT_RING_SIZE * kfd->device_info->ih_ring_entry_size; | ||
62 | atomic_set(&kfd->interrupt_ring_wptr, 0); | ||
63 | atomic_set(&kfd->interrupt_ring_rptr, 0); | ||
64 | |||
65 | spin_lock_init(&kfd->interrupt_lock); | ||
66 | |||
67 | INIT_WORK(&kfd->interrupt_work, interrupt_wq); | ||
68 | |||
69 | kfd->interrupts_active = true; | ||
70 | |||
71 | /* | ||
72 | * After this function returns, the interrupt will be enabled. This | ||
73 | * barrier ensures that the interrupt running on a different processor | ||
74 | * sees all the above writes. | ||
75 | */ | ||
76 | smp_wmb(); | ||
77 | |||
78 | return 0; | ||
79 | } | ||
80 | |||
81 | void kfd_interrupt_exit(struct kfd_dev *kfd) | ||
82 | { | ||
83 | /* | ||
84 | * Stop the interrupt handler from writing to the ring and scheduling | ||
85 | * workqueue items. The spinlock ensures that any interrupt running | ||
86 | * after we have unlocked sees interrupts_active = false. | ||
87 | */ | ||
88 | unsigned long flags; | ||
89 | |||
90 | spin_lock_irqsave(&kfd->interrupt_lock, flags); | ||
91 | kfd->interrupts_active = false; | ||
92 | spin_unlock_irqrestore(&kfd->interrupt_lock, flags); | ||
93 | |||
94 | /* | ||
95 | * Flush_scheduled_work ensures that there are no outstanding | ||
96 | * work-queue items that will access interrupt_ring. New work items | ||
97 | * can't be created because we stopped interrupt handling above. | ||
98 | */ | ||
99 | flush_scheduled_work(); | ||
100 | |||
101 | kfree(kfd->interrupt_ring); | ||
102 | } | ||
103 | |||
104 | /* | ||
105 | * This assumes that it can't be called concurrently with itself | ||
106 | * but only with dequeue_ih_ring_entry. | ||
107 | */ | ||
108 | bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry) | ||
109 | { | ||
110 | unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr); | ||
111 | unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr); | ||
112 | |||
113 | if ((rptr - wptr) % kfd->interrupt_ring_size == | ||
114 | kfd->device_info->ih_ring_entry_size) { | ||
115 | /* This is very bad, the system is likely to hang. */ | ||
116 | dev_err_ratelimited(kfd_chardev(), | ||
117 | "Interrupt ring overflow, dropping interrupt.\n"); | ||
118 | return false; | ||
119 | } | ||
120 | |||
121 | memcpy(kfd->interrupt_ring + wptr, ih_ring_entry, | ||
122 | kfd->device_info->ih_ring_entry_size); | ||
123 | |||
124 | wptr = (wptr + kfd->device_info->ih_ring_entry_size) % | ||
125 | kfd->interrupt_ring_size; | ||
126 | smp_wmb(); /* Ensure memcpy'd data is visible before wptr update. */ | ||
127 | atomic_set(&kfd->interrupt_ring_wptr, wptr); | ||
128 | |||
129 | return true; | ||
130 | } | ||
131 | |||
132 | /* | ||
133 | * This assumes that it can't be called concurrently with itself | ||
134 | * but only with enqueue_ih_ring_entry. | ||
135 | */ | ||
136 | static bool dequeue_ih_ring_entry(struct kfd_dev *kfd, void *ih_ring_entry) | ||
137 | { | ||
138 | /* | ||
139 | * Assume that wait queues have an implicit barrier, i.e. anything that | ||
140 | * happened in the ISR before it queued work is visible. | ||
141 | */ | ||
142 | |||
143 | unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr); | ||
144 | unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr); | ||
145 | |||
146 | if (rptr == wptr) | ||
147 | return false; | ||
148 | |||
149 | memcpy(ih_ring_entry, kfd->interrupt_ring + rptr, | ||
150 | kfd->device_info->ih_ring_entry_size); | ||
151 | |||
152 | rptr = (rptr + kfd->device_info->ih_ring_entry_size) % | ||
153 | kfd->interrupt_ring_size; | ||
154 | |||
155 | /* | ||
156 | * Ensure the rptr write update is not visible until | ||
157 | * memcpy has finished reading. | ||
158 | */ | ||
159 | smp_mb(); | ||
160 | atomic_set(&kfd->interrupt_ring_rptr, rptr); | ||
161 | |||
162 | return true; | ||
163 | } | ||
164 | |||
165 | static void interrupt_wq(struct work_struct *work) | ||
166 | { | ||
167 | struct kfd_dev *dev = container_of(work, struct kfd_dev, | ||
168 | interrupt_work); | ||
169 | |||
170 | uint32_t ih_ring_entry[DIV_ROUND_UP( | ||
171 | dev->device_info->ih_ring_entry_size, | ||
172 | sizeof(uint32_t))]; | ||
173 | |||
174 | while (dequeue_ih_ring_entry(dev, ih_ring_entry)) | ||
175 | ; | ||
176 | } | ||
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index a5edb29507e3..b3dc13c83169 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h | |||
@@ -135,22 +135,10 @@ struct kfd_dev { | |||
135 | 135 | ||
136 | struct kgd2kfd_shared_resources shared_resources; | 136 | struct kgd2kfd_shared_resources shared_resources; |
137 | 137 | ||
138 | void *interrupt_ring; | ||
139 | size_t interrupt_ring_size; | ||
140 | atomic_t interrupt_ring_rptr; | ||
141 | atomic_t interrupt_ring_wptr; | ||
142 | struct work_struct interrupt_work; | ||
143 | spinlock_t interrupt_lock; | ||
144 | |||
145 | /* QCM Device instance */ | 138 | /* QCM Device instance */ |
146 | struct device_queue_manager *dqm; | 139 | struct device_queue_manager *dqm; |
147 | 140 | ||
148 | bool init_complete; | 141 | bool init_complete; |
149 | /* | ||
150 | * Interrupts of interest to KFD are copied | ||
151 | * from the HW ring into a SW ring. | ||
152 | */ | ||
153 | bool interrupts_active; | ||
154 | }; | 142 | }; |
155 | 143 | ||
156 | /* KGD2KFD callbacks */ | 144 | /* KGD2KFD callbacks */ |
@@ -531,10 +519,7 @@ struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev); | |||
531 | struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx); | 519 | struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx); |
532 | 520 | ||
533 | /* Interrupts */ | 521 | /* Interrupts */ |
534 | int kfd_interrupt_init(struct kfd_dev *dev); | ||
535 | void kfd_interrupt_exit(struct kfd_dev *dev); | ||
536 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); | 522 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); |
537 | bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry); | ||
538 | 523 | ||
539 | /* Power Management */ | 524 | /* Power Management */ |
540 | void kgd2kfd_suspend(struct kfd_dev *kfd); | 525 | void kgd2kfd_suspend(struct kfd_dev *kfd); |