diff options
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/Makefile | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device.c | 20 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c | 176 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 15 |
5 files changed, 3 insertions, 213 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile index be6246de5091..307a309110e6 100644 --- a/drivers/gpu/drm/amd/amdkfd/Makefile +++ b/drivers/gpu/drm/amd/amdkfd/Makefile | |||
| @@ -8,7 +8,6 @@ amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \ | |||
| 8 | kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \ | 8 | kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \ |
| 9 | kfd_process.o kfd_queue.o kfd_mqd_manager.o \ | 9 | kfd_process.o kfd_queue.o kfd_mqd_manager.o \ |
| 10 | kfd_kernel_queue.o kfd_packet_manager.o \ | 10 | kfd_kernel_queue.o kfd_packet_manager.o \ |
| 11 | kfd_process_queue_manager.o kfd_device_queue_manager.o \ | 11 | kfd_process_queue_manager.o kfd_device_queue_manager.o |
| 12 | kfd_interrupt.o | ||
| 13 | 12 | ||
| 14 | obj-$(CONFIG_HSA_AMD) += amdkfd.o | 13 | obj-$(CONFIG_HSA_AMD) += amdkfd.o |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 43884ebd4303..633532a2e7ec 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c | |||
| @@ -192,13 +192,6 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, | |||
| 192 | goto kfd_topology_add_device_error; | 192 | goto kfd_topology_add_device_error; |
| 193 | } | 193 | } |
| 194 | 194 | ||
| 195 | if (kfd_interrupt_init(kfd)) { | ||
| 196 | dev_err(kfd_device, | ||
| 197 | "Error initializing interrupts for device (%x:%x)\n", | ||
| 198 | kfd->pdev->vendor, kfd->pdev->device); | ||
| 199 | goto kfd_interrupt_error; | ||
| 200 | } | ||
| 201 | |||
| 202 | if (!device_iommu_pasid_init(kfd)) { | 195 | if (!device_iommu_pasid_init(kfd)) { |
| 203 | dev_err(kfd_device, | 196 | dev_err(kfd_device, |
| 204 | "Error initializing iommuv2 for device (%x:%x)\n", | 197 | "Error initializing iommuv2 for device (%x:%x)\n", |
| @@ -237,8 +230,6 @@ dqm_start_error: | |||
| 237 | device_queue_manager_error: | 230 | device_queue_manager_error: |
| 238 | amd_iommu_free_device(kfd->pdev); | 231 | amd_iommu_free_device(kfd->pdev); |
| 239 | device_iommu_pasid_error: | 232 | device_iommu_pasid_error: |
| 240 | kfd_interrupt_exit(kfd); | ||
| 241 | kfd_interrupt_error: | ||
| 242 | kfd_topology_remove_device(kfd); | 233 | kfd_topology_remove_device(kfd); |
| 243 | kfd_topology_add_device_error: | 234 | kfd_topology_add_device_error: |
| 244 | kfd2kgd->fini_sa_manager(kfd->kgd); | 235 | kfd2kgd->fini_sa_manager(kfd->kgd); |
| @@ -254,7 +245,6 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd) | |||
| 254 | if (kfd->init_complete) { | 245 | if (kfd->init_complete) { |
| 255 | device_queue_manager_uninit(kfd->dqm); | 246 | device_queue_manager_uninit(kfd->dqm); |
| 256 | amd_iommu_free_device(kfd->pdev); | 247 | amd_iommu_free_device(kfd->pdev); |
| 257 | kfd_interrupt_exit(kfd); | ||
| 258 | kfd_topology_remove_device(kfd); | 248 | kfd_topology_remove_device(kfd); |
| 259 | } | 249 | } |
| 260 | 250 | ||
| @@ -296,13 +286,5 @@ int kgd2kfd_resume(struct kfd_dev *kfd) | |||
| 296 | /* This is called directly from KGD at ISR. */ | 286 | /* This is called directly from KGD at ISR. */ |
| 297 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) | 287 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) |
| 298 | { | 288 | { |
| 299 | if (kfd->init_complete) { | 289 | /* Process interrupts / schedule work as necessary */ |
| 300 | spin_lock(&kfd->interrupt_lock); | ||
| 301 | |||
| 302 | if (kfd->interrupts_active | ||
| 303 | && enqueue_ih_ring_entry(kfd, ih_ring_entry)) | ||
| 304 | schedule_work(&kfd->interrupt_work); | ||
| 305 | |||
| 306 | spin_unlock(&kfd->interrupt_lock); | ||
| 307 | } | ||
| 308 | } | 290 | } |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 9c8961d22360..30c8fda9622e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | |||
| @@ -280,7 +280,7 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, | |||
| 280 | q->queue); | 280 | q->queue); |
| 281 | 281 | ||
| 282 | retval = mqd->load_mqd(mqd, q->mqd, q->pipe, | 282 | retval = mqd->load_mqd(mqd, q->mqd, q->pipe, |
| 283 | q->queue, q->properties.write_ptr); | 283 | q->queue, (uint32_t __user *) q->properties.write_ptr); |
| 284 | if (retval != 0) { | 284 | if (retval != 0) { |
| 285 | deallocate_hqd(dqm, q); | 285 | deallocate_hqd(dqm, q); |
| 286 | mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); | 286 | mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj); |
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c b/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c deleted file mode 100644 index 5b999095a1f7..000000000000 --- a/drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c +++ /dev/null | |||
| @@ -1,176 +0,0 @@ | |||
| 1 | /* | ||
| 2 | * Copyright 2014 Advanced Micro Devices, Inc. | ||
| 3 | * | ||
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
| 5 | * copy of this software and associated documentation files (the "Software"), | ||
| 6 | * to deal in the Software without restriction, including without limitation | ||
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
| 8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
| 9 | * Software is furnished to do so, subject to the following conditions: | ||
| 10 | * | ||
| 11 | * The above copyright notice and this permission notice shall be included in | ||
| 12 | * all copies or substantial portions of the Software. | ||
| 13 | * | ||
| 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
| 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
| 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
| 17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | ||
| 18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | ||
| 19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | ||
| 20 | * OTHER DEALINGS IN THE SOFTWARE. | ||
| 21 | */ | ||
| 22 | |||
| 23 | /* | ||
| 24 | * KFD Interrupts. | ||
| 25 | * | ||
| 26 | * AMD GPUs deliver interrupts by pushing an interrupt description onto the | ||
| 27 | * interrupt ring and then sending an interrupt. KGD receives the interrupt | ||
| 28 | * in ISR and sends us a pointer to each new entry on the interrupt ring. | ||
| 29 | * | ||
| 30 | * We generally can't process interrupt-signaled events from ISR, so we call | ||
| 31 | * out to each interrupt client module (currently only the scheduler) to ask if | ||
| 32 | * each interrupt is interesting. If they return true, then it requires further | ||
| 33 | * processing so we copy it to an internal interrupt ring and call each | ||
| 34 | * interrupt client again from a work-queue. | ||
| 35 | * | ||
| 36 | * There's no acknowledgment for the interrupts we use. The hardware simply | ||
| 37 | * queues a new interrupt each time without waiting. | ||
| 38 | * | ||
| 39 | * The fixed-size internal queue means that it's possible for us to lose | ||
| 40 | * interrupts because we have no back-pressure to the hardware. | ||
| 41 | */ | ||
| 42 | |||
| 43 | #include <linux/slab.h> | ||
| 44 | #include <linux/device.h> | ||
| 45 | #include "kfd_priv.h" | ||
| 46 | |||
| 47 | #define KFD_INTERRUPT_RING_SIZE 256 | ||
| 48 | |||
| 49 | static void interrupt_wq(struct work_struct *); | ||
| 50 | |||
| 51 | int kfd_interrupt_init(struct kfd_dev *kfd) | ||
| 52 | { | ||
| 53 | void *interrupt_ring = kmalloc_array(KFD_INTERRUPT_RING_SIZE, | ||
| 54 | kfd->device_info->ih_ring_entry_size, | ||
| 55 | GFP_KERNEL); | ||
| 56 | if (!interrupt_ring) | ||
| 57 | return -ENOMEM; | ||
| 58 | |||
| 59 | kfd->interrupt_ring = interrupt_ring; | ||
| 60 | kfd->interrupt_ring_size = | ||
| 61 | KFD_INTERRUPT_RING_SIZE * kfd->device_info->ih_ring_entry_size; | ||
| 62 | atomic_set(&kfd->interrupt_ring_wptr, 0); | ||
| 63 | atomic_set(&kfd->interrupt_ring_rptr, 0); | ||
| 64 | |||
| 65 | spin_lock_init(&kfd->interrupt_lock); | ||
| 66 | |||
| 67 | INIT_WORK(&kfd->interrupt_work, interrupt_wq); | ||
| 68 | |||
| 69 | kfd->interrupts_active = true; | ||
| 70 | |||
| 71 | /* | ||
| 72 | * After this function returns, the interrupt will be enabled. This | ||
| 73 | * barrier ensures that the interrupt running on a different processor | ||
| 74 | * sees all the above writes. | ||
| 75 | */ | ||
| 76 | smp_wmb(); | ||
| 77 | |||
| 78 | return 0; | ||
| 79 | } | ||
| 80 | |||
| 81 | void kfd_interrupt_exit(struct kfd_dev *kfd) | ||
| 82 | { | ||
| 83 | /* | ||
| 84 | * Stop the interrupt handler from writing to the ring and scheduling | ||
| 85 | * workqueue items. The spinlock ensures that any interrupt running | ||
| 86 | * after we have unlocked sees interrupts_active = false. | ||
| 87 | */ | ||
| 88 | unsigned long flags; | ||
| 89 | |||
| 90 | spin_lock_irqsave(&kfd->interrupt_lock, flags); | ||
| 91 | kfd->interrupts_active = false; | ||
| 92 | spin_unlock_irqrestore(&kfd->interrupt_lock, flags); | ||
| 93 | |||
| 94 | /* | ||
| 95 | * Flush_scheduled_work ensures that there are no outstanding | ||
| 96 | * work-queue items that will access interrupt_ring. New work items | ||
| 97 | * can't be created because we stopped interrupt handling above. | ||
| 98 | */ | ||
| 99 | flush_scheduled_work(); | ||
| 100 | |||
| 101 | kfree(kfd->interrupt_ring); | ||
| 102 | } | ||
| 103 | |||
| 104 | /* | ||
| 105 | * This assumes that it can't be called concurrently with itself | ||
| 106 | * but only with dequeue_ih_ring_entry. | ||
| 107 | */ | ||
| 108 | bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry) | ||
| 109 | { | ||
| 110 | unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr); | ||
| 111 | unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr); | ||
| 112 | |||
| 113 | if ((rptr - wptr) % kfd->interrupt_ring_size == | ||
| 114 | kfd->device_info->ih_ring_entry_size) { | ||
| 115 | /* This is very bad, the system is likely to hang. */ | ||
| 116 | dev_err_ratelimited(kfd_chardev(), | ||
| 117 | "Interrupt ring overflow, dropping interrupt.\n"); | ||
| 118 | return false; | ||
| 119 | } | ||
| 120 | |||
| 121 | memcpy(kfd->interrupt_ring + wptr, ih_ring_entry, | ||
| 122 | kfd->device_info->ih_ring_entry_size); | ||
| 123 | |||
| 124 | wptr = (wptr + kfd->device_info->ih_ring_entry_size) % | ||
| 125 | kfd->interrupt_ring_size; | ||
| 126 | smp_wmb(); /* Ensure memcpy'd data is visible before wptr update. */ | ||
| 127 | atomic_set(&kfd->interrupt_ring_wptr, wptr); | ||
| 128 | |||
| 129 | return true; | ||
| 130 | } | ||
| 131 | |||
| 132 | /* | ||
| 133 | * This assumes that it can't be called concurrently with itself | ||
| 134 | * but only with enqueue_ih_ring_entry. | ||
| 135 | */ | ||
| 136 | static bool dequeue_ih_ring_entry(struct kfd_dev *kfd, void *ih_ring_entry) | ||
| 137 | { | ||
| 138 | /* | ||
| 139 | * Assume that wait queues have an implicit barrier, i.e. anything that | ||
| 140 | * happened in the ISR before it queued work is visible. | ||
| 141 | */ | ||
| 142 | |||
| 143 | unsigned int wptr = atomic_read(&kfd->interrupt_ring_wptr); | ||
| 144 | unsigned int rptr = atomic_read(&kfd->interrupt_ring_rptr); | ||
| 145 | |||
| 146 | if (rptr == wptr) | ||
| 147 | return false; | ||
| 148 | |||
| 149 | memcpy(ih_ring_entry, kfd->interrupt_ring + rptr, | ||
| 150 | kfd->device_info->ih_ring_entry_size); | ||
| 151 | |||
| 152 | rptr = (rptr + kfd->device_info->ih_ring_entry_size) % | ||
| 153 | kfd->interrupt_ring_size; | ||
| 154 | |||
| 155 | /* | ||
| 156 | * Ensure the rptr write update is not visible until | ||
| 157 | * memcpy has finished reading. | ||
| 158 | */ | ||
| 159 | smp_mb(); | ||
| 160 | atomic_set(&kfd->interrupt_ring_rptr, rptr); | ||
| 161 | |||
| 162 | return true; | ||
| 163 | } | ||
| 164 | |||
| 165 | static void interrupt_wq(struct work_struct *work) | ||
| 166 | { | ||
| 167 | struct kfd_dev *dev = container_of(work, struct kfd_dev, | ||
| 168 | interrupt_work); | ||
| 169 | |||
| 170 | uint32_t ih_ring_entry[DIV_ROUND_UP( | ||
| 171 | dev->device_info->ih_ring_entry_size, | ||
| 172 | sizeof(uint32_t))]; | ||
| 173 | |||
| 174 | while (dequeue_ih_ring_entry(dev, ih_ring_entry)) | ||
| 175 | ; | ||
| 176 | } | ||
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index a5edb29507e3..b3dc13c83169 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h | |||
| @@ -135,22 +135,10 @@ struct kfd_dev { | |||
| 135 | 135 | ||
| 136 | struct kgd2kfd_shared_resources shared_resources; | 136 | struct kgd2kfd_shared_resources shared_resources; |
| 137 | 137 | ||
| 138 | void *interrupt_ring; | ||
| 139 | size_t interrupt_ring_size; | ||
| 140 | atomic_t interrupt_ring_rptr; | ||
| 141 | atomic_t interrupt_ring_wptr; | ||
| 142 | struct work_struct interrupt_work; | ||
| 143 | spinlock_t interrupt_lock; | ||
| 144 | |||
| 145 | /* QCM Device instance */ | 138 | /* QCM Device instance */ |
| 146 | struct device_queue_manager *dqm; | 139 | struct device_queue_manager *dqm; |
| 147 | 140 | ||
| 148 | bool init_complete; | 141 | bool init_complete; |
| 149 | /* | ||
| 150 | * Interrupts of interest to KFD are copied | ||
| 151 | * from the HW ring into a SW ring. | ||
| 152 | */ | ||
| 153 | bool interrupts_active; | ||
| 154 | }; | 142 | }; |
| 155 | 143 | ||
| 156 | /* KGD2KFD callbacks */ | 144 | /* KGD2KFD callbacks */ |
| @@ -531,10 +519,7 @@ struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev); | |||
| 531 | struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx); | 519 | struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx); |
| 532 | 520 | ||
| 533 | /* Interrupts */ | 521 | /* Interrupts */ |
| 534 | int kfd_interrupt_init(struct kfd_dev *dev); | ||
| 535 | void kfd_interrupt_exit(struct kfd_dev *dev); | ||
| 536 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); | 522 | void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); |
| 537 | bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry); | ||
| 538 | 523 | ||
| 539 | /* Power Management */ | 524 | /* Power Management */ |
| 540 | void kgd2kfd_suspend(struct kfd_dev *kfd); | 525 | void kgd2kfd_suspend(struct kfd_dev *kfd); |
