diff options
Diffstat (limited to 'drivers/gpu/nvgpu')
-rw-r--r-- | drivers/gpu/nvgpu/Kconfig | 7 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/Makefile | 3 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/linux/nvidia_p2p.h | 196 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/os/linux/nvidia_p2p.c | 296 |
4 files changed, 502 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/Kconfig b/drivers/gpu/nvgpu/Kconfig index bbd5f6f0..7dba61a3 100644 --- a/drivers/gpu/nvgpu/Kconfig +++ b/drivers/gpu/nvgpu/Kconfig | |||
@@ -40,6 +40,13 @@ config GK20A_PM_QOS | |||
40 | Enable support to pass PM_QOS constraints to devfreq based | 40 | Enable support to pass PM_QOS constraints to devfreq based |
41 | scaling. | 41 | scaling. |
42 | 42 | ||
43 | config GK20A_RDMA | ||
44 | bool "Support GK20A RDMA" | ||
45 | depends on GK20A && MMU_NOTIFIER | ||
46 | default n | ||
47 | help | ||
48 | Say Y here to enable GK20A RDMA features. | ||
49 | |||
43 | config NVGPU_TRACK_MEM_USAGE | 50 | config NVGPU_TRACK_MEM_USAGE |
44 | bool "Track the usage of system memory in nvgpu" | 51 | bool "Track the usage of system memory in nvgpu" |
45 | depends on GK20A | 52 | depends on GK20A |
diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index e0fd70e6..fdfaf092 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile | |||
@@ -176,6 +176,9 @@ nvgpu-$(CONFIG_NVGPU_SUPPORT_CDE) += \ | |||
176 | os/linux/cde_gm20b.o \ | 176 | os/linux/cde_gm20b.o \ |
177 | os/linux/cde_gp10b.o | 177 | os/linux/cde_gp10b.o |
178 | 178 | ||
179 | nvgpu-$(CONFIG_GK20A_RDMA) += \ | ||
180 | os/linux/nvidia_p2p.o | ||
181 | |||
179 | ifeq ($(CONFIG_DEBUG_FS),y) | 182 | ifeq ($(CONFIG_DEBUG_FS),y) |
180 | nvgpu-$(CONFIG_NVGPU_SUPPORT_CDE) += \ | 183 | nvgpu-$(CONFIG_NVGPU_SUPPORT_CDE) += \ |
181 | os/linux/debug_cde.o | 184 | os/linux/debug_cde.o |
diff --git a/drivers/gpu/nvgpu/include/nvgpu/linux/nvidia_p2p.h b/drivers/gpu/nvgpu/include/nvgpu/linux/nvidia_p2p.h new file mode 100644 index 00000000..c1dee7cf --- /dev/null +++ b/drivers/gpu/nvgpu/include/nvgpu/linux/nvidia_p2p.h | |||
@@ -0,0 +1,196 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #ifndef __NVIDIA_P2P_H__ | ||
24 | #define __NVIDIA_P2P_H__ | ||
25 | |||
26 | #include <linux/dma-mapping.h> | ||
27 | #include <linux/module.h> | ||
28 | #include <linux/kernel.h> | ||
29 | #include <linux/init.h> | ||
30 | #include <linux/miscdevice.h> | ||
31 | #include <linux/fs.h> | ||
32 | #include <linux/mm.h> | ||
33 | #include <linux/mman.h> | ||
34 | #include <linux/mmu_notifier.h> | ||
35 | #include <linux/types.h> | ||
36 | #include <linux/uaccess.h> | ||
37 | #include <linux/sched.h> | ||
38 | #include <linux/spinlock.h> | ||
39 | #include <linux/slab.h> | ||
40 | #include <linux/highmem.h> | ||
41 | #include <linux/mutex.h> | ||
42 | #include <linux/spinlock.h> | ||
43 | #include <nvgpu/linux/lock.h> | ||
44 | |||
45 | #define NVIDIA_P2P_UNINITIALIZED 0x0 | ||
46 | #define NVIDIA_P2P_PINNED 0x1 | ||
47 | #define NVIDIA_P2P_MAPPED 0x2 | ||
48 | |||
49 | enum nvidia_p2p_page_size_type { | ||
50 | NVIDIA_P2P_PAGE_SIZE_4KB = 0, | ||
51 | NVIDIA_P2P_PAGE_SIZE_64KB, | ||
52 | NVIDIA_P2P_PAGE_SIZE_128KB, | ||
53 | NVIDIA_P2P_PAGE_SIZE_COUNT | ||
54 | }; | ||
55 | |||
56 | struct nvidia_p2p_page_table { | ||
57 | u32 page_size; | ||
58 | u64 size; | ||
59 | u32 entries; | ||
60 | struct page **pages; | ||
61 | |||
62 | u64 vaddr; | ||
63 | u32 mapped; | ||
64 | |||
65 | struct mm_struct *mm; | ||
66 | struct mmu_notifier mn; | ||
67 | struct nvgpu_mutex lock; | ||
68 | void (*free_callback)(void *data); | ||
69 | void *data; | ||
70 | }; | ||
71 | |||
72 | struct nvidia_p2p_dma_mapping { | ||
73 | dma_addr_t *hw_address; | ||
74 | u32 *hw_len; | ||
75 | u32 entries; | ||
76 | |||
77 | struct sg_table *sgt; | ||
78 | struct device *dev; | ||
79 | struct nvidia_p2p_page_table *page_table; | ||
80 | enum dma_data_direction direction; | ||
81 | }; | ||
82 | |||
83 | /* | ||
84 | * @brief | ||
85 | * Make the pages underlying a range of GPU virtual memory | ||
86 | * accessible to a third-party device. | ||
87 | * | ||
88 | * @param[in] vaddr | ||
89 | * A GPU Virtual Address | ||
90 | * @param[in] size | ||
91 | * The size of the requested mapping. | ||
92 | * Size must be a multiple of Page size. | ||
93 | * @param[out] **page_table | ||
94 | * A pointer to struct nvidia_p2p_page_table | ||
95 | * @param[in] free_callback | ||
96 | * A non-NULL pointer to the function to be invoked when the pages | ||
97 | * underlying the virtual address range are freed | ||
98 | * implicitly. Must be non NULL. | ||
99 | * @param[in] data | ||
100 | * A non-NULL opaque pointer to private data to be passed to the | ||
101 | * callback function. | ||
102 | * | ||
103 | * @return | ||
104 | * 0 upon successful completion. | ||
105 | * Negative number if any error | ||
106 | */ | ||
107 | int nvidia_p2p_get_pages(u64 vaddr, u64 size, | ||
108 | struct nvidia_p2p_page_table **page_table, | ||
109 | void (*free_callback)(void *data), void *data); | ||
110 | /* | ||
111 | * @brief | ||
112 | * Release the pages previously made accessible to | ||
113 | * a third-party device. | ||
114 | * | ||
115 | * @param[in] *page_table | ||
116 | * A pointer to struct nvidia_p2p_page_table | ||
117 | * | ||
118 | * @return | ||
119 | * 0 upon successful completion. | ||
120 | * -ENOMEM if the driver failed to allocate memory or if | ||
121 | * insufficient resources were available to complete the operation. | ||
122 | * Negative number if any other error | ||
123 | */ | ||
124 | int nvidia_p2p_put_pages(struct nvidia_p2p_page_table *page_table); | ||
125 | |||
126 | /* | ||
127 | * @brief | ||
128 | * Release the pages previously made accessible to | ||
129 | * a third-party device. This is called during the | ||
130 | * execution of the free_callback(). | ||
131 | * | ||
132 | * @param[in] *page_table | ||
133 | * A pointer to struct nvidia_p2p_page_table | ||
134 | * | ||
135 | * @return | ||
136 | * 0 upon successful completion. | ||
137 | * -ENOMEM if the driver failed to allocate memory or if | ||
138 | * insufficient resources were available to complete the operation. | ||
139 | * Negative number if any other error | ||
140 | */ | ||
141 | int nvidia_p2p_free_page_table(struct nvidia_p2p_page_table *page_table); | ||
142 | |||
143 | /* | ||
144 | * @brief | ||
145 | * Map the pages retrieved using nvidia_p2p_get_pages and | ||
146 | * pass the dma address to a third-party device. | ||
147 | * | ||
148 | * @param[in] *dev | ||
149 | * The peer device that needs to DMA to/from the | ||
150 | * mapping. | ||
151 | * @param[in] *page_table | ||
152 | * A pointer to struct nvidia_p2p_page_table | ||
153 | * @param[out] **map | ||
154 | * A pointer to struct nvidia_p2p_dma_mapping. | ||
155 | * The DMA mapping containing the DMA addresses to use. | ||
156 | * @param[in] direction | ||
157 | * DMA direction | ||
158 | * | ||
159 | * @return | ||
160 | * 0 upon successful completion. | ||
161 | * Negative number if any other error | ||
162 | */ | ||
163 | int nvidia_p2p_map_pages(struct device *dev, | ||
164 | struct nvidia_p2p_page_table *page_table, | ||
165 | struct nvidia_p2p_dma_mapping **map, | ||
166 | enum dma_data_direction direction); | ||
167 | /* | ||
168 | * @brief | ||
169 | * Unmap the pages previously mapped using nvidia_p2p_map_pages | ||
170 | * | ||
171 | * @param[in] *map | ||
172 | * A pointer to struct nvidia_p2p_dma_mapping. | ||
173 | * The DMA mapping containing the DMA addresses to use. | ||
174 | * | ||
175 | * @return | ||
176 | * 0 upon successful completion. | ||
177 | * Negative number if any other error | ||
178 | */ | ||
179 | int nvidia_p2p_unmap_pages(struct nvidia_p2p_dma_mapping *map); | ||
180 | |||
181 | /* | ||
182 | * @brief | ||
183 | * Unmap the pages previously mapped using nvidia_p2p_map_pages. | ||
184 | * This is called during the execution of the free_callback(). | ||
185 | * | ||
186 | * @param[in] *map | ||
187 | * A pointer to struct nvidia_p2p_dma_mapping. | ||
188 | * The DMA mapping containing the DMA addresses to use. | ||
189 | * | ||
190 | * @return | ||
191 | * 0 upon successful completion. | ||
192 | * Negative number if any other error | ||
193 | */ | ||
194 | int nvidia_p2p_free_dma_mapping(struct nvidia_p2p_dma_mapping *dma_mapping); | ||
195 | |||
196 | #endif | ||
diff --git a/drivers/gpu/nvgpu/os/linux/nvidia_p2p.c b/drivers/gpu/nvgpu/os/linux/nvidia_p2p.c new file mode 100644 index 00000000..dcd37bee --- /dev/null +++ b/drivers/gpu/nvgpu/os/linux/nvidia_p2p.c | |||
@@ -0,0 +1,296 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #include <linux/slab.h> | ||
24 | #include <nvgpu/linux/nvidia_p2p.h> | ||
25 | |||
26 | static void nvidia_p2p_mn_release(struct mmu_notifier *mn, | ||
27 | struct mm_struct *mm) | ||
28 | { | ||
29 | struct nvidia_p2p_page_table *page_table = container_of(mn, | ||
30 | struct nvidia_p2p_page_table, | ||
31 | mn); | ||
32 | |||
33 | page_table->free_callback(page_table->data); | ||
34 | } | ||
35 | |||
36 | static void nvidia_p2p_mn_invl_range_start(struct mmu_notifier *mn, | ||
37 | struct mm_struct *mm, unsigned long start, unsigned long end) | ||
38 | { | ||
39 | struct nvidia_p2p_page_table *page_table = container_of(mn, | ||
40 | struct nvidia_p2p_page_table, | ||
41 | mn); | ||
42 | u64 vaddr = 0; | ||
43 | u64 size = 0; | ||
44 | |||
45 | vaddr = page_table->vaddr; | ||
46 | size = page_table->size; | ||
47 | |||
48 | if (vaddr >= start && vaddr <= end) { | ||
49 | mmu_notifier_unregister_no_release(&page_table->mn, page_table->mm); | ||
50 | page_table->free_callback(page_table->data); | ||
51 | } | ||
52 | } | ||
53 | |||
54 | static struct mmu_notifier_ops nvidia_p2p_mmu_ops = { | ||
55 | .release = nvidia_p2p_mn_release, | ||
56 | .invalidate_range_start = nvidia_p2p_mn_invl_range_start, | ||
57 | }; | ||
58 | |||
59 | int nvidia_p2p_get_pages(u64 vaddr, u64 size, | ||
60 | struct nvidia_p2p_page_table **page_table, | ||
61 | void (*free_callback)(void *data), void *data) | ||
62 | { | ||
63 | int ret = 0; | ||
64 | int user_pages = 0; | ||
65 | int locked = 0; | ||
66 | int nr_pages = size >> PAGE_SHIFT; | ||
67 | struct page **pages; | ||
68 | |||
69 | if (nr_pages <= 0) { | ||
70 | return -EINVAL; | ||
71 | } | ||
72 | |||
73 | *page_table = kzalloc(sizeof(**page_table), GFP_KERNEL); | ||
74 | if (!*page_table) { | ||
75 | return -ENOMEM; | ||
76 | } | ||
77 | |||
78 | pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); | ||
79 | if (!pages) { | ||
80 | ret = -ENOMEM; | ||
81 | goto free_page_table; | ||
82 | } | ||
83 | down_read(¤t->mm->mmap_sem); | ||
84 | locked = 1; | ||
85 | user_pages = get_user_pages_locked(vaddr & PAGE_MASK, nr_pages, | ||
86 | FOLL_WRITE | FOLL_FORCE, | ||
87 | pages, &locked); | ||
88 | up_read(¤t->mm->mmap_sem); | ||
89 | if (user_pages != nr_pages) { | ||
90 | ret = user_pages < 0 ? user_pages : -ENOMEM; | ||
91 | goto free_pages; | ||
92 | } | ||
93 | |||
94 | (*page_table)->pages = pages; | ||
95 | (*page_table)->entries = user_pages; | ||
96 | (*page_table)->page_size = NVIDIA_P2P_PAGE_SIZE_4KB; | ||
97 | (*page_table)->size = size; | ||
98 | |||
99 | (*page_table)->mn.ops = &nvidia_p2p_mmu_ops; | ||
100 | (*page_table)->mm = current->mm; | ||
101 | (*page_table)->free_callback = free_callback; | ||
102 | (*page_table)->data = data; | ||
103 | (*page_table)->vaddr = vaddr; | ||
104 | nvgpu_mutex_init(&(*page_table)->lock); | ||
105 | (*page_table)->mapped = NVIDIA_P2P_PINNED; | ||
106 | |||
107 | ret = mmu_notifier_register(&(*page_table)->mn, (*page_table)->mm); | ||
108 | if (ret) { | ||
109 | goto free_pages; | ||
110 | } | ||
111 | |||
112 | return 0; | ||
113 | free_pages: | ||
114 | while (--user_pages >= 0) { | ||
115 | put_page(pages[user_pages]); | ||
116 | } | ||
117 | kfree(pages); | ||
118 | free_page_table: | ||
119 | kfree(*page_table); | ||
120 | *page_table = NULL; | ||
121 | return ret; | ||
122 | } | ||
123 | EXPORT_SYMBOL(nvidia_p2p_get_pages); | ||
124 | |||
125 | int nvidia_p2p_put_pages(struct nvidia_p2p_page_table *page_table) | ||
126 | { | ||
127 | if (!page_table) { | ||
128 | return -EINVAL; | ||
129 | } | ||
130 | |||
131 | mmu_notifier_unregister(&page_table->mn, page_table->mm); | ||
132 | |||
133 | return 0; | ||
134 | } | ||
135 | EXPORT_SYMBOL(nvidia_p2p_put_pages); | ||
136 | |||
137 | int nvidia_p2p_free_page_table(struct nvidia_p2p_page_table *page_table) | ||
138 | { | ||
139 | int user_pages = 0; | ||
140 | struct page **pages = NULL; | ||
141 | |||
142 | if (!page_table) { | ||
143 | return 0; | ||
144 | } | ||
145 | |||
146 | nvgpu_mutex_acquire(&page_table->lock); | ||
147 | |||
148 | if (page_table->mapped & NVIDIA_P2P_MAPPED) { | ||
149 | WARN(1, "Attempting to free unmapped pages"); | ||
150 | } | ||
151 | |||
152 | if (page_table->mapped & NVIDIA_P2P_PINNED) { | ||
153 | pages = page_table->pages; | ||
154 | user_pages = page_table->entries; | ||
155 | |||
156 | while (--user_pages >= 0) { | ||
157 | put_page(pages[user_pages]); | ||
158 | } | ||
159 | |||
160 | kfree(pages); | ||
161 | page_table->mapped &= (u32)~NVIDIA_P2P_PINNED; | ||
162 | } | ||
163 | |||
164 | nvgpu_mutex_release(&page_table->lock); | ||
165 | |||
166 | return 0; | ||
167 | } | ||
168 | EXPORT_SYMBOL(nvidia_p2p_free_page_table); | ||
169 | |||
170 | int nvidia_p2p_map_pages(struct device *dev, | ||
171 | struct nvidia_p2p_page_table *page_table, | ||
172 | struct nvidia_p2p_dma_mapping **dma_mapping, | ||
173 | enum dma_data_direction direction) | ||
174 | { | ||
175 | struct sg_table *sgt = NULL; | ||
176 | struct scatterlist *sg; | ||
177 | struct page **pages = NULL; | ||
178 | u32 nr_pages = 0; | ||
179 | int ret = 0; | ||
180 | int i, count; | ||
181 | |||
182 | if (!page_table) { | ||
183 | return -EINVAL; | ||
184 | } | ||
185 | |||
186 | nvgpu_mutex_acquire(&page_table->lock); | ||
187 | |||
188 | pages = page_table->pages; | ||
189 | nr_pages = page_table->entries; | ||
190 | if (nr_pages <= 0) { | ||
191 | nvgpu_mutex_release(&page_table->lock); | ||
192 | return -EINVAL; | ||
193 | } | ||
194 | |||
195 | *dma_mapping = kzalloc(sizeof(**dma_mapping), GFP_KERNEL); | ||
196 | if (!*dma_mapping) { | ||
197 | nvgpu_mutex_release(&page_table->lock); | ||
198 | return -ENOMEM; | ||
199 | } | ||
200 | sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); | ||
201 | if (!sgt) { | ||
202 | ret = -ENOMEM; | ||
203 | goto free_dma_mapping; | ||
204 | } | ||
205 | ret = sg_alloc_table_from_pages(sgt, pages, | ||
206 | nr_pages, 0, page_table->size, GFP_KERNEL); | ||
207 | if (ret) { | ||
208 | goto free_sgt; | ||
209 | } | ||
210 | (*dma_mapping)->sgt = sgt; | ||
211 | (*dma_mapping)->dev = dev; | ||
212 | (*dma_mapping)->direction = direction; | ||
213 | (*dma_mapping)->page_table = page_table; | ||
214 | |||
215 | count = dma_map_sg(dev, sgt->sgl, sgt->nents, direction); | ||
216 | if (count < 1) { | ||
217 | goto free_sg_table; | ||
218 | } | ||
219 | |||
220 | (*dma_mapping)->entries = count; | ||
221 | |||
222 | (*dma_mapping)->hw_address = kcalloc(count, sizeof(u64), GFP_KERNEL); | ||
223 | if (!((*dma_mapping)->hw_address)) { | ||
224 | ret = -ENOMEM; | ||
225 | goto unmap_sg; | ||
226 | } | ||
227 | (*dma_mapping)->hw_len = kcalloc(count, sizeof(u64), GFP_KERNEL); | ||
228 | if (!((*dma_mapping)->hw_len)) { | ||
229 | ret = -ENOMEM; | ||
230 | goto free_hw_address; | ||
231 | } | ||
232 | |||
233 | for_each_sg(sgt->sgl, sg, count, i) { | ||
234 | (*dma_mapping)->hw_address[i] = sg_dma_address(sg); | ||
235 | (*dma_mapping)->hw_len[i] = sg_dma_len(sg); | ||
236 | } | ||
237 | (*dma_mapping)->page_table->mapped |= NVIDIA_P2P_MAPPED; | ||
238 | nvgpu_mutex_release(&page_table->lock); | ||
239 | |||
240 | return 0; | ||
241 | free_hw_address: | ||
242 | kfree((*dma_mapping)->hw_address); | ||
243 | unmap_sg: | ||
244 | dma_unmap_sg(dev, sgt->sgl, | ||
245 | sgt->nents, direction); | ||
246 | free_sg_table: | ||
247 | sg_free_table(sgt); | ||
248 | free_sgt: | ||
249 | kfree(sgt); | ||
250 | free_dma_mapping: | ||
251 | kfree(*dma_mapping); | ||
252 | *dma_mapping = NULL; | ||
253 | nvgpu_mutex_release(&page_table->lock); | ||
254 | |||
255 | return ret; | ||
256 | } | ||
257 | EXPORT_SYMBOL(nvidia_p2p_map_pages); | ||
258 | |||
259 | int nvidia_p2p_unmap_pages(struct nvidia_p2p_dma_mapping *dma_mapping) | ||
260 | { | ||
261 | struct nvidia_p2p_page_table *page_table = NULL; | ||
262 | |||
263 | if (!dma_mapping) { | ||
264 | return -EINVAL; | ||
265 | } | ||
266 | |||
267 | page_table = dma_mapping->page_table; | ||
268 | if (!page_table) { | ||
269 | return -EFAULT; | ||
270 | } | ||
271 | |||
272 | nvgpu_mutex_acquire(&page_table->lock); | ||
273 | if (page_table->mapped & NVIDIA_P2P_MAPPED) { | ||
274 | kfree(dma_mapping->hw_len); | ||
275 | kfree(dma_mapping->hw_address); | ||
276 | if (dma_mapping->entries) | ||
277 | dma_unmap_sg(dma_mapping->dev, | ||
278 | dma_mapping->sgt->sgl, | ||
279 | dma_mapping->sgt->nents, | ||
280 | dma_mapping->direction); | ||
281 | sg_free_table(dma_mapping->sgt); | ||
282 | kfree(dma_mapping->sgt); | ||
283 | kfree(dma_mapping); | ||
284 | page_table->mapped &= (u32)~NVIDIA_P2P_MAPPED; | ||
285 | } | ||
286 | nvgpu_mutex_release(&page_table->lock); | ||
287 | |||
288 | return 0; | ||
289 | } | ||
290 | EXPORT_SYMBOL(nvidia_p2p_unmap_pages); | ||
291 | |||
292 | int nvidia_p2p_free_dma_mapping(struct nvidia_p2p_dma_mapping *dma_mapping) | ||
293 | { | ||
294 | return nvidia_p2p_unmap_pages(dma_mapping); | ||
295 | } | ||
296 | EXPORT_SYMBOL(nvidia_p2p_free_dma_mapping); | ||