diff options
author | Dan Williams <dan.j.williams@intel.com> | 2013-12-30 15:37:29 -0500 |
---|---|---|
committer | Dan Williams <dan.j.williams@intel.com> | 2014-09-28 10:05:16 -0400 |
commit | 7bced397510ab569d31de4c70b39e13355046387 (patch) | |
tree | faa4067a53e42acffc752e9a153e7dbaed4126e5 /drivers/dma | |
parent | 08223d80df38e666a42d7c82eb340db55c6e03bd (diff) |
net_dma: simple removal
Per commit "77873803363c net_dma: mark broken" net_dma is no longer used
and there is no plan to fix it.
This is the mechanical removal of bits in CONFIG_NET_DMA ifdef guards.
Reverting the remainder of the net_dma induced changes is deferred to
subsequent patches.
Marked for stable due to Roman's report of a memory leak in
dma_pin_iovec_pages():
https://lkml.org/lkml/2014/9/3/177
Cc: Dave Jiang <dave.jiang@intel.com>
Cc: Vinod Koul <vinod.koul@intel.com>
Cc: David Whipple <whipple@securedatainnovations.ch>
Cc: Alexander Duyck <alexander.h.duyck@intel.com>
Cc: <stable@vger.kernel.org>
Reported-by: Roman Gushchin <klamm@yandex-team.ru>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Diffstat (limited to 'drivers/dma')
-rw-r--r-- | drivers/dma/Kconfig | 12 | ||||
-rw-r--r-- | drivers/dma/Makefile | 1 | ||||
-rw-r--r-- | drivers/dma/dmaengine.c | 104 | ||||
-rw-r--r-- | drivers/dma/ioat/dma.c | 1 | ||||
-rw-r--r-- | drivers/dma/ioat/dma.h | 7 | ||||
-rw-r--r-- | drivers/dma/ioat/dma_v2.c | 1 | ||||
-rw-r--r-- | drivers/dma/ioat/dma_v3.c | 1 | ||||
-rw-r--r-- | drivers/dma/iovlock.c | 280 |
8 files changed, 0 insertions, 407 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 605b016bcea4..6b5f37e01a70 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig | |||
@@ -368,18 +368,6 @@ config DMA_OF | |||
368 | comment "DMA Clients" | 368 | comment "DMA Clients" |
369 | depends on DMA_ENGINE | 369 | depends on DMA_ENGINE |
370 | 370 | ||
371 | config NET_DMA | ||
372 | bool "Network: TCP receive copy offload" | ||
373 | depends on DMA_ENGINE && NET | ||
374 | default (INTEL_IOATDMA || FSL_DMA) | ||
375 | depends on BROKEN | ||
376 | help | ||
377 | This enables the use of DMA engines in the network stack to | ||
378 | offload receive copy-to-user operations, freeing CPU cycles. | ||
379 | |||
380 | Say Y here if you enabled INTEL_IOATDMA or FSL_DMA, otherwise | ||
381 | say N. | ||
382 | |||
383 | config ASYNC_TX_DMA | 371 | config ASYNC_TX_DMA |
384 | bool "Async_tx: Offload support for the async_tx api" | 372 | bool "Async_tx: Offload support for the async_tx api" |
385 | depends on DMA_ENGINE | 373 | depends on DMA_ENGINE |
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index a029d0f4a1be..0c9dc7549327 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile | |||
@@ -6,7 +6,6 @@ obj-$(CONFIG_DMA_VIRTUAL_CHANNELS) += virt-dma.o | |||
6 | obj-$(CONFIG_DMA_ACPI) += acpi-dma.o | 6 | obj-$(CONFIG_DMA_ACPI) += acpi-dma.o |
7 | obj-$(CONFIG_DMA_OF) += of-dma.o | 7 | obj-$(CONFIG_DMA_OF) += of-dma.o |
8 | 8 | ||
9 | obj-$(CONFIG_NET_DMA) += iovlock.o | ||
10 | obj-$(CONFIG_INTEL_MID_DMAC) += intel_mid_dma.o | 9 | obj-$(CONFIG_INTEL_MID_DMAC) += intel_mid_dma.o |
11 | obj-$(CONFIG_DMATEST) += dmatest.o | 10 | obj-$(CONFIG_DMATEST) += dmatest.o |
12 | obj-$(CONFIG_INTEL_IOATDMA) += ioat/ | 11 | obj-$(CONFIG_INTEL_IOATDMA) += ioat/ |
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index ed610b497518..268de183b519 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c | |||
@@ -1084,110 +1084,6 @@ dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags) | |||
1084 | } | 1084 | } |
1085 | EXPORT_SYMBOL(dmaengine_get_unmap_data); | 1085 | EXPORT_SYMBOL(dmaengine_get_unmap_data); |
1086 | 1086 | ||
1087 | /** | ||
1088 | * dma_async_memcpy_pg_to_pg - offloaded copy from page to page | ||
1089 | * @chan: DMA channel to offload copy to | ||
1090 | * @dest_pg: destination page | ||
1091 | * @dest_off: offset in page to copy to | ||
1092 | * @src_pg: source page | ||
1093 | * @src_off: offset in page to copy from | ||
1094 | * @len: length | ||
1095 | * | ||
1096 | * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus | ||
1097 | * address according to the DMA mapping API rules for streaming mappings. | ||
1098 | * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident | ||
1099 | * (kernel memory or locked user space pages). | ||
1100 | */ | ||
1101 | dma_cookie_t | ||
1102 | dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg, | ||
1103 | unsigned int dest_off, struct page *src_pg, unsigned int src_off, | ||
1104 | size_t len) | ||
1105 | { | ||
1106 | struct dma_device *dev = chan->device; | ||
1107 | struct dma_async_tx_descriptor *tx; | ||
1108 | struct dmaengine_unmap_data *unmap; | ||
1109 | dma_cookie_t cookie; | ||
1110 | unsigned long flags; | ||
1111 | |||
1112 | unmap = dmaengine_get_unmap_data(dev->dev, 2, GFP_NOWAIT); | ||
1113 | if (!unmap) | ||
1114 | return -ENOMEM; | ||
1115 | |||
1116 | unmap->to_cnt = 1; | ||
1117 | unmap->from_cnt = 1; | ||
1118 | unmap->addr[0] = dma_map_page(dev->dev, src_pg, src_off, len, | ||
1119 | DMA_TO_DEVICE); | ||
1120 | unmap->addr[1] = dma_map_page(dev->dev, dest_pg, dest_off, len, | ||
1121 | DMA_FROM_DEVICE); | ||
1122 | unmap->len = len; | ||
1123 | flags = DMA_CTRL_ACK; | ||
1124 | tx = dev->device_prep_dma_memcpy(chan, unmap->addr[1], unmap->addr[0], | ||
1125 | len, flags); | ||
1126 | |||
1127 | if (!tx) { | ||
1128 | dmaengine_unmap_put(unmap); | ||
1129 | return -ENOMEM; | ||
1130 | } | ||
1131 | |||
1132 | dma_set_unmap(tx, unmap); | ||
1133 | cookie = tx->tx_submit(tx); | ||
1134 | dmaengine_unmap_put(unmap); | ||
1135 | |||
1136 | preempt_disable(); | ||
1137 | __this_cpu_add(chan->local->bytes_transferred, len); | ||
1138 | __this_cpu_inc(chan->local->memcpy_count); | ||
1139 | preempt_enable(); | ||
1140 | |||
1141 | return cookie; | ||
1142 | } | ||
1143 | EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg); | ||
1144 | |||
1145 | /** | ||
1146 | * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses | ||
1147 | * @chan: DMA channel to offload copy to | ||
1148 | * @dest: destination address (virtual) | ||
1149 | * @src: source address (virtual) | ||
1150 | * @len: length | ||
1151 | * | ||
1152 | * Both @dest and @src must be mappable to a bus address according to the | ||
1153 | * DMA mapping API rules for streaming mappings. | ||
1154 | * Both @dest and @src must stay memory resident (kernel memory or locked | ||
1155 | * user space pages). | ||
1156 | */ | ||
1157 | dma_cookie_t | ||
1158 | dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest, | ||
1159 | void *src, size_t len) | ||
1160 | { | ||
1161 | return dma_async_memcpy_pg_to_pg(chan, virt_to_page(dest), | ||
1162 | (unsigned long) dest & ~PAGE_MASK, | ||
1163 | virt_to_page(src), | ||
1164 | (unsigned long) src & ~PAGE_MASK, len); | ||
1165 | } | ||
1166 | EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf); | ||
1167 | |||
1168 | /** | ||
1169 | * dma_async_memcpy_buf_to_pg - offloaded copy from address to page | ||
1170 | * @chan: DMA channel to offload copy to | ||
1171 | * @page: destination page | ||
1172 | * @offset: offset in page to copy to | ||
1173 | * @kdata: source address (virtual) | ||
1174 | * @len: length | ||
1175 | * | ||
1176 | * Both @page/@offset and @kdata must be mappable to a bus address according | ||
1177 | * to the DMA mapping API rules for streaming mappings. | ||
1178 | * Both @page/@offset and @kdata must stay memory resident (kernel memory or | ||
1179 | * locked user space pages) | ||
1180 | */ | ||
1181 | dma_cookie_t | ||
1182 | dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page, | ||
1183 | unsigned int offset, void *kdata, size_t len) | ||
1184 | { | ||
1185 | return dma_async_memcpy_pg_to_pg(chan, page, offset, | ||
1186 | virt_to_page(kdata), | ||
1187 | (unsigned long) kdata & ~PAGE_MASK, len); | ||
1188 | } | ||
1189 | EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg); | ||
1190 | |||
1191 | void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, | 1087 | void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, |
1192 | struct dma_chan *chan) | 1088 | struct dma_chan *chan) |
1193 | { | 1089 | { |
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index b76c1485933b..940c1502a8b5 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c | |||
@@ -1222,7 +1222,6 @@ int ioat1_dma_probe(struct ioatdma_device *device, int dca) | |||
1222 | err = ioat_probe(device); | 1222 | err = ioat_probe(device); |
1223 | if (err) | 1223 | if (err) |
1224 | return err; | 1224 | return err; |
1225 | ioat_set_tcp_copy_break(4096); | ||
1226 | err = ioat_register(device); | 1225 | err = ioat_register(device); |
1227 | if (err) | 1226 | if (err) |
1228 | return err; | 1227 | return err; |
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index e982f00a9843..d63f68b1aa35 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h | |||
@@ -214,13 +214,6 @@ __dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw, | |||
214 | #define dump_desc_dbg(c, d) \ | 214 | #define dump_desc_dbg(c, d) \ |
215 | ({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; }) | 215 | ({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; }) |
216 | 216 | ||
217 | static inline void ioat_set_tcp_copy_break(unsigned long copybreak) | ||
218 | { | ||
219 | #ifdef CONFIG_NET_DMA | ||
220 | sysctl_tcp_dma_copybreak = copybreak; | ||
221 | #endif | ||
222 | } | ||
223 | |||
224 | static inline struct ioat_chan_common * | 217 | static inline struct ioat_chan_common * |
225 | ioat_chan_by_index(struct ioatdma_device *device, int index) | 218 | ioat_chan_by_index(struct ioatdma_device *device, int index) |
226 | { | 219 | { |
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c index 2ce9be498608..695483e6be32 100644 --- a/drivers/dma/ioat/dma_v2.c +++ b/drivers/dma/ioat/dma_v2.c | |||
@@ -900,7 +900,6 @@ int ioat2_dma_probe(struct ioatdma_device *device, int dca) | |||
900 | err = ioat_probe(device); | 900 | err = ioat_probe(device); |
901 | if (err) | 901 | if (err) |
902 | return err; | 902 | return err; |
903 | ioat_set_tcp_copy_break(2048); | ||
904 | 903 | ||
905 | list_for_each_entry(c, &dma->channels, device_node) { | 904 | list_for_each_entry(c, &dma->channels, device_node) { |
906 | chan = to_chan_common(c); | 905 | chan = to_chan_common(c); |
diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 85971d6e9646..895f869d6c2c 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c | |||
@@ -1655,7 +1655,6 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca) | |||
1655 | err = ioat_probe(device); | 1655 | err = ioat_probe(device); |
1656 | if (err) | 1656 | if (err) |
1657 | return err; | 1657 | return err; |
1658 | ioat_set_tcp_copy_break(262144); | ||
1659 | 1658 | ||
1660 | list_for_each_entry(c, &dma->channels, device_node) { | 1659 | list_for_each_entry(c, &dma->channels, device_node) { |
1661 | chan = to_chan_common(c); | 1660 | chan = to_chan_common(c); |
diff --git a/drivers/dma/iovlock.c b/drivers/dma/iovlock.c deleted file mode 100644 index bb48a57c2fc1..000000000000 --- a/drivers/dma/iovlock.c +++ /dev/null | |||
@@ -1,280 +0,0 @@ | |||
1 | /* | ||
2 | * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. | ||
3 | * Portions based on net/core/datagram.c and copyrighted by their authors. | ||
4 | * | ||
5 | * This program is free software; you can redistribute it and/or modify it | ||
6 | * under the terms of the GNU General Public License as published by the Free | ||
7 | * Software Foundation; either version 2 of the License, or (at your option) | ||
8 | * any later version. | ||
9 | * | ||
10 | * This program is distributed in the hope that it will be useful, but WITHOUT | ||
11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
13 | * more details. | ||
14 | * | ||
15 | * You should have received a copy of the GNU General Public License along with | ||
16 | * this program; if not, write to the Free Software Foundation, Inc., 59 | ||
17 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. | ||
18 | * | ||
19 | * The full GNU General Public License is included in this distribution in the | ||
20 | * file called COPYING. | ||
21 | */ | ||
22 | |||
23 | /* | ||
24 | * This code allows the net stack to make use of a DMA engine for | ||
25 | * skb to iovec copies. | ||
26 | */ | ||
27 | |||
28 | #include <linux/dmaengine.h> | ||
29 | #include <linux/pagemap.h> | ||
30 | #include <linux/slab.h> | ||
31 | #include <net/tcp.h> /* for memcpy_toiovec */ | ||
32 | #include <asm/io.h> | ||
33 | #include <asm/uaccess.h> | ||
34 | |||
35 | static int num_pages_spanned(struct iovec *iov) | ||
36 | { | ||
37 | return | ||
38 | ((PAGE_ALIGN((unsigned long)iov->iov_base + iov->iov_len) - | ||
39 | ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT); | ||
40 | } | ||
41 | |||
42 | /* | ||
43 | * Pin down all the iovec pages needed for len bytes. | ||
44 | * Return a struct dma_pinned_list to keep track of pages pinned down. | ||
45 | * | ||
46 | * We are allocating a single chunk of memory, and then carving it up into | ||
47 | * 3 sections, the latter 2 whose size depends on the number of iovecs and the | ||
48 | * total number of pages, respectively. | ||
49 | */ | ||
50 | struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len) | ||
51 | { | ||
52 | struct dma_pinned_list *local_list; | ||
53 | struct page **pages; | ||
54 | int i; | ||
55 | int ret; | ||
56 | int nr_iovecs = 0; | ||
57 | int iovec_len_used = 0; | ||
58 | int iovec_pages_used = 0; | ||
59 | |||
60 | /* don't pin down non-user-based iovecs */ | ||
61 | if (segment_eq(get_fs(), KERNEL_DS)) | ||
62 | return NULL; | ||
63 | |||
64 | /* determine how many iovecs/pages there are, up front */ | ||
65 | do { | ||
66 | iovec_len_used += iov[nr_iovecs].iov_len; | ||
67 | iovec_pages_used += num_pages_spanned(&iov[nr_iovecs]); | ||
68 | nr_iovecs++; | ||
69 | } while (iovec_len_used < len); | ||
70 | |||
71 | /* single kmalloc for pinned list, page_list[], and the page arrays */ | ||
72 | local_list = kmalloc(sizeof(*local_list) | ||
73 | + (nr_iovecs * sizeof (struct dma_page_list)) | ||
74 | + (iovec_pages_used * sizeof (struct page*)), GFP_KERNEL); | ||
75 | if (!local_list) | ||
76 | goto out; | ||
77 | |||
78 | /* list of pages starts right after the page list array */ | ||
79 | pages = (struct page **) &local_list->page_list[nr_iovecs]; | ||
80 | |||
81 | local_list->nr_iovecs = 0; | ||
82 | |||
83 | for (i = 0; i < nr_iovecs; i++) { | ||
84 | struct dma_page_list *page_list = &local_list->page_list[i]; | ||
85 | |||
86 | len -= iov[i].iov_len; | ||
87 | |||
88 | if (!access_ok(VERIFY_WRITE, iov[i].iov_base, iov[i].iov_len)) | ||
89 | goto unpin; | ||
90 | |||
91 | page_list->nr_pages = num_pages_spanned(&iov[i]); | ||
92 | page_list->base_address = iov[i].iov_base; | ||
93 | |||
94 | page_list->pages = pages; | ||
95 | pages += page_list->nr_pages; | ||
96 | |||
97 | /* pin pages down */ | ||
98 | down_read(¤t->mm->mmap_sem); | ||
99 | ret = get_user_pages( | ||
100 | current, | ||
101 | current->mm, | ||
102 | (unsigned long) iov[i].iov_base, | ||
103 | page_list->nr_pages, | ||
104 | 1, /* write */ | ||
105 | 0, /* force */ | ||
106 | page_list->pages, | ||
107 | NULL); | ||
108 | up_read(¤t->mm->mmap_sem); | ||
109 | |||
110 | if (ret != page_list->nr_pages) | ||
111 | goto unpin; | ||
112 | |||
113 | local_list->nr_iovecs = i + 1; | ||
114 | } | ||
115 | |||
116 | return local_list; | ||
117 | |||
118 | unpin: | ||
119 | dma_unpin_iovec_pages(local_list); | ||
120 | out: | ||
121 | return NULL; | ||
122 | } | ||
123 | |||
124 | void dma_unpin_iovec_pages(struct dma_pinned_list *pinned_list) | ||
125 | { | ||
126 | int i, j; | ||
127 | |||
128 | if (!pinned_list) | ||
129 | return; | ||
130 | |||
131 | for (i = 0; i < pinned_list->nr_iovecs; i++) { | ||
132 | struct dma_page_list *page_list = &pinned_list->page_list[i]; | ||
133 | for (j = 0; j < page_list->nr_pages; j++) { | ||
134 | set_page_dirty_lock(page_list->pages[j]); | ||
135 | page_cache_release(page_list->pages[j]); | ||
136 | } | ||
137 | } | ||
138 | |||
139 | kfree(pinned_list); | ||
140 | } | ||
141 | |||
142 | |||
143 | /* | ||
144 | * We have already pinned down the pages we will be using in the iovecs. | ||
145 | * Each entry in iov array has corresponding entry in pinned_list->page_list. | ||
146 | * Using array indexing to keep iov[] and page_list[] in sync. | ||
147 | * Initial elements in iov array's iov->iov_len will be 0 if already copied into | ||
148 | * by another call. | ||
149 | * iov array length remaining guaranteed to be bigger than len. | ||
150 | */ | ||
151 | dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov, | ||
152 | struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len) | ||
153 | { | ||
154 | int iov_byte_offset; | ||
155 | int copy; | ||
156 | dma_cookie_t dma_cookie = 0; | ||
157 | int iovec_idx; | ||
158 | int page_idx; | ||
159 | |||
160 | if (!chan) | ||
161 | return memcpy_toiovec(iov, kdata, len); | ||
162 | |||
163 | iovec_idx = 0; | ||
164 | while (iovec_idx < pinned_list->nr_iovecs) { | ||
165 | struct dma_page_list *page_list; | ||
166 | |||
167 | /* skip already used-up iovecs */ | ||
168 | while (!iov[iovec_idx].iov_len) | ||
169 | iovec_idx++; | ||
170 | |||
171 | page_list = &pinned_list->page_list[iovec_idx]; | ||
172 | |||
173 | iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); | ||
174 | page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) | ||
175 | - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; | ||
176 | |||
177 | /* break up copies to not cross page boundary */ | ||
178 | while (iov[iovec_idx].iov_len) { | ||
179 | copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); | ||
180 | copy = min_t(int, copy, iov[iovec_idx].iov_len); | ||
181 | |||
182 | dma_cookie = dma_async_memcpy_buf_to_pg(chan, | ||
183 | page_list->pages[page_idx], | ||
184 | iov_byte_offset, | ||
185 | kdata, | ||
186 | copy); | ||
187 | /* poll for a descriptor slot */ | ||
188 | if (unlikely(dma_cookie < 0)) { | ||
189 | dma_async_issue_pending(chan); | ||
190 | continue; | ||
191 | } | ||
192 | |||
193 | len -= copy; | ||
194 | iov[iovec_idx].iov_len -= copy; | ||
195 | iov[iovec_idx].iov_base += copy; | ||
196 | |||
197 | if (!len) | ||
198 | return dma_cookie; | ||
199 | |||
200 | kdata += copy; | ||
201 | iov_byte_offset = 0; | ||
202 | page_idx++; | ||
203 | } | ||
204 | iovec_idx++; | ||
205 | } | ||
206 | |||
207 | /* really bad if we ever run out of iovecs */ | ||
208 | BUG(); | ||
209 | return -EFAULT; | ||
210 | } | ||
211 | |||
212 | dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov, | ||
213 | struct dma_pinned_list *pinned_list, struct page *page, | ||
214 | unsigned int offset, size_t len) | ||
215 | { | ||
216 | int iov_byte_offset; | ||
217 | int copy; | ||
218 | dma_cookie_t dma_cookie = 0; | ||
219 | int iovec_idx; | ||
220 | int page_idx; | ||
221 | int err; | ||
222 | |||
223 | /* this needs as-yet-unimplemented buf-to-buff, so punt. */ | ||
224 | /* TODO: use dma for this */ | ||
225 | if (!chan || !pinned_list) { | ||
226 | u8 *vaddr = kmap(page); | ||
227 | err = memcpy_toiovec(iov, vaddr + offset, len); | ||
228 | kunmap(page); | ||
229 | return err; | ||
230 | } | ||
231 | |||
232 | iovec_idx = 0; | ||
233 | while (iovec_idx < pinned_list->nr_iovecs) { | ||
234 | struct dma_page_list *page_list; | ||
235 | |||
236 | /* skip already used-up iovecs */ | ||
237 | while (!iov[iovec_idx].iov_len) | ||
238 | iovec_idx++; | ||
239 | |||
240 | page_list = &pinned_list->page_list[iovec_idx]; | ||
241 | |||
242 | iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); | ||
243 | page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) | ||
244 | - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; | ||
245 | |||
246 | /* break up copies to not cross page boundary */ | ||
247 | while (iov[iovec_idx].iov_len) { | ||
248 | copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); | ||
249 | copy = min_t(int, copy, iov[iovec_idx].iov_len); | ||
250 | |||
251 | dma_cookie = dma_async_memcpy_pg_to_pg(chan, | ||
252 | page_list->pages[page_idx], | ||
253 | iov_byte_offset, | ||
254 | page, | ||
255 | offset, | ||
256 | copy); | ||
257 | /* poll for a descriptor slot */ | ||
258 | if (unlikely(dma_cookie < 0)) { | ||
259 | dma_async_issue_pending(chan); | ||
260 | continue; | ||
261 | } | ||
262 | |||
263 | len -= copy; | ||
264 | iov[iovec_idx].iov_len -= copy; | ||
265 | iov[iovec_idx].iov_base += copy; | ||
266 | |||
267 | if (!len) | ||
268 | return dma_cookie; | ||
269 | |||
270 | offset += copy; | ||
271 | iov_byte_offset = 0; | ||
272 | page_idx++; | ||
273 | } | ||
274 | iovec_idx++; | ||
275 | } | ||
276 | |||
277 | /* really bad if we ever run out of iovecs */ | ||
278 | BUG(); | ||
279 | return -EFAULT; | ||
280 | } | ||