aboutsummaryrefslogtreecommitdiffstats
path: root/block
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2016-01-23 21:45:06 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2016-01-23 21:45:06 -0500
commit048ccca8c1c8f583deec3367d7df521bb1f542ae (patch)
treeefa882c88f658f711d63581a3063203c63682338 /block
parentb3e27d5d4a29bcc8e057b496d5ef5194addaaac0 (diff)
parent34356f64ac0df2326fa50e2d4bca6f7c03ed16c1 (diff)
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma
Pull rdma updates from Doug Ledford: "Initial roundup of 4.5 merge window patches - Remove usage of ib_query_device and instead store attributes in ib_device struct - Move iopoll out of block and into lib, rename to irqpoll, and use in several places in the rdma stack as our new completion queue polling library mechanism. Update the other block drivers that already used iopoll to use the new mechanism too. - Replace the per-entry GID table locks with a single GID table lock - IPoIB multicast cleanup - Cleanups to the IB MR facility - Add support for 64bit extended IB counters - Fix for netlink oops while parsing RDMA nl messages - RoCEv2 support for the core IB code - mlx4 RoCEv2 support - mlx5 RoCEv2 support - Cross Channel support for mlx5 - Timestamp support for mlx5 - Atomic support for mlx5 - Raw QP support for mlx5 - MAINTAINERS update for mlx4/mlx5 - Misc ocrdma, qib, nes, usNIC, cxgb3, cxgb4, mlx4, mlx5 updates - Add support for remote invalidate to the iSER driver (pushed through the RDMA tree due to dependencies, acknowledged by nab) - Update to NFSoRDMA (pushed through the RDMA tree due to dependencies, acknowledged by Bruce)" * tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma: (169 commits) IB/mlx5: Unify CQ create flags check IB/mlx5: Expose Raw Packet QP to user space consumers {IB, net}/mlx5: Move the modify QP operation table to mlx5_ib IB/mlx5: Support setting Ethernet priority for Raw Packet QPs IB/mlx5: Add Raw Packet QP query functionality IB/mlx5: Add create and destroy functionality for Raw Packet QP IB/mlx5: Refactor mlx5_ib_qp to accommodate other QP types IB/mlx5: Allocate a Transport Domain for each ucontext net/mlx5_core: Warn on unsupported events of QP/RQ/SQ net/mlx5_core: Add RQ and SQ event handling net/mlx5_core: Export transport objects IB/mlx5: Expose CQE version to user-space IB/mlx5: Add CQE version 1 support to user QPs and SRQs IB/mlx5: Fix data validation in mlx5_ib_alloc_ucontext IB/sa: Fix netlink local service GFP crash IB/srpt: Remove redundant wc array IB/qib: Improve ipoib UD performance IB/mlx4: Advertise RoCE v2 support IB/mlx4: Create and use another QP1 for RoCEv2 IB/mlx4: Enable send of RoCE QP1 packets with IP/UDP headers ...
Diffstat (limited to 'block')
-rw-r--r--block/Makefile2
-rw-r--r--block/blk-iopoll.c224
2 files changed, 1 insertions, 225 deletions
diff --git a/block/Makefile b/block/Makefile
index db5f622c9d67..9eda2322b2d4 100644
--- a/block/Makefile
+++ b/block/Makefile
@@ -5,7 +5,7 @@
5obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \ 5obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \
6 blk-flush.o blk-settings.o blk-ioc.o blk-map.o \ 6 blk-flush.o blk-settings.o blk-ioc.o blk-map.o \
7 blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \ 7 blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \
8 blk-iopoll.o blk-lib.o blk-mq.o blk-mq-tag.o \ 8 blk-lib.o blk-mq.o blk-mq-tag.o \
9 blk-mq-sysfs.o blk-mq-cpu.o blk-mq-cpumap.o ioctl.o \ 9 blk-mq-sysfs.o blk-mq-cpu.o blk-mq-cpumap.o ioctl.o \
10 genhd.o scsi_ioctl.o partition-generic.o ioprio.o \ 10 genhd.o scsi_ioctl.o partition-generic.o ioprio.o \
11 badblocks.o partitions/ 11 badblocks.o partitions/
diff --git a/block/blk-iopoll.c b/block/blk-iopoll.c
deleted file mode 100644
index 0736729d6494..000000000000
--- a/block/blk-iopoll.c
+++ /dev/null
@@ -1,224 +0,0 @@
1/*
2 * Functions related to interrupt-poll handling in the block layer. This
3 * is similar to NAPI for network devices.
4 */
5#include <linux/kernel.h>
6#include <linux/module.h>
7#include <linux/init.h>
8#include <linux/bio.h>
9#include <linux/blkdev.h>
10#include <linux/interrupt.h>
11#include <linux/cpu.h>
12#include <linux/blk-iopoll.h>
13#include <linux/delay.h>
14
15#include "blk.h"
16
17static unsigned int blk_iopoll_budget __read_mostly = 256;
18
19static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll);
20
21/**
22 * blk_iopoll_sched - Schedule a run of the iopoll handler
23 * @iop: The parent iopoll structure
24 *
25 * Description:
26 * Add this blk_iopoll structure to the pending poll list and trigger the
27 * raise of the blk iopoll softirq. The driver must already have gotten a
28 * successful return from blk_iopoll_sched_prep() before calling this.
29 **/
30void blk_iopoll_sched(struct blk_iopoll *iop)
31{
32 unsigned long flags;
33
34 local_irq_save(flags);
35 list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll));
36 __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ);
37 local_irq_restore(flags);
38}
39EXPORT_SYMBOL(blk_iopoll_sched);
40
41/**
42 * __blk_iopoll_complete - Mark this @iop as un-polled again
43 * @iop: The parent iopoll structure
44 *
45 * Description:
46 * See blk_iopoll_complete(). This function must be called with interrupts
47 * disabled.
48 **/
49void __blk_iopoll_complete(struct blk_iopoll *iop)
50{
51 list_del(&iop->list);
52 smp_mb__before_atomic();
53 clear_bit_unlock(IOPOLL_F_SCHED, &iop->state);
54}
55EXPORT_SYMBOL(__blk_iopoll_complete);
56
57/**
58 * blk_iopoll_complete - Mark this @iop as un-polled again
59 * @iop: The parent iopoll structure
60 *
61 * Description:
62 * If a driver consumes less than the assigned budget in its run of the
63 * iopoll handler, it'll end the polled mode by calling this function. The
64 * iopoll handler will not be invoked again before blk_iopoll_sched_prep()
65 * is called.
66 **/
67void blk_iopoll_complete(struct blk_iopoll *iop)
68{
69 unsigned long flags;
70
71 local_irq_save(flags);
72 __blk_iopoll_complete(iop);
73 local_irq_restore(flags);
74}
75EXPORT_SYMBOL(blk_iopoll_complete);
76
77static void blk_iopoll_softirq(struct softirq_action *h)
78{
79 struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll);
80 int rearm = 0, budget = blk_iopoll_budget;
81 unsigned long start_time = jiffies;
82
83 local_irq_disable();
84
85 while (!list_empty(list)) {
86 struct blk_iopoll *iop;
87 int work, weight;
88
89 /*
90 * If softirq window is exhausted then punt.
91 */
92 if (budget <= 0 || time_after(jiffies, start_time)) {
93 rearm = 1;
94 break;
95 }
96
97 local_irq_enable();
98
99 /* Even though interrupts have been re-enabled, this
100 * access is safe because interrupts can only add new
101 * entries to the tail of this list, and only ->poll()
102 * calls can remove this head entry from the list.
103 */
104 iop = list_entry(list->next, struct blk_iopoll, list);
105
106 weight = iop->weight;
107 work = 0;
108 if (test_bit(IOPOLL_F_SCHED, &iop->state))
109 work = iop->poll(iop, weight);
110
111 budget -= work;
112
113 local_irq_disable();
114
115 /*
116 * Drivers must not modify the iopoll state, if they
117 * consume their assigned weight (or more, some drivers can't
118 * easily just stop processing, they have to complete an
119 * entire mask of commands).In such cases this code
120 * still "owns" the iopoll instance and therefore can
121 * move the instance around on the list at-will.
122 */
123 if (work >= weight) {
124 if (blk_iopoll_disable_pending(iop))
125 __blk_iopoll_complete(iop);
126 else
127 list_move_tail(&iop->list, list);
128 }
129 }
130
131 if (rearm)
132 __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ);
133
134 local_irq_enable();
135}
136
137/**
138 * blk_iopoll_disable - Disable iopoll on this @iop
139 * @iop: The parent iopoll structure
140 *
141 * Description:
142 * Disable io polling and wait for any pending callbacks to have completed.
143 **/
144void blk_iopoll_disable(struct blk_iopoll *iop)
145{
146 set_bit(IOPOLL_F_DISABLE, &iop->state);
147 while (test_and_set_bit(IOPOLL_F_SCHED, &iop->state))
148 msleep(1);
149 clear_bit(IOPOLL_F_DISABLE, &iop->state);
150}
151EXPORT_SYMBOL(blk_iopoll_disable);
152
153/**
154 * blk_iopoll_enable - Enable iopoll on this @iop
155 * @iop: The parent iopoll structure
156 *
157 * Description:
158 * Enable iopoll on this @iop. Note that the handler run will not be
159 * scheduled, it will only mark it as active.
160 **/
161void blk_iopoll_enable(struct blk_iopoll *iop)
162{
163 BUG_ON(!test_bit(IOPOLL_F_SCHED, &iop->state));
164 smp_mb__before_atomic();
165 clear_bit_unlock(IOPOLL_F_SCHED, &iop->state);
166}
167EXPORT_SYMBOL(blk_iopoll_enable);
168
169/**
170 * blk_iopoll_init - Initialize this @iop
171 * @iop: The parent iopoll structure
172 * @weight: The default weight (or command completion budget)
173 * @poll_fn: The handler to invoke
174 *
175 * Description:
176 * Initialize this blk_iopoll structure. Before being actively used, the
177 * driver must call blk_iopoll_enable().
178 **/
179void blk_iopoll_init(struct blk_iopoll *iop, int weight, blk_iopoll_fn *poll_fn)
180{
181 memset(iop, 0, sizeof(*iop));
182 INIT_LIST_HEAD(&iop->list);
183 iop->weight = weight;
184 iop->poll = poll_fn;
185 set_bit(IOPOLL_F_SCHED, &iop->state);
186}
187EXPORT_SYMBOL(blk_iopoll_init);
188
189static int blk_iopoll_cpu_notify(struct notifier_block *self,
190 unsigned long action, void *hcpu)
191{
192 /*
193 * If a CPU goes away, splice its entries to the current CPU
194 * and trigger a run of the softirq
195 */
196 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
197 int cpu = (unsigned long) hcpu;
198
199 local_irq_disable();
200 list_splice_init(&per_cpu(blk_cpu_iopoll, cpu),
201 this_cpu_ptr(&blk_cpu_iopoll));
202 __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ);
203 local_irq_enable();
204 }
205
206 return NOTIFY_OK;
207}
208
209static struct notifier_block blk_iopoll_cpu_notifier = {
210 .notifier_call = blk_iopoll_cpu_notify,
211};
212
213static __init int blk_iopoll_setup(void)
214{
215 int i;
216
217 for_each_possible_cpu(i)
218 INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i));
219
220 open_softirq(BLOCK_IOPOLL_SOFTIRQ, blk_iopoll_softirq);
221 register_hotcpu_notifier(&blk_iopoll_cpu_notifier);
222 return 0;
223}
224subsys_initcall(blk_iopoll_setup);
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-13 11:27:29 -0400