diff options
author | Doug Ledford <dledford@redhat.com> | 2015-12-15 14:10:44 -0500 |
---|---|---|
committer | Doug Ledford <dledford@redhat.com> | 2015-12-15 14:10:44 -0500 |
commit | c6333f9f9f7646e311248a09e8ed96126a97aba8 (patch) | |
tree | 1a86124c5b43fa858f1d8d7beb99bcd29d6d6ab7 /block | |
parent | 9f9499ae8e6415cefc4fe0a96ad0e27864353c89 (diff) | |
parent | cfeb91b375ad5f52665e00d374a4e403d2e6063e (diff) |
Merge branch 'rdma-cq.2' of git://git.infradead.org/users/hch/rdma into 4.5/rdma-cq
Signed-off-by: Doug Ledford <dledford@redhat.com>
Conflicts:
drivers/infiniband/ulp/srp/ib_srp.c - Conflicts with changes in
ib_srp.c introduced during 4.4-rc updates
Diffstat (limited to 'block')
-rw-r--r-- | block/Makefile | 2 | ||||
-rw-r--r-- | block/blk-iopoll.c | 224 |
2 files changed, 1 insertions, 225 deletions
diff --git a/block/Makefile b/block/Makefile index 00ecc97629db..e8504748c7cb 100644 --- a/block/Makefile +++ b/block/Makefile | |||
@@ -5,7 +5,7 @@ | |||
5 | obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \ | 5 | obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \ |
6 | blk-flush.o blk-settings.o blk-ioc.o blk-map.o \ | 6 | blk-flush.o blk-settings.o blk-ioc.o blk-map.o \ |
7 | blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \ | 7 | blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \ |
8 | blk-iopoll.o blk-lib.o blk-mq.o blk-mq-tag.o \ | 8 | blk-lib.o blk-mq.o blk-mq-tag.o \ |
9 | blk-mq-sysfs.o blk-mq-cpu.o blk-mq-cpumap.o ioctl.o \ | 9 | blk-mq-sysfs.o blk-mq-cpu.o blk-mq-cpumap.o ioctl.o \ |
10 | genhd.o scsi_ioctl.o partition-generic.o ioprio.o \ | 10 | genhd.o scsi_ioctl.o partition-generic.o ioprio.o \ |
11 | partitions/ | 11 | partitions/ |
diff --git a/block/blk-iopoll.c b/block/blk-iopoll.c deleted file mode 100644 index 0736729d6494..000000000000 --- a/block/blk-iopoll.c +++ /dev/null | |||
@@ -1,224 +0,0 @@ | |||
1 | /* | ||
2 | * Functions related to interrupt-poll handling in the block layer. This | ||
3 | * is similar to NAPI for network devices. | ||
4 | */ | ||
5 | #include <linux/kernel.h> | ||
6 | #include <linux/module.h> | ||
7 | #include <linux/init.h> | ||
8 | #include <linux/bio.h> | ||
9 | #include <linux/blkdev.h> | ||
10 | #include <linux/interrupt.h> | ||
11 | #include <linux/cpu.h> | ||
12 | #include <linux/blk-iopoll.h> | ||
13 | #include <linux/delay.h> | ||
14 | |||
15 | #include "blk.h" | ||
16 | |||
17 | static unsigned int blk_iopoll_budget __read_mostly = 256; | ||
18 | |||
19 | static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll); | ||
20 | |||
21 | /** | ||
22 | * blk_iopoll_sched - Schedule a run of the iopoll handler | ||
23 | * @iop: The parent iopoll structure | ||
24 | * | ||
25 | * Description: | ||
26 | * Add this blk_iopoll structure to the pending poll list and trigger the | ||
27 | * raise of the blk iopoll softirq. The driver must already have gotten a | ||
28 | * successful return from blk_iopoll_sched_prep() before calling this. | ||
29 | **/ | ||
30 | void blk_iopoll_sched(struct blk_iopoll *iop) | ||
31 | { | ||
32 | unsigned long flags; | ||
33 | |||
34 | local_irq_save(flags); | ||
35 | list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll)); | ||
36 | __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); | ||
37 | local_irq_restore(flags); | ||
38 | } | ||
39 | EXPORT_SYMBOL(blk_iopoll_sched); | ||
40 | |||
41 | /** | ||
42 | * __blk_iopoll_complete - Mark this @iop as un-polled again | ||
43 | * @iop: The parent iopoll structure | ||
44 | * | ||
45 | * Description: | ||
46 | * See blk_iopoll_complete(). This function must be called with interrupts | ||
47 | * disabled. | ||
48 | **/ | ||
49 | void __blk_iopoll_complete(struct blk_iopoll *iop) | ||
50 | { | ||
51 | list_del(&iop->list); | ||
52 | smp_mb__before_atomic(); | ||
53 | clear_bit_unlock(IOPOLL_F_SCHED, &iop->state); | ||
54 | } | ||
55 | EXPORT_SYMBOL(__blk_iopoll_complete); | ||
56 | |||
57 | /** | ||
58 | * blk_iopoll_complete - Mark this @iop as un-polled again | ||
59 | * @iop: The parent iopoll structure | ||
60 | * | ||
61 | * Description: | ||
62 | * If a driver consumes less than the assigned budget in its run of the | ||
63 | * iopoll handler, it'll end the polled mode by calling this function. The | ||
64 | * iopoll handler will not be invoked again before blk_iopoll_sched_prep() | ||
65 | * is called. | ||
66 | **/ | ||
67 | void blk_iopoll_complete(struct blk_iopoll *iop) | ||
68 | { | ||
69 | unsigned long flags; | ||
70 | |||
71 | local_irq_save(flags); | ||
72 | __blk_iopoll_complete(iop); | ||
73 | local_irq_restore(flags); | ||
74 | } | ||
75 | EXPORT_SYMBOL(blk_iopoll_complete); | ||
76 | |||
77 | static void blk_iopoll_softirq(struct softirq_action *h) | ||
78 | { | ||
79 | struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll); | ||
80 | int rearm = 0, budget = blk_iopoll_budget; | ||
81 | unsigned long start_time = jiffies; | ||
82 | |||
83 | local_irq_disable(); | ||
84 | |||
85 | while (!list_empty(list)) { | ||
86 | struct blk_iopoll *iop; | ||
87 | int work, weight; | ||
88 | |||
89 | /* | ||
90 | * If softirq window is exhausted then punt. | ||
91 | */ | ||
92 | if (budget <= 0 || time_after(jiffies, start_time)) { | ||
93 | rearm = 1; | ||
94 | break; | ||
95 | } | ||
96 | |||
97 | local_irq_enable(); | ||
98 | |||
99 | /* Even though interrupts have been re-enabled, this | ||
100 | * access is safe because interrupts can only add new | ||
101 | * entries to the tail of this list, and only ->poll() | ||
102 | * calls can remove this head entry from the list. | ||
103 | */ | ||
104 | iop = list_entry(list->next, struct blk_iopoll, list); | ||
105 | |||
106 | weight = iop->weight; | ||
107 | work = 0; | ||
108 | if (test_bit(IOPOLL_F_SCHED, &iop->state)) | ||
109 | work = iop->poll(iop, weight); | ||
110 | |||
111 | budget -= work; | ||
112 | |||
113 | local_irq_disable(); | ||
114 | |||
115 | /* | ||
116 | * Drivers must not modify the iopoll state, if they | ||
117 | * consume their assigned weight (or more, some drivers can't | ||
118 | * easily just stop processing, they have to complete an | ||
119 | * entire mask of commands).In such cases this code | ||
120 | * still "owns" the iopoll instance and therefore can | ||
121 | * move the instance around on the list at-will. | ||
122 | */ | ||
123 | if (work >= weight) { | ||
124 | if (blk_iopoll_disable_pending(iop)) | ||
125 | __blk_iopoll_complete(iop); | ||
126 | else | ||
127 | list_move_tail(&iop->list, list); | ||
128 | } | ||
129 | } | ||
130 | |||
131 | if (rearm) | ||
132 | __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); | ||
133 | |||
134 | local_irq_enable(); | ||
135 | } | ||
136 | |||
137 | /** | ||
138 | * blk_iopoll_disable - Disable iopoll on this @iop | ||
139 | * @iop: The parent iopoll structure | ||
140 | * | ||
141 | * Description: | ||
142 | * Disable io polling and wait for any pending callbacks to have completed. | ||
143 | **/ | ||
144 | void blk_iopoll_disable(struct blk_iopoll *iop) | ||
145 | { | ||
146 | set_bit(IOPOLL_F_DISABLE, &iop->state); | ||
147 | while (test_and_set_bit(IOPOLL_F_SCHED, &iop->state)) | ||
148 | msleep(1); | ||
149 | clear_bit(IOPOLL_F_DISABLE, &iop->state); | ||
150 | } | ||
151 | EXPORT_SYMBOL(blk_iopoll_disable); | ||
152 | |||
153 | /** | ||
154 | * blk_iopoll_enable - Enable iopoll on this @iop | ||
155 | * @iop: The parent iopoll structure | ||
156 | * | ||
157 | * Description: | ||
158 | * Enable iopoll on this @iop. Note that the handler run will not be | ||
159 | * scheduled, it will only mark it as active. | ||
160 | **/ | ||
161 | void blk_iopoll_enable(struct blk_iopoll *iop) | ||
162 | { | ||
163 | BUG_ON(!test_bit(IOPOLL_F_SCHED, &iop->state)); | ||
164 | smp_mb__before_atomic(); | ||
165 | clear_bit_unlock(IOPOLL_F_SCHED, &iop->state); | ||
166 | } | ||
167 | EXPORT_SYMBOL(blk_iopoll_enable); | ||
168 | |||
169 | /** | ||
170 | * blk_iopoll_init - Initialize this @iop | ||
171 | * @iop: The parent iopoll structure | ||
172 | * @weight: The default weight (or command completion budget) | ||
173 | * @poll_fn: The handler to invoke | ||
174 | * | ||
175 | * Description: | ||
176 | * Initialize this blk_iopoll structure. Before being actively used, the | ||
177 | * driver must call blk_iopoll_enable(). | ||
178 | **/ | ||
179 | void blk_iopoll_init(struct blk_iopoll *iop, int weight, blk_iopoll_fn *poll_fn) | ||
180 | { | ||
181 | memset(iop, 0, sizeof(*iop)); | ||
182 | INIT_LIST_HEAD(&iop->list); | ||
183 | iop->weight = weight; | ||
184 | iop->poll = poll_fn; | ||
185 | set_bit(IOPOLL_F_SCHED, &iop->state); | ||
186 | } | ||
187 | EXPORT_SYMBOL(blk_iopoll_init); | ||
188 | |||
189 | static int blk_iopoll_cpu_notify(struct notifier_block *self, | ||
190 | unsigned long action, void *hcpu) | ||
191 | { | ||
192 | /* | ||
193 | * If a CPU goes away, splice its entries to the current CPU | ||
194 | * and trigger a run of the softirq | ||
195 | */ | ||
196 | if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { | ||
197 | int cpu = (unsigned long) hcpu; | ||
198 | |||
199 | local_irq_disable(); | ||
200 | list_splice_init(&per_cpu(blk_cpu_iopoll, cpu), | ||
201 | this_cpu_ptr(&blk_cpu_iopoll)); | ||
202 | __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); | ||
203 | local_irq_enable(); | ||
204 | } | ||
205 | |||
206 | return NOTIFY_OK; | ||
207 | } | ||
208 | |||
209 | static struct notifier_block blk_iopoll_cpu_notifier = { | ||
210 | .notifier_call = blk_iopoll_cpu_notify, | ||
211 | }; | ||
212 | |||
213 | static __init int blk_iopoll_setup(void) | ||
214 | { | ||
215 | int i; | ||
216 | |||
217 | for_each_possible_cpu(i) | ||
218 | INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i)); | ||
219 | |||
220 | open_softirq(BLOCK_IOPOLL_SOFTIRQ, blk_iopoll_softirq); | ||
221 | register_hotcpu_notifier(&blk_iopoll_cpu_notifier); | ||
222 | return 0; | ||
223 | } | ||
224 | subsys_initcall(blk_iopoll_setup); | ||