summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChristoph Hellwig <hch@lst.de>2015-11-10 08:56:14 -0500
committerChristoph Hellwig <hch@lst.de>2015-12-11 14:52:24 -0500
commit511cbce2ff8b9d322077909ee90c5d4b67b29b75 (patch)
tree27e1f9e09332d204ec1a1a506e674ca80ed9e580
parenta2dbb7b56f2c29fc78b18a3fbe47ad80f6912092 (diff)
irq_poll: make blk-iopoll available outside the block layer
The new name is irq_poll as iopoll is already taken. Better suggestions welcome. Signed-off-by: Christoph Hellwig <hch@lst.de> Reviewed-by: Bart Van Assche <bart.vanassche@sandisk.com>
-rw-r--r--Documentation/kernel-per-CPU-kthreads.txt2
-rw-r--r--block/Makefile2
-rw-r--r--drivers/scsi/Kconfig1
-rw-r--r--drivers/scsi/be2iscsi/Kconfig1
-rw-r--r--drivers/scsi/be2iscsi/be.h4
-rw-r--r--drivers/scsi/be2iscsi/be_iscsi.c4
-rw-r--r--drivers/scsi/be2iscsi/be_main.c24
-rw-r--r--drivers/scsi/ipr.c28
-rw-r--r--drivers/scsi/ipr.h4
-rw-r--r--include/linux/blk-iopoll.h46
-rw-r--r--include/linux/interrupt.h2
-rw-r--r--include/linux/irq_poll.h46
-rw-r--r--include/trace/events/irq.h2
-rw-r--r--lib/Kconfig5
-rw-r--r--lib/Makefile1
-rw-r--r--lib/irq_poll.c (renamed from block/blk-iopoll.c)105
-rw-r--r--tools/lib/traceevent/event-parse.c2
-rw-r--r--tools/perf/util/trace-event-parse.c2
18 files changed, 143 insertions, 138 deletions
diff --git a/Documentation/kernel-per-CPU-kthreads.txt b/Documentation/kernel-per-CPU-kthreads.txt
index f4cbfe0ba108..edec3a3e648d 100644
--- a/Documentation/kernel-per-CPU-kthreads.txt
+++ b/Documentation/kernel-per-CPU-kthreads.txt
@@ -90,7 +90,7 @@ BLOCK_SOFTIRQ: Do all of the following:
90 from being initiated from tasks that might run on the CPU to 90 from being initiated from tasks that might run on the CPU to
91 be de-jittered. (It is OK to force this CPU offline and then 91 be de-jittered. (It is OK to force this CPU offline and then
92 bring it back online before you start your application.) 92 bring it back online before you start your application.)
93BLOCK_IOPOLL_SOFTIRQ: Do all of the following: 93IRQ_POLL_SOFTIRQ: Do all of the following:
941. Force block-device interrupts onto some other CPU. 941. Force block-device interrupts onto some other CPU.
952. Initiate any block I/O and block-I/O polling on other CPUs. 952. Initiate any block I/O and block-I/O polling on other CPUs.
963. Once your application has started, prevent CPU-hotplug operations 963. Once your application has started, prevent CPU-hotplug operations
diff --git a/block/Makefile b/block/Makefile
index 00ecc97629db..e8504748c7cb 100644
--- a/block/Makefile
+++ b/block/Makefile
@@ -5,7 +5,7 @@
5obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \ 5obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \
6 blk-flush.o blk-settings.o blk-ioc.o blk-map.o \ 6 blk-flush.o blk-settings.o blk-ioc.o blk-map.o \
7 blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \ 7 blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \
8 blk-iopoll.o blk-lib.o blk-mq.o blk-mq-tag.o \ 8 blk-lib.o blk-mq.o blk-mq-tag.o \
9 blk-mq-sysfs.o blk-mq-cpu.o blk-mq-cpumap.o ioctl.o \ 9 blk-mq-sysfs.o blk-mq-cpu.o blk-mq-cpumap.o ioctl.o \
10 genhd.o scsi_ioctl.o partition-generic.o ioprio.o \ 10 genhd.o scsi_ioctl.o partition-generic.o ioprio.o \
11 partitions/ 11 partitions/
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index 5f692ae40749..92099657b046 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -1102,6 +1102,7 @@ config SCSI_IPR
1102 tristate "IBM Power Linux RAID adapter support" 1102 tristate "IBM Power Linux RAID adapter support"
1103 depends on PCI && SCSI && ATA 1103 depends on PCI && SCSI && ATA
1104 select FW_LOADER 1104 select FW_LOADER
1105 select IRQ_POLL
1105 ---help--- 1106 ---help---
1106 This driver supports the IBM Power Linux family RAID adapters. 1107 This driver supports the IBM Power Linux family RAID adapters.
1107 This includes IBM pSeries 5712, 5703, 5709, and 570A, as well 1108 This includes IBM pSeries 5712, 5703, 5709, and 570A, as well
diff --git a/drivers/scsi/be2iscsi/Kconfig b/drivers/scsi/be2iscsi/Kconfig
index 4e7cad272469..bad5f32e1f67 100644
--- a/drivers/scsi/be2iscsi/Kconfig
+++ b/drivers/scsi/be2iscsi/Kconfig
@@ -3,6 +3,7 @@ config BE2ISCSI
3 depends on PCI && SCSI && NET 3 depends on PCI && SCSI && NET
4 select SCSI_ISCSI_ATTRS 4 select SCSI_ISCSI_ATTRS
5 select ISCSI_BOOT_SYSFS 5 select ISCSI_BOOT_SYSFS
6 select IRQ_POLL
6 7
7 help 8 help
8 This driver implements the iSCSI functionality for Emulex 9 This driver implements the iSCSI functionality for Emulex
diff --git a/drivers/scsi/be2iscsi/be.h b/drivers/scsi/be2iscsi/be.h
index 77f992e74726..a41c6432f444 100644
--- a/drivers/scsi/be2iscsi/be.h
+++ b/drivers/scsi/be2iscsi/be.h
@@ -20,7 +20,7 @@
20 20
21#include <linux/pci.h> 21#include <linux/pci.h>
22#include <linux/if_vlan.h> 22#include <linux/if_vlan.h>
23#include <linux/blk-iopoll.h> 23#include <linux/irq_poll.h>
24#define FW_VER_LEN 32 24#define FW_VER_LEN 32
25#define MCC_Q_LEN 128 25#define MCC_Q_LEN 128
26#define MCC_CQ_LEN 256 26#define MCC_CQ_LEN 256
@@ -101,7 +101,7 @@ struct be_eq_obj {
101 struct beiscsi_hba *phba; 101 struct beiscsi_hba *phba;
102 struct be_queue_info *cq; 102 struct be_queue_info *cq;
103 struct work_struct work_cqs; /* Work Item */ 103 struct work_struct work_cqs; /* Work Item */
104 struct blk_iopoll iopoll; 104 struct irq_poll iopoll;
105}; 105};
106 106
107struct be_mcc_obj { 107struct be_mcc_obj {
diff --git a/drivers/scsi/be2iscsi/be_iscsi.c b/drivers/scsi/be2iscsi/be_iscsi.c
index b7087ba69d8d..022e87b62e40 100644
--- a/drivers/scsi/be2iscsi/be_iscsi.c
+++ b/drivers/scsi/be2iscsi/be_iscsi.c
@@ -1292,9 +1292,9 @@ static void beiscsi_flush_cq(struct beiscsi_hba *phba)
1292 1292
1293 for (i = 0; i < phba->num_cpus; i++) { 1293 for (i = 0; i < phba->num_cpus; i++) {
1294 pbe_eq = &phwi_context->be_eq[i]; 1294 pbe_eq = &phwi_context->be_eq[i];
1295 blk_iopoll_disable(&pbe_eq->iopoll); 1295 irq_poll_disable(&pbe_eq->iopoll);
1296 beiscsi_process_cq(pbe_eq); 1296 beiscsi_process_cq(pbe_eq);
1297 blk_iopoll_enable(&pbe_eq->iopoll); 1297 irq_poll_enable(&pbe_eq->iopoll);
1298 } 1298 }
1299} 1299}
1300 1300
diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c
index fe0c5143f8e6..1d879ef406d8 100644
--- a/drivers/scsi/be2iscsi/be_main.c
+++ b/drivers/scsi/be2iscsi/be_main.c
@@ -910,8 +910,8 @@ static irqreturn_t be_isr_msix(int irq, void *dev_id)
910 num_eq_processed = 0; 910 num_eq_processed = 0;
911 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 911 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
912 & EQE_VALID_MASK) { 912 & EQE_VALID_MASK) {
913 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll)) 913 if (!irq_poll_sched_prep(&pbe_eq->iopoll))
914 blk_iopoll_sched(&pbe_eq->iopoll); 914 irq_poll_sched(&pbe_eq->iopoll);
915 915
916 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 916 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
917 queue_tail_inc(eq); 917 queue_tail_inc(eq);
@@ -972,8 +972,8 @@ static irqreturn_t be_isr(int irq, void *dev_id)
972 spin_unlock_irqrestore(&phba->isr_lock, flags); 972 spin_unlock_irqrestore(&phba->isr_lock, flags);
973 num_mcceq_processed++; 973 num_mcceq_processed++;
974 } else { 974 } else {
975 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll)) 975 if (!irq_poll_sched_prep(&pbe_eq->iopoll))
976 blk_iopoll_sched(&pbe_eq->iopoll); 976 irq_poll_sched(&pbe_eq->iopoll);
977 num_ioeq_processed++; 977 num_ioeq_processed++;
978 } 978 }
979 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0); 979 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
@@ -2295,7 +2295,7 @@ void beiscsi_process_all_cqs(struct work_struct *work)
2295 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1); 2295 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2296} 2296}
2297 2297
2298static int be_iopoll(struct blk_iopoll *iop, int budget) 2298static int be_iopoll(struct irq_poll *iop, int budget)
2299{ 2299{
2300 unsigned int ret; 2300 unsigned int ret;
2301 struct beiscsi_hba *phba; 2301 struct beiscsi_hba *phba;
@@ -2306,7 +2306,7 @@ static int be_iopoll(struct blk_iopoll *iop, int budget)
2306 pbe_eq->cq_count += ret; 2306 pbe_eq->cq_count += ret;
2307 if (ret < budget) { 2307 if (ret < budget) {
2308 phba = pbe_eq->phba; 2308 phba = pbe_eq->phba;
2309 blk_iopoll_complete(iop); 2309 irq_poll_complete(iop);
2310 beiscsi_log(phba, KERN_INFO, 2310 beiscsi_log(phba, KERN_INFO,
2311 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO, 2311 BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2312 "BM_%d : rearm pbe_eq->q.id =%d\n", 2312 "BM_%d : rearm pbe_eq->q.id =%d\n",
@@ -5293,7 +5293,7 @@ static void beiscsi_quiesce(struct beiscsi_hba *phba,
5293 5293
5294 for (i = 0; i < phba->num_cpus; i++) { 5294 for (i = 0; i < phba->num_cpus; i++) {
5295 pbe_eq = &phwi_context->be_eq[i]; 5295 pbe_eq = &phwi_context->be_eq[i];
5296 blk_iopoll_disable(&pbe_eq->iopoll); 5296 irq_poll_disable(&pbe_eq->iopoll);
5297 } 5297 }
5298 5298
5299 if (unload_state == BEISCSI_CLEAN_UNLOAD) { 5299 if (unload_state == BEISCSI_CLEAN_UNLOAD) {
@@ -5579,9 +5579,9 @@ static void beiscsi_eeh_resume(struct pci_dev *pdev)
5579 5579
5580 for (i = 0; i < phba->num_cpus; i++) { 5580 for (i = 0; i < phba->num_cpus; i++) {
5581 pbe_eq = &phwi_context->be_eq[i]; 5581 pbe_eq = &phwi_context->be_eq[i];
5582 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget, 5582 irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget,
5583 be_iopoll); 5583 be_iopoll);
5584 blk_iopoll_enable(&pbe_eq->iopoll); 5584 irq_poll_enable(&pbe_eq->iopoll);
5585 } 5585 }
5586 5586
5587 i = (phba->msix_enabled) ? i : 0; 5587 i = (phba->msix_enabled) ? i : 0;
@@ -5752,9 +5752,9 @@ static int beiscsi_dev_probe(struct pci_dev *pcidev,
5752 5752
5753 for (i = 0; i < phba->num_cpus; i++) { 5753 for (i = 0; i < phba->num_cpus; i++) {
5754 pbe_eq = &phwi_context->be_eq[i]; 5754 pbe_eq = &phwi_context->be_eq[i];
5755 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget, 5755 irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget,
5756 be_iopoll); 5756 be_iopoll);
5757 blk_iopoll_enable(&pbe_eq->iopoll); 5757 irq_poll_enable(&pbe_eq->iopoll);
5758 } 5758 }
5759 5759
5760 i = (phba->msix_enabled) ? i : 0; 5760 i = (phba->msix_enabled) ? i : 0;
@@ -5795,7 +5795,7 @@ free_blkenbld:
5795 destroy_workqueue(phba->wq); 5795 destroy_workqueue(phba->wq);
5796 for (i = 0; i < phba->num_cpus; i++) { 5796 for (i = 0; i < phba->num_cpus; i++) {
5797 pbe_eq = &phwi_context->be_eq[i]; 5797 pbe_eq = &phwi_context->be_eq[i];
5798 blk_iopoll_disable(&pbe_eq->iopoll); 5798 irq_poll_disable(&pbe_eq->iopoll);
5799 } 5799 }
5800free_twq: 5800free_twq:
5801 beiscsi_clean_port(phba); 5801 beiscsi_clean_port(phba);
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index 536cd5a80422..6b9c738cdc18 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -3638,7 +3638,7 @@ static struct device_attribute ipr_ioa_reset_attr = {
3638 .store = ipr_store_reset_adapter 3638 .store = ipr_store_reset_adapter
3639}; 3639};
3640 3640
3641static int ipr_iopoll(struct blk_iopoll *iop, int budget); 3641static int ipr_iopoll(struct irq_poll *iop, int budget);
3642 /** 3642 /**
3643 * ipr_show_iopoll_weight - Show ipr polling mode 3643 * ipr_show_iopoll_weight - Show ipr polling mode
3644 * @dev: class device struct 3644 * @dev: class device struct
@@ -3681,34 +3681,34 @@ static ssize_t ipr_store_iopoll_weight(struct device *dev,
3681 int i; 3681 int i;
3682 3682
3683 if (!ioa_cfg->sis64) { 3683 if (!ioa_cfg->sis64) {
3684 dev_info(&ioa_cfg->pdev->dev, "blk-iopoll not supported on this adapter\n"); 3684 dev_info(&ioa_cfg->pdev->dev, "irq_poll not supported on this adapter\n");
3685 return -EINVAL; 3685 return -EINVAL;
3686 } 3686 }
3687 if (kstrtoul(buf, 10, &user_iopoll_weight)) 3687 if (kstrtoul(buf, 10, &user_iopoll_weight))
3688 return -EINVAL; 3688 return -EINVAL;
3689 3689
3690 if (user_iopoll_weight > 256) { 3690 if (user_iopoll_weight > 256) {
3691 dev_info(&ioa_cfg->pdev->dev, "Invalid blk-iopoll weight. It must be less than 256\n"); 3691 dev_info(&ioa_cfg->pdev->dev, "Invalid irq_poll weight. It must be less than 256\n");
3692 return -EINVAL; 3692 return -EINVAL;
3693 } 3693 }
3694 3694
3695 if (user_iopoll_weight == ioa_cfg->iopoll_weight) { 3695 if (user_iopoll_weight == ioa_cfg->iopoll_weight) {
3696 dev_info(&ioa_cfg->pdev->dev, "Current blk-iopoll weight has the same weight\n"); 3696 dev_info(&ioa_cfg->pdev->dev, "Current irq_poll weight has the same weight\n");
3697 return strlen(buf); 3697 return strlen(buf);
3698 } 3698 }
3699 3699
3700 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) { 3700 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
3701 for (i = 1; i < ioa_cfg->hrrq_num; i++) 3701 for (i = 1; i < ioa_cfg->hrrq_num; i++)
3702 blk_iopoll_disable(&ioa_cfg->hrrq[i].iopoll); 3702 irq_poll_disable(&ioa_cfg->hrrq[i].iopoll);
3703 } 3703 }
3704 3704
3705 spin_lock_irqsave(shost->host_lock, lock_flags); 3705 spin_lock_irqsave(shost->host_lock, lock_flags);
3706 ioa_cfg->iopoll_weight = user_iopoll_weight; 3706 ioa_cfg->iopoll_weight = user_iopoll_weight;
3707 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) { 3707 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
3708 for (i = 1; i < ioa_cfg->hrrq_num; i++) { 3708 for (i = 1; i < ioa_cfg->hrrq_num; i++) {
3709 blk_iopoll_init(&ioa_cfg->hrrq[i].iopoll, 3709 irq_poll_init(&ioa_cfg->hrrq[i].iopoll,
3710 ioa_cfg->iopoll_weight, ipr_iopoll); 3710 ioa_cfg->iopoll_weight, ipr_iopoll);
3711 blk_iopoll_enable(&ioa_cfg->hrrq[i].iopoll); 3711 irq_poll_enable(&ioa_cfg->hrrq[i].iopoll);
3712 } 3712 }
3713 } 3713 }
3714 spin_unlock_irqrestore(shost->host_lock, lock_flags); 3714 spin_unlock_irqrestore(shost->host_lock, lock_flags);
@@ -5569,7 +5569,7 @@ static int ipr_process_hrrq(struct ipr_hrr_queue *hrr_queue, int budget,
5569 return num_hrrq; 5569 return num_hrrq;
5570} 5570}
5571 5571
5572static int ipr_iopoll(struct blk_iopoll *iop, int budget) 5572static int ipr_iopoll(struct irq_poll *iop, int budget)
5573{ 5573{
5574 struct ipr_ioa_cfg *ioa_cfg; 5574 struct ipr_ioa_cfg *ioa_cfg;
5575 struct ipr_hrr_queue *hrrq; 5575 struct ipr_hrr_queue *hrrq;
@@ -5585,7 +5585,7 @@ static int ipr_iopoll(struct blk_iopoll *iop, int budget)
5585 completed_ops = ipr_process_hrrq(hrrq, budget, &doneq); 5585 completed_ops = ipr_process_hrrq(hrrq, budget, &doneq);
5586 5586
5587 if (completed_ops < budget) 5587 if (completed_ops < budget)
5588 blk_iopoll_complete(iop); 5588 irq_poll_complete(iop);
5589 spin_unlock_irqrestore(hrrq->lock, hrrq_flags); 5589 spin_unlock_irqrestore(hrrq->lock, hrrq_flags);
5590 5590
5591 list_for_each_entry_safe(ipr_cmd, temp, &doneq, queue) { 5591 list_for_each_entry_safe(ipr_cmd, temp, &doneq, queue) {
@@ -5693,8 +5693,8 @@ static irqreturn_t ipr_isr_mhrrq(int irq, void *devp)
5693 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) { 5693 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
5694 if ((be32_to_cpu(*hrrq->hrrq_curr) & IPR_HRRQ_TOGGLE_BIT) == 5694 if ((be32_to_cpu(*hrrq->hrrq_curr) & IPR_HRRQ_TOGGLE_BIT) ==
5695 hrrq->toggle_bit) { 5695 hrrq->toggle_bit) {
5696 if (!blk_iopoll_sched_prep(&hrrq->iopoll)) 5696 if (!irq_poll_sched_prep(&hrrq->iopoll))
5697 blk_iopoll_sched(&hrrq->iopoll); 5697 irq_poll_sched(&hrrq->iopoll);
5698 spin_unlock_irqrestore(hrrq->lock, hrrq_flags); 5698 spin_unlock_irqrestore(hrrq->lock, hrrq_flags);
5699 return IRQ_HANDLED; 5699 return IRQ_HANDLED;
5700 } 5700 }
@@ -10405,9 +10405,9 @@ static int ipr_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id)
10405 10405
10406 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) { 10406 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
10407 for (i = 1; i < ioa_cfg->hrrq_num; i++) { 10407 for (i = 1; i < ioa_cfg->hrrq_num; i++) {
10408 blk_iopoll_init(&ioa_cfg->hrrq[i].iopoll, 10408 irq_poll_init(&ioa_cfg->hrrq[i].iopoll,
10409 ioa_cfg->iopoll_weight, ipr_iopoll); 10409 ioa_cfg->iopoll_weight, ipr_iopoll);
10410 blk_iopoll_enable(&ioa_cfg->hrrq[i].iopoll); 10410 irq_poll_enable(&ioa_cfg->hrrq[i].iopoll);
10411 } 10411 }
10412 } 10412 }
10413 10413
@@ -10436,7 +10436,7 @@ static void ipr_shutdown(struct pci_dev *pdev)
10436 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) { 10436 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
10437 ioa_cfg->iopoll_weight = 0; 10437 ioa_cfg->iopoll_weight = 0;
10438 for (i = 1; i < ioa_cfg->hrrq_num; i++) 10438 for (i = 1; i < ioa_cfg->hrrq_num; i++)
10439 blk_iopoll_disable(&ioa_cfg->hrrq[i].iopoll); 10439 irq_poll_disable(&ioa_cfg->hrrq[i].iopoll);
10440 } 10440 }
10441 10441
10442 while (ioa_cfg->in_reset_reload) { 10442 while (ioa_cfg->in_reset_reload) {
diff --git a/drivers/scsi/ipr.h b/drivers/scsi/ipr.h
index a34c7a5a995e..56c57068300a 100644
--- a/drivers/scsi/ipr.h
+++ b/drivers/scsi/ipr.h
@@ -32,7 +32,7 @@
32#include <linux/libata.h> 32#include <linux/libata.h>
33#include <linux/list.h> 33#include <linux/list.h>
34#include <linux/kref.h> 34#include <linux/kref.h>
35#include <linux/blk-iopoll.h> 35#include <linux/irq_poll.h>
36#include <scsi/scsi.h> 36#include <scsi/scsi.h>
37#include <scsi/scsi_cmnd.h> 37#include <scsi/scsi_cmnd.h>
38 38
@@ -517,7 +517,7 @@ struct ipr_hrr_queue {
517 u8 allow_cmds:1; 517 u8 allow_cmds:1;
518 u8 removing_ioa:1; 518 u8 removing_ioa:1;
519 519
520 struct blk_iopoll iopoll; 520 struct irq_poll iopoll;
521}; 521};
522 522
523/* Command packet structure */ 523/* Command packet structure */
diff --git a/include/linux/blk-iopoll.h b/include/linux/blk-iopoll.h
deleted file mode 100644
index 77ae77c0b704..000000000000
--- a/include/linux/blk-iopoll.h
+++ /dev/null
@@ -1,46 +0,0 @@
1#ifndef BLK_IOPOLL_H
2#define BLK_IOPOLL_H
3
4struct blk_iopoll;
5typedef int (blk_iopoll_fn)(struct blk_iopoll *, int);
6
7struct blk_iopoll {
8 struct list_head list;
9 unsigned long state;
10 unsigned long data;
11 int weight;
12 int max;
13 blk_iopoll_fn *poll;
14};
15
16enum {
17 IOPOLL_F_SCHED = 0,
18 IOPOLL_F_DISABLE = 1,
19};
20
21/*
22 * Returns 0 if we successfully set the IOPOLL_F_SCHED bit, indicating
23 * that we were the first to acquire this iop for scheduling. If this iop
24 * is currently disabled, return "failure".
25 */
26static inline int blk_iopoll_sched_prep(struct blk_iopoll *iop)
27{
28 if (!test_bit(IOPOLL_F_DISABLE, &iop->state))
29 return test_and_set_bit(IOPOLL_F_SCHED, &iop->state);
30
31 return 1;
32}
33
34static inline int blk_iopoll_disable_pending(struct blk_iopoll *iop)
35{
36 return test_bit(IOPOLL_F_DISABLE, &iop->state);
37}
38
39extern void blk_iopoll_sched(struct blk_iopoll *);
40extern void blk_iopoll_init(struct blk_iopoll *, int, blk_iopoll_fn *);
41extern void blk_iopoll_complete(struct blk_iopoll *);
42extern void __blk_iopoll_complete(struct blk_iopoll *);
43extern void blk_iopoll_enable(struct blk_iopoll *);
44extern void blk_iopoll_disable(struct blk_iopoll *);
45
46#endif
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index ad16809c8596..7ff98c23199a 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -412,7 +412,7 @@ enum
412 NET_TX_SOFTIRQ, 412 NET_TX_SOFTIRQ,
413 NET_RX_SOFTIRQ, 413 NET_RX_SOFTIRQ,
414 BLOCK_SOFTIRQ, 414 BLOCK_SOFTIRQ,
415 BLOCK_IOPOLL_SOFTIRQ, 415 IRQ_POLL_SOFTIRQ,
416 TASKLET_SOFTIRQ, 416 TASKLET_SOFTIRQ,
417 SCHED_SOFTIRQ, 417 SCHED_SOFTIRQ,
418 HRTIMER_SOFTIRQ, /* Unused, but kept as tools rely on the 418 HRTIMER_SOFTIRQ, /* Unused, but kept as tools rely on the
diff --git a/include/linux/irq_poll.h b/include/linux/irq_poll.h
new file mode 100644
index 000000000000..50c39dcd2cba
--- /dev/null
+++ b/include/linux/irq_poll.h
@@ -0,0 +1,46 @@
1#ifndef IRQ_POLL_H
2#define IRQ_POLL_H
3
4struct irq_poll;
5typedef int (irq_poll_fn)(struct irq_poll *, int);
6
7struct irq_poll {
8 struct list_head list;
9 unsigned long state;
10 unsigned long data;
11 int weight;
12 int max;
13 irq_poll_fn *poll;
14};
15
16enum {
17 IRQ_POLL_F_SCHED = 0,
18 IRQ_POLL_F_DISABLE = 1,
19};
20
21/*
22 * Returns 0 if we successfully set the IRQ_POLL_F_SCHED bit, indicating
23 * that we were the first to acquire this iop for scheduling. If this iop
24 * is currently disabled, return "failure".
25 */
26static inline int irq_poll_sched_prep(struct irq_poll *iop)
27{
28 if (!test_bit(IRQ_POLL_F_DISABLE, &iop->state))
29 return test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state);
30
31 return 1;
32}
33
34static inline int irq_poll_disable_pending(struct irq_poll *iop)
35{
36 return test_bit(IRQ_POLL_F_DISABLE, &iop->state);
37}
38
39extern void irq_poll_sched(struct irq_poll *);
40extern void irq_poll_init(struct irq_poll *, int, irq_poll_fn *);
41extern void irq_poll_complete(struct irq_poll *);
42extern void __irq_poll_complete(struct irq_poll *);
43extern void irq_poll_enable(struct irq_poll *);
44extern void irq_poll_disable(struct irq_poll *);
45
46#endif
diff --git a/include/trace/events/irq.h b/include/trace/events/irq.h
index ff8f6c091a15..f95f25e786ef 100644
--- a/include/trace/events/irq.h
+++ b/include/trace/events/irq.h
@@ -15,7 +15,7 @@ struct softirq_action;
15 softirq_name(NET_TX) \ 15 softirq_name(NET_TX) \
16 softirq_name(NET_RX) \ 16 softirq_name(NET_RX) \
17 softirq_name(BLOCK) \ 17 softirq_name(BLOCK) \
18 softirq_name(BLOCK_IOPOLL) \ 18 softirq_name(IRQ_POLL) \
19 softirq_name(TASKLET) \ 19 softirq_name(TASKLET) \
20 softirq_name(SCHED) \ 20 softirq_name(SCHED) \
21 softirq_name(HRTIMER) \ 21 softirq_name(HRTIMER) \
diff --git a/lib/Kconfig b/lib/Kconfig
index f0df318104e7..e00e1960260a 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -475,6 +475,11 @@ config DDR
475 information. This data is useful for drivers handling 475 information. This data is useful for drivers handling
476 DDR SDRAM controllers. 476 DDR SDRAM controllers.
477 477
478config IRQ_POLL
479 bool "IRQ polling library"
480 help
481 Helper library to poll interrupt mitigation using polling.
482
478config MPILIB 483config MPILIB
479 tristate 484 tristate
480 select CLZ_TAB 485 select CLZ_TAB
diff --git a/lib/Makefile b/lib/Makefile
index 7f1de26613d2..1478ae256561 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -164,6 +164,7 @@ obj-$(CONFIG_GENERIC_NET_UTILS) += net_utils.o
164 164
165obj-$(CONFIG_SG_SPLIT) += sg_split.o 165obj-$(CONFIG_SG_SPLIT) += sg_split.o
166obj-$(CONFIG_STMP_DEVICE) += stmp_device.o 166obj-$(CONFIG_STMP_DEVICE) += stmp_device.o
167obj-$(CONFIG_IRQ_POLL) += irq_poll.o
167 168
168libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \ 169libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \
169 fdt_empty_tree.o 170 fdt_empty_tree.o
diff --git a/block/blk-iopoll.c b/lib/irq_poll.c
index 0736729d6494..e6fd1dc0908b 100644
--- a/block/blk-iopoll.c
+++ b/lib/irq_poll.c
@@ -6,84 +6,81 @@
6#include <linux/module.h> 6#include <linux/module.h>
7#include <linux/init.h> 7#include <linux/init.h>
8#include <linux/bio.h> 8#include <linux/bio.h>
9#include <linux/blkdev.h>
10#include <linux/interrupt.h> 9#include <linux/interrupt.h>
11#include <linux/cpu.h> 10#include <linux/cpu.h>
12#include <linux/blk-iopoll.h> 11#include <linux/irq_poll.h>
13#include <linux/delay.h> 12#include <linux/delay.h>
14 13
15#include "blk.h" 14static unsigned int irq_poll_budget __read_mostly = 256;
16
17static unsigned int blk_iopoll_budget __read_mostly = 256;
18 15
19static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll); 16static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll);
20 17
21/** 18/**
22 * blk_iopoll_sched - Schedule a run of the iopoll handler 19 * irq_poll_sched - Schedule a run of the iopoll handler
23 * @iop: The parent iopoll structure 20 * @iop: The parent iopoll structure
24 * 21 *
25 * Description: 22 * Description:
26 * Add this blk_iopoll structure to the pending poll list and trigger the 23 * Add this irq_poll structure to the pending poll list and trigger the
27 * raise of the blk iopoll softirq. The driver must already have gotten a 24 * raise of the blk iopoll softirq. The driver must already have gotten a
28 * successful return from blk_iopoll_sched_prep() before calling this. 25 * successful return from irq_poll_sched_prep() before calling this.
29 **/ 26 **/
30void blk_iopoll_sched(struct blk_iopoll *iop) 27void irq_poll_sched(struct irq_poll *iop)
31{ 28{
32 unsigned long flags; 29 unsigned long flags;
33 30
34 local_irq_save(flags); 31 local_irq_save(flags);
35 list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll)); 32 list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll));
36 __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); 33 __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
37 local_irq_restore(flags); 34 local_irq_restore(flags);
38} 35}
39EXPORT_SYMBOL(blk_iopoll_sched); 36EXPORT_SYMBOL(irq_poll_sched);
40 37
41/** 38/**
42 * __blk_iopoll_complete - Mark this @iop as un-polled again 39 * __irq_poll_complete - Mark this @iop as un-polled again
43 * @iop: The parent iopoll structure 40 * @iop: The parent iopoll structure
44 * 41 *
45 * Description: 42 * Description:
46 * See blk_iopoll_complete(). This function must be called with interrupts 43 * See irq_poll_complete(). This function must be called with interrupts
47 * disabled. 44 * disabled.
48 **/ 45 **/
49void __blk_iopoll_complete(struct blk_iopoll *iop) 46void __irq_poll_complete(struct irq_poll *iop)
50{ 47{
51 list_del(&iop->list); 48 list_del(&iop->list);
52 smp_mb__before_atomic(); 49 smp_mb__before_atomic();
53 clear_bit_unlock(IOPOLL_F_SCHED, &iop->state); 50 clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state);
54} 51}
55EXPORT_SYMBOL(__blk_iopoll_complete); 52EXPORT_SYMBOL(__irq_poll_complete);
56 53
57/** 54/**
58 * blk_iopoll_complete - Mark this @iop as un-polled again 55 * irq_poll_complete - Mark this @iop as un-polled again
59 * @iop: The parent iopoll structure 56 * @iop: The parent iopoll structure
60 * 57 *
61 * Description: 58 * Description:
62 * If a driver consumes less than the assigned budget in its run of the 59 * If a driver consumes less than the assigned budget in its run of the
63 * iopoll handler, it'll end the polled mode by calling this function. The 60 * iopoll handler, it'll end the polled mode by calling this function. The
64 * iopoll handler will not be invoked again before blk_iopoll_sched_prep() 61 * iopoll handler will not be invoked again before irq_poll_sched_prep()
65 * is called. 62 * is called.
66 **/ 63 **/
67void blk_iopoll_complete(struct blk_iopoll *iop) 64void irq_poll_complete(struct irq_poll *iop)
68{ 65{
69 unsigned long flags; 66 unsigned long flags;
70 67
71 local_irq_save(flags); 68 local_irq_save(flags);
72 __blk_iopoll_complete(iop); 69 __irq_poll_complete(iop);
73 local_irq_restore(flags); 70 local_irq_restore(flags);
74} 71}
75EXPORT_SYMBOL(blk_iopoll_complete); 72EXPORT_SYMBOL(irq_poll_complete);
76 73
77static void blk_iopoll_softirq(struct softirq_action *h) 74static void irq_poll_softirq(struct softirq_action *h)
78{ 75{
79 struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll); 76 struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll);
80 int rearm = 0, budget = blk_iopoll_budget; 77 int rearm = 0, budget = irq_poll_budget;
81 unsigned long start_time = jiffies; 78 unsigned long start_time = jiffies;
82 79
83 local_irq_disable(); 80 local_irq_disable();
84 81
85 while (!list_empty(list)) { 82 while (!list_empty(list)) {
86 struct blk_iopoll *iop; 83 struct irq_poll *iop;
87 int work, weight; 84 int work, weight;
88 85
89 /* 86 /*
@@ -101,11 +98,11 @@ static void blk_iopoll_softirq(struct softirq_action *h)
101 * entries to the tail of this list, and only ->poll() 98 * entries to the tail of this list, and only ->poll()
102 * calls can remove this head entry from the list. 99 * calls can remove this head entry from the list.
103 */ 100 */
104 iop = list_entry(list->next, struct blk_iopoll, list); 101 iop = list_entry(list->next, struct irq_poll, list);
105 102
106 weight = iop->weight; 103 weight = iop->weight;
107 work = 0; 104 work = 0;
108 if (test_bit(IOPOLL_F_SCHED, &iop->state)) 105 if (test_bit(IRQ_POLL_F_SCHED, &iop->state))
109 work = iop->poll(iop, weight); 106 work = iop->poll(iop, weight);
110 107
111 budget -= work; 108 budget -= work;
@@ -121,72 +118,72 @@ static void blk_iopoll_softirq(struct softirq_action *h)
121 * move the instance around on the list at-will. 118 * move the instance around on the list at-will.
122 */ 119 */
123 if (work >= weight) { 120 if (work >= weight) {
124 if (blk_iopoll_disable_pending(iop)) 121 if (irq_poll_disable_pending(iop))
125 __blk_iopoll_complete(iop); 122 __irq_poll_complete(iop);
126 else 123 else
127 list_move_tail(&iop->list, list); 124 list_move_tail(&iop->list, list);
128 } 125 }
129 } 126 }
130 127
131 if (rearm) 128 if (rearm)
132 __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); 129 __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
133 130
134 local_irq_enable(); 131 local_irq_enable();
135} 132}
136 133
137/** 134/**
138 * blk_iopoll_disable - Disable iopoll on this @iop 135 * irq_poll_disable - Disable iopoll on this @iop
139 * @iop: The parent iopoll structure 136 * @iop: The parent iopoll structure
140 * 137 *
141 * Description: 138 * Description:
142 * Disable io polling and wait for any pending callbacks to have completed. 139 * Disable io polling and wait for any pending callbacks to have completed.
143 **/ 140 **/
144void blk_iopoll_disable(struct blk_iopoll *iop) 141void irq_poll_disable(struct irq_poll *iop)
145{ 142{
146 set_bit(IOPOLL_F_DISABLE, &iop->state); 143 set_bit(IRQ_POLL_F_DISABLE, &iop->state);
147 while (test_and_set_bit(IOPOLL_F_SCHED, &iop->state)) 144 while (test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state))
148 msleep(1); 145 msleep(1);
149 clear_bit(IOPOLL_F_DISABLE, &iop->state); 146 clear_bit(IRQ_POLL_F_DISABLE, &iop->state);
150} 147}
151EXPORT_SYMBOL(blk_iopoll_disable); 148EXPORT_SYMBOL(irq_poll_disable);
152 149
153/** 150/**
154 * blk_iopoll_enable - Enable iopoll on this @iop 151 * irq_poll_enable - Enable iopoll on this @iop
155 * @iop: The parent iopoll structure 152 * @iop: The parent iopoll structure
156 * 153 *
157 * Description: 154 * Description:
158 * Enable iopoll on this @iop. Note that the handler run will not be 155 * Enable iopoll on this @iop. Note that the handler run will not be
159 * scheduled, it will only mark it as active. 156 * scheduled, it will only mark it as active.
160 **/ 157 **/
161void blk_iopoll_enable(struct blk_iopoll *iop) 158void irq_poll_enable(struct irq_poll *iop)
162{ 159{
163 BUG_ON(!test_bit(IOPOLL_F_SCHED, &iop->state)); 160 BUG_ON(!test_bit(IRQ_POLL_F_SCHED, &iop->state));
164 smp_mb__before_atomic(); 161 smp_mb__before_atomic();
165 clear_bit_unlock(IOPOLL_F_SCHED, &iop->state); 162 clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state);
166} 163}
167EXPORT_SYMBOL(blk_iopoll_enable); 164EXPORT_SYMBOL(irq_poll_enable);
168 165
169/** 166/**
170 * blk_iopoll_init - Initialize this @iop 167 * irq_poll_init - Initialize this @iop
171 * @iop: The parent iopoll structure 168 * @iop: The parent iopoll structure
172 * @weight: The default weight (or command completion budget) 169 * @weight: The default weight (or command completion budget)
173 * @poll_fn: The handler to invoke 170 * @poll_fn: The handler to invoke
174 * 171 *
175 * Description: 172 * Description:
176 * Initialize this blk_iopoll structure. Before being actively used, the 173 * Initialize this irq_poll structure. Before being actively used, the
177 * driver must call blk_iopoll_enable(). 174 * driver must call irq_poll_enable().
178 **/ 175 **/
179void blk_iopoll_init(struct blk_iopoll *iop, int weight, blk_iopoll_fn *poll_fn) 176void irq_poll_init(struct irq_poll *iop, int weight, irq_poll_fn *poll_fn)
180{ 177{
181 memset(iop, 0, sizeof(*iop)); 178 memset(iop, 0, sizeof(*iop));
182 INIT_LIST_HEAD(&iop->list); 179 INIT_LIST_HEAD(&iop->list);
183 iop->weight = weight; 180 iop->weight = weight;
184 iop->poll = poll_fn; 181 iop->poll = poll_fn;
185 set_bit(IOPOLL_F_SCHED, &iop->state); 182 set_bit(IRQ_POLL_F_SCHED, &iop->state);
186} 183}
187EXPORT_SYMBOL(blk_iopoll_init); 184EXPORT_SYMBOL(irq_poll_init);
188 185
189static int blk_iopoll_cpu_notify(struct notifier_block *self, 186static int irq_poll_cpu_notify(struct notifier_block *self,
190 unsigned long action, void *hcpu) 187 unsigned long action, void *hcpu)
191{ 188{
192 /* 189 /*
@@ -199,26 +196,26 @@ static int blk_iopoll_cpu_notify(struct notifier_block *self,
199 local_irq_disable(); 196 local_irq_disable();
200 list_splice_init(&per_cpu(blk_cpu_iopoll, cpu), 197 list_splice_init(&per_cpu(blk_cpu_iopoll, cpu),
201 this_cpu_ptr(&blk_cpu_iopoll)); 198 this_cpu_ptr(&blk_cpu_iopoll));
202 __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); 199 __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
203 local_irq_enable(); 200 local_irq_enable();
204 } 201 }
205 202
206 return NOTIFY_OK; 203 return NOTIFY_OK;
207} 204}
208 205
209static struct notifier_block blk_iopoll_cpu_notifier = { 206static struct notifier_block irq_poll_cpu_notifier = {
210 .notifier_call = blk_iopoll_cpu_notify, 207 .notifier_call = irq_poll_cpu_notify,
211}; 208};
212 209
213static __init int blk_iopoll_setup(void) 210static __init int irq_poll_setup(void)
214{ 211{
215 int i; 212 int i;
216 213
217 for_each_possible_cpu(i) 214 for_each_possible_cpu(i)
218 INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i)); 215 INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i));
219 216
220 open_softirq(BLOCK_IOPOLL_SOFTIRQ, blk_iopoll_softirq); 217 open_softirq(IRQ_POLL_SOFTIRQ, irq_poll_softirq);
221 register_hotcpu_notifier(&blk_iopoll_cpu_notifier); 218 register_hotcpu_notifier(&irq_poll_cpu_notifier);
222 return 0; 219 return 0;
223} 220}
224subsys_initcall(blk_iopoll_setup); 221subsys_initcall(irq_poll_setup);
diff --git a/tools/lib/traceevent/event-parse.c b/tools/lib/traceevent/event-parse.c
index 2a912df6771b..af5a31661086 100644
--- a/tools/lib/traceevent/event-parse.c
+++ b/tools/lib/traceevent/event-parse.c
@@ -3746,7 +3746,7 @@ static const struct flag flags[] = {
3746 { "NET_TX_SOFTIRQ", 2 }, 3746 { "NET_TX_SOFTIRQ", 2 },
3747 { "NET_RX_SOFTIRQ", 3 }, 3747 { "NET_RX_SOFTIRQ", 3 },
3748 { "BLOCK_SOFTIRQ", 4 }, 3748 { "BLOCK_SOFTIRQ", 4 },
3749 { "BLOCK_IOPOLL_SOFTIRQ", 5 }, 3749 { "IRQ_POLL_SOFTIRQ", 5 },
3750 { "TASKLET_SOFTIRQ", 6 }, 3750 { "TASKLET_SOFTIRQ", 6 },
3751 { "SCHED_SOFTIRQ", 7 }, 3751 { "SCHED_SOFTIRQ", 7 },
3752 { "HRTIMER_SOFTIRQ", 8 }, 3752 { "HRTIMER_SOFTIRQ", 8 },
diff --git a/tools/perf/util/trace-event-parse.c b/tools/perf/util/trace-event-parse.c
index 8ff7d620d942..33b52eaa39db 100644
--- a/tools/perf/util/trace-event-parse.c
+++ b/tools/perf/util/trace-event-parse.c
@@ -209,7 +209,7 @@ static const struct flag flags[] = {
209 { "NET_TX_SOFTIRQ", 2 }, 209 { "NET_TX_SOFTIRQ", 2 },
210 { "NET_RX_SOFTIRQ", 3 }, 210 { "NET_RX_SOFTIRQ", 3 },
211 { "BLOCK_SOFTIRQ", 4 }, 211 { "BLOCK_SOFTIRQ", 4 },
212 { "BLOCK_IOPOLL_SOFTIRQ", 5 }, 212 { "IRQ_POLL_SOFTIRQ", 5 },
213 { "TASKLET_SOFTIRQ", 6 }, 213 { "TASKLET_SOFTIRQ", 6 },
214 { "SCHED_SOFTIRQ", 7 }, 214 { "SCHED_SOFTIRQ", 7 },
215 { "HRTIMER_SOFTIRQ", 8 }, 215 { "HRTIMER_SOFTIRQ", 8 },