1 files changed, 242 insertions, 109 deletions

diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c
index 00520f9a7a8e..570d4da10696 100644
--- a/drivers/s390/cio/qdio_main.c
+++ b/drivers/s390/cio/qdio_main.c
@@ -14,6 +14,7 @@
 #include <linux/timer.h>
 #include <linux/delay.h>
 #include <linux/gfp.h>
+#include <linux/kernel_stat.h>
 #include <asm/atomic.h>
 #include <asm/debug.h>
 #include <asm/qdio.h>
@@ -29,11 +30,12 @@ MODULE_AUTHOR("Utz Bacher <utz.bacher@de.ibm.com>,"\
 MODULE_DESCRIPTION("QDIO base support");
 MODULE_LICENSE("GPL");
 
-static inline int do_siga_sync(struct subchannel_id schid,
-                               unsigned int out_mask, unsigned int in_mask)
+static inline int do_siga_sync(unsigned long schid,
+                               unsigned int out_mask, unsigned int in_mask,
+                               unsigned int fc)
 {
-        register unsigned long __fc asm ("0") = 2;
-        register struct subchannel_id __schid asm ("1") = schid;
+        register unsigned long __fc asm ("0") = fc;
+        register unsigned long __schid asm ("1") = schid;
         register unsigned long out asm ("2") = out_mask;
         register unsigned long in asm ("3") = in_mask;
         int cc;
@@ -47,10 +49,11 @@ static inline int do_siga_sync(struct subchannel_id schid,
         return cc;
 }
 
-static inline int do_siga_input(struct subchannel_id schid, unsigned int mask)
+static inline int do_siga_input(unsigned long schid, unsigned int mask,
+                                unsigned int fc)
 {
-        register unsigned long __fc asm ("0") = 1;
-        register struct subchannel_id __schid asm ("1") = schid;
+        register unsigned long __fc asm ("0") = fc;
+        register unsigned long __schid asm ("1") = schid;
         register unsigned long __mask asm ("2") = mask;
         int cc;
 
@@ -279,16 +282,20 @@ void qdio_init_buf_states(struct qdio_irq *irq_ptr)
 static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output,
                           unsigned int input)
 {
+        unsigned long schid = *((u32 *) &q->irq_ptr->schid);
+        unsigned int fc = QDIO_SIGA_SYNC;
         int cc;
 
-        if (!need_siga_sync(q))
-                return 0;
-
         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);
         qperf_inc(q, siga_sync);
 
-        cc = do_siga_sync(q->irq_ptr->schid, output, input);
-        if (cc)
+        if (is_qebsm(q)) {
+                schid = q->irq_ptr->sch_token;
+                fc |= QDIO_SIGA_QEBSM_FLAG;
+        }
+
+        cc = do_siga_sync(schid, output, input, fc);
+        if (unlikely(cc))
                 DBF_ERROR("%4x SIGA-S:%2d", SCH_NO(q), cc);
         return cc;
 }
@@ -301,38 +308,22 @@ static inline int qdio_siga_sync_q(struct qdio_q *q)
                 return qdio_siga_sync(q, q->mask, 0);
 }
 
-static inline int qdio_siga_sync_out(struct qdio_q *q)
-{
-        return qdio_siga_sync(q, ~0U, 0);
-}
-
-static inline int qdio_siga_sync_all(struct qdio_q *q)
-{
-        return qdio_siga_sync(q, ~0U, ~0U);
-}
-
 static int qdio_siga_output(struct qdio_q *q, unsigned int *busy_bit)
 {
-        unsigned long schid;
-        unsigned int fc = 0;
+        unsigned long schid = *((u32 *) &q->irq_ptr->schid);
+        unsigned int fc = QDIO_SIGA_WRITE;
         u64 start_time = 0;
         int cc;
 
-        if (q->u.out.use_enh_siga)
-                fc = 3;
-
         if (is_qebsm(q)) {
                 schid = q->irq_ptr->sch_token;
-                fc |= 0x80;
+                fc |= QDIO_SIGA_QEBSM_FLAG;
         }
-        else
-                schid = *((u32 *)&q->irq_ptr->schid);
-
 again:
         cc = do_siga_output(schid, q->mask, busy_bit, fc);
 
         /* hipersocket busy condition */
-        if (*busy_bit) {
+        if (unlikely(*busy_bit)) {
                 WARN_ON(queue_type(q) != QDIO_IQDIO_QFMT || cc != 2);
 
                 if (!start_time) {
@@ -347,32 +338,41 @@ again:
 
 static inline int qdio_siga_input(struct qdio_q *q)
 {
+        unsigned long schid = *((u32 *) &q->irq_ptr->schid);
+        unsigned int fc = QDIO_SIGA_READ;
         int cc;
 
         DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);
         qperf_inc(q, siga_read);
 
-        cc = do_siga_input(q->irq_ptr->schid, q->mask);
-        if (cc)
+        if (is_qebsm(q)) {
+                schid = q->irq_ptr->sch_token;
+                fc |= QDIO_SIGA_QEBSM_FLAG;
+        }
+
+        cc = do_siga_input(schid, q->mask, fc);
+        if (unlikely(cc))
                 DBF_ERROR("%4x SIGA-R:%2d", SCH_NO(q), cc);
         return cc;
 }
 
-static inline void qdio_sync_after_thinint(struct qdio_q *q)
+#define qdio_siga_sync_out(q) qdio_siga_sync(q, ~0U, 0)
+#define qdio_siga_sync_all(q) qdio_siga_sync(q, ~0U, ~0U)
+
+static inline void qdio_sync_queues(struct qdio_q *q)
 {
-        if (pci_out_supported(q)) {
-                if (need_siga_sync_thinint(q))
-                        qdio_siga_sync_all(q);
-                else if (need_siga_sync_out_thinint(q))
-                        qdio_siga_sync_out(q);
-        } else
+        /* PCI capable outbound queues will also be scanned so sync them too */
+        if (pci_out_supported(q))
+                qdio_siga_sync_all(q);
+        else
                 qdio_siga_sync_q(q);
 }
 
 int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr,
                         unsigned char *state)
 {
-        qdio_siga_sync_q(q);
+        if (need_siga_sync(q))
+                qdio_siga_sync_q(q);
         return get_buf_states(q, bufnr, state, 1, 0);
 }
 
@@ -407,13 +407,16 @@ static inline void account_sbals(struct qdio_q *q, int count)
         q->q_stats.nr_sbals[pos]++;
 }
 
-static void announce_buffer_error(struct qdio_q *q, int count)
+static void process_buffer_error(struct qdio_q *q, int count)
 {
+        unsigned char state = (q->is_input_q) ? SLSB_P_INPUT_NOT_INIT :
+                                        SLSB_P_OUTPUT_NOT_INIT;
+
         q->qdio_error |= QDIO_ERROR_SLSB_STATE;
 
         /* special handling for no target buffer empty */
         if ((!q->is_input_q &&
-            (q->sbal[q->first_to_check]->element[15].flags & 0xff) == 0x10)) {
+            (q->sbal[q->first_to_check]->element[15].sflags) == 0x10)) {
                 qperf_inc(q, target_full);
                 DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x",
                               q->first_to_check);
@@ -424,8 +427,14 @@ static void announce_buffer_error(struct qdio_q *q, int count)
         DBF_ERROR((q->is_input_q) ? "IN:%2d" : "OUT:%2d", q->nr);
         DBF_ERROR("FTC:%3d C:%3d", q->first_to_check, count);
         DBF_ERROR("F14:%2x F15:%2x",
-                  q->sbal[q->first_to_check]->element[14].flags & 0xff,
-                  q->sbal[q->first_to_check]->element[15].flags & 0xff);
+                  q->sbal[q->first_to_check]->element[14].sflags,
+                  q->sbal[q->first_to_check]->element[15].sflags);
+
+        /*
+         * Interrupts may be avoided as long as the error is present
+         * so change the buffer state immediately to avoid starvation.
+         */
+        set_buf_states(q, q->first_to_check, state, count);
 }
 
 static inline void inbound_primed(struct qdio_q *q, int count)
@@ -476,7 +485,7 @@ static inline void inbound_primed(struct qdio_q *q, int count)
 static int get_inbound_buffer_frontier(struct qdio_q *q)
 {
         int count, stop;
-        unsigned char state;
+        unsigned char state = 0;
 
         /*
          * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
@@ -506,8 +515,7 @@ static int get_inbound_buffer_frontier(struct qdio_q *q)
                         account_sbals(q, count);
                 break;
         case SLSB_P_INPUT_ERROR:
-                announce_buffer_error(q, count);
-                /* process the buffer, the upper layer will take care of it */
+                process_buffer_error(q, count);
                 q->first_to_check = add_buf(q->first_to_check, count);
                 atomic_sub(count, &q->nr_buf_used);
                 if (q->irq_ptr->perf_stat_enabled)
@@ -549,7 +557,8 @@ static inline int qdio_inbound_q_done(struct qdio_q *q)
         if (!atomic_read(&q->nr_buf_used))
                 return 1;
 
-        qdio_siga_sync_q(q);
+        if (need_siga_sync(q))
+                qdio_siga_sync_q(q);
         get_buf_state(q, q->first_to_check, &state, 0);
 
         if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR)
@@ -642,11 +651,14 @@ void qdio_inbound_processing(unsigned long data)
 static int get_outbound_buffer_frontier(struct qdio_q *q)
 {
         int count, stop;
-        unsigned char state;
+        unsigned char state = 0;
 
-        if (((queue_type(q) != QDIO_IQDIO_QFMT) && !pci_out_supported(q)) ||
-            (queue_type(q) == QDIO_IQDIO_QFMT && multicast_outbound(q)))
-                qdio_siga_sync_q(q);
+        if (need_siga_sync(q))
+                if (((queue_type(q) != QDIO_IQDIO_QFMT) &&
+                    !pci_out_supported(q)) ||
+                    (queue_type(q) == QDIO_IQDIO_QFMT &&
+                    multicast_outbound(q)))
+                        qdio_siga_sync_q(q);
 
         /*
          * Don't check 128 buffers, as otherwise qdio_inbound_q_moved
@@ -673,8 +685,7 @@ static int get_outbound_buffer_frontier(struct qdio_q *q)
                         account_sbals(q, count);
                 break;
         case SLSB_P_OUTPUT_ERROR:
-                announce_buffer_error(q, count);
-                /* process the buffer, the upper layer will take care of it */
+                process_buffer_error(q, count);
                 q->first_to_check = add_buf(q->first_to_check, count);
                 atomic_sub(count, &q->nr_buf_used);
                 if (q->irq_ptr->perf_stat_enabled)
@@ -818,7 +829,8 @@ static inline void qdio_check_outbound_after_thinint(struct qdio_q *q)
 static void __tiqdio_inbound_processing(struct qdio_q *q)
 {
         qperf_inc(q, tasklet_inbound);
-        qdio_sync_after_thinint(q);
+        if (need_siga_sync(q) && need_siga_sync_after_ai(q))
+                qdio_sync_queues(q);
 
         /*
          * The interrupt could be caused by a PCI request. Check the
@@ -884,19 +896,28 @@ static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)
         if (unlikely(irq_ptr->state == QDIO_IRQ_STATE_STOPPED))
                 return;
 
-        for_each_input_queue(irq_ptr, q, i)
-                tasklet_schedule(&q->tasklet);
+        for_each_input_queue(irq_ptr, q, i) {
+                if (q->u.in.queue_start_poll) {
+                        /* skip if polling is enabled or already in work */
+                        if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
+                                     &q->u.in.queue_irq_state)) {
+                                qperf_inc(q, int_discarded);
+                                continue;
+                        }
+                        q->u.in.queue_start_poll(q->irq_ptr->cdev, q->nr,
+                                                 q->irq_ptr->int_parm);
+                } else
+                        tasklet_schedule(&q->tasklet);
+        }
 
-        if (!(irq_ptr->qib.ac & QIB_AC_OUTBOUND_PCI_SUPPORTED))
+        if (!pci_out_supported(q))
                 return;
 
         for_each_output_queue(irq_ptr, q, i) {
                 if (qdio_outbound_q_done(q))
                         continue;
-
-                if (!siga_syncs_out_pci(q))
+                if (need_siga_sync(q) && need_siga_sync_out_after_pci(q))
                         qdio_siga_sync_q(q);
-
                 tasklet_schedule(&q->tasklet);
         }
 }
@@ -959,6 +980,7 @@ void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm,
                 return;
         }
 
+        kstat_cpu(smp_processor_id()).irqs[IOINT_QDI]++;
         if (irq_ptr->perf_stat_enabled)
                 irq_ptr->perf_stat.qdio_int++;
 
@@ -1262,7 +1284,6 @@ int qdio_establish(struct qdio_initialize *init_data)
         }
 
         qdio_setup_ssqd_info(irq_ptr);
-        DBF_EVENT("qDmmwc:%2x", irq_ptr->ssqd_desc.mmwc);
         DBF_EVENT("qib ac:%4x", irq_ptr->qib.ac);
 
         /* qebsm is now setup if available, initialize buffer states */
@@ -1425,7 +1446,7 @@ set:
 static int handle_outbound(struct qdio_q *q, unsigned int callflags,
                            int bufnr, int count)
 {
-        unsigned char state;
+        unsigned char state = 0;
         int used, rc = 0;
 
         qperf_inc(q, outbound_call);
@@ -1434,52 +1455,38 @@ static int handle_outbound(struct qdio_q *q, unsigned int callflags,
         used = atomic_add_return(count, &q->nr_buf_used);
         BUG_ON(used > QDIO_MAX_BUFFERS_PER_Q);
 
+        if (used == QDIO_MAX_BUFFERS_PER_Q)
+                qperf_inc(q, outbound_queue_full);
+
         if (callflags & QDIO_FLAG_PCI_OUT) {
                 q->u.out.pci_out_enabled = 1;
                 qperf_inc(q, pci_request_int);
-        }
-        else
+        } else
                 q->u.out.pci_out_enabled = 0;
 
         if (queue_type(q) == QDIO_IQDIO_QFMT) {
-                if (multicast_outbound(q))
+                /* One SIGA-W per buffer required for unicast HiperSockets. */
+                WARN_ON_ONCE(count > 1 && !multicast_outbound(q));
+
+                rc = qdio_kick_outbound_q(q);
+        } else if (need_siga_sync(q)) {
+                rc = qdio_siga_sync_q(q);
+        } else {
+                /* try to fast requeue buffers */
+                get_buf_state(q, prev_buf(bufnr), &state, 0);
+                if (state != SLSB_CU_OUTPUT_PRIMED)
                         rc = qdio_kick_outbound_q(q);
                 else
-                        if ((q->irq_ptr->ssqd_desc.mmwc > 1) &&
-                            (count > 1) &&
-                            (count <= q->irq_ptr->ssqd_desc.mmwc)) {
-                                /* exploit enhanced SIGA */
-                                q->u.out.use_enh_siga = 1;
-                                rc = qdio_kick_outbound_q(q);
-                        } else {
-                                /*
-                                * One siga-w per buffer required for unicast
-                                * HiperSockets.
-                                */
-                                q->u.out.use_enh_siga = 0;
-                                while (count--) {
-                                        rc = qdio_kick_outbound_q(q);
-                                        if (rc)
-                                                goto out;
-                                }
-                        }
-                goto out;
+                        qperf_inc(q, fast_requeue);
         }
 
-        if (need_siga_sync(q)) {
-                qdio_siga_sync_q(q);
-                goto out;
-        }
-
-        /* try to fast requeue buffers */
-        get_buf_state(q, prev_buf(bufnr), &state, 0);
-        if (state != SLSB_CU_OUTPUT_PRIMED)
-                rc = qdio_kick_outbound_q(q);
+        /* in case of SIGA errors we must process the error immediately */
+        if (used >= q->u.out.scan_threshold || rc)
+                tasklet_schedule(&q->tasklet);
         else
-                qperf_inc(q, fast_requeue);
-
-out:
-        tasklet_schedule(&q->tasklet);
+                /* free the SBALs in case of no further traffic */
+                if (!timer_pending(&q->u.out.timer))
+                        mod_timer(&q->u.out.timer, jiffies + HZ);
         return rc;
 }
 
@@ -1508,7 +1515,8 @@ int do_QDIO(struct ccw_device *cdev, unsigned int callflags,
 
         if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)
                 return -EBUSY;
-
+        if (!count)
+                return 0;
         if (callflags & QDIO_FLAG_SYNC_INPUT)
                 return handle_inbound(irq_ptr->input_qs[q_nr],
                                       callflags, bufnr, count);
@@ -1519,30 +1527,155 @@ int do_QDIO(struct ccw_device *cdev, unsigned int callflags,
 }
 EXPORT_SYMBOL_GPL(do_QDIO);
 
+/**
+ * qdio_start_irq - process input buffers
+ * @cdev: associated ccw_device for the qdio subchannel
+ * @nr: input queue number
+ *
+ * Return codes
+ *   0 - success
+ *   1 - irqs not started since new data is available
+ */
+int qdio_start_irq(struct ccw_device *cdev, int nr)
+{
+        struct qdio_q *q;
+        struct qdio_irq *irq_ptr = cdev->private->qdio_data;
+
+        if (!irq_ptr)
+                return -ENODEV;
+        q = irq_ptr->input_qs[nr];
+
+        WARN_ON(queue_irqs_enabled(q));
+
+        if (!shared_ind(q->irq_ptr->dsci))
+                xchg(q->irq_ptr->dsci, 0);
+
+        qdio_stop_polling(q);
+        clear_bit(QDIO_QUEUE_IRQS_DISABLED, &q->u.in.queue_irq_state);
+
+        /*
+         * We need to check again to not lose initiative after
+         * resetting the ACK state.
+         */
+        if (!shared_ind(q->irq_ptr->dsci) && *q->irq_ptr->dsci)
+                goto rescan;
+        if (!qdio_inbound_q_done(q))
+                goto rescan;
+        return 0;
+
+rescan:
+        if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
+                             &q->u.in.queue_irq_state))
+                return 0;
+        else
+                return 1;
+
+}
+EXPORT_SYMBOL(qdio_start_irq);
+
+/**
+ * qdio_get_next_buffers - process input buffers
+ * @cdev: associated ccw_device for the qdio subchannel
+ * @nr: input queue number
+ * @bufnr: first filled buffer number
+ * @error: buffers are in error state
+ *
+ * Return codes
+ *   < 0 - error
+ *   = 0 - no new buffers found
+ *   > 0 - number of processed buffers
+ */
+int qdio_get_next_buffers(struct ccw_device *cdev, int nr, int *bufnr,
+                          int *error)
+{
+        struct qdio_q *q;
+        int start, end;
+        struct qdio_irq *irq_ptr = cdev->private->qdio_data;
+
+        if (!irq_ptr)
+                return -ENODEV;
+        q = irq_ptr->input_qs[nr];
+        WARN_ON(queue_irqs_enabled(q));
+
+        /*
+         * Cannot rely on automatic sync after interrupt since queues may
+         * also be examined without interrupt.
+         */
+        if (need_siga_sync(q))
+                qdio_sync_queues(q);
+
+        /* check the PCI capable outbound queues. */
+        qdio_check_outbound_after_thinint(q);
+
+        if (!qdio_inbound_q_moved(q))
+                return 0;
+
+        /* Note: upper-layer MUST stop processing immediately here ... */
+        if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))
+                return -EIO;
+
+        start = q->first_to_kick;
+        end = q->first_to_check;
+        *bufnr = start;
+        *error = q->qdio_error;
+
+        /* for the next time */
+        q->first_to_kick = end;
+        q->qdio_error = 0;
+        return sub_buf(end, start);
+}
+EXPORT_SYMBOL(qdio_get_next_buffers);
+
+/**
+ * qdio_stop_irq - disable interrupt processing for the device
+ * @cdev: associated ccw_device for the qdio subchannel
+ * @nr: input queue number
+ *
+ * Return codes
+ *   0 - interrupts were already disabled
+ *   1 - interrupts successfully disabled
+ */
+int qdio_stop_irq(struct ccw_device *cdev, int nr)
+{
+        struct qdio_q *q;
+        struct qdio_irq *irq_ptr = cdev->private->qdio_data;
+
+        if (!irq_ptr)
+                return -ENODEV;
+        q = irq_ptr->input_qs[nr];
+
+        if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED,
+                             &q->u.in.queue_irq_state))
+                return 0;
+        else
+                return 1;
+}
+EXPORT_SYMBOL(qdio_stop_irq);
+
 static int __init init_QDIO(void)
 {
         int rc;
 
-        rc = qdio_setup_init();
+        rc = qdio_debug_init();
         if (rc)
                 return rc;
+        rc = qdio_setup_init();
+        if (rc)
+                goto out_debug;
         rc = tiqdio_allocate_memory();
         if (rc)
                 goto out_cache;
-        rc = qdio_debug_init();
-        if (rc)
-                goto out_ti;
         rc = tiqdio_register_thinints();
         if (rc)
-                goto out_debug;
+                goto out_ti;
         return 0;
 
-out_debug:
-        qdio_debug_exit();
 out_ti:
         tiqdio_free_memory();
 out_cache:
         qdio_setup_exit();
+out_debug:
+        qdio_debug_exit();
         return rc;
 }
 
@@ -1550,8 +1683,8 @@ static void __exit exit_QDIO(void)
 {
         tiqdio_unregister_thinints();
         tiqdio_free_memory();
-        qdio_debug_exit();
         qdio_setup_exit();
+        qdio_debug_exit();
 }
 
 module_init(init_QDIO);