IPoIB: P_Key change event handling

This patch causes the network interface to respond to P_Key change
events correctly.  As a result, you'll see a child interface in the
"RUNNING" state (netif_carrier_on()) only when the corresponding P_Key
is configured by the SM.  When SM removes a P_Key, the "RUNNING" state
will be disabled for the corresponding network interface.  To
implement this, I added IB_EVENT_PKEY_CHANGE event handling.  To
prevent flushing the device before the device is open by the "delay
open" mechanism, I added an additional device flag called
IPOIB_FLAG_INITIALIZED.

This also prevents the child network interface from trying to join to
multicast groups until the PKEY is configured.  We used to get error
messages like:

    ib0.f2f2: couldn't attach QP to multicast group ff12:401b:f2f2:0:0:0:ffff:ffff

in this case.  To fix this, I just check IPOIB_FLAG_OPER_UP flag in
ipoib_set_mcast_list().

Signed-off-by: Leonid Arsh <leonida@voltaire.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

4 files changed, 44 insertions, 19 deletions

diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h
index 1251f86ec856..b640107fb732 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -72,13 +72,14 @@ enum {
         IPOIB_MAX_MCAST_QUEUE     = 3,
 
         IPOIB_FLAG_OPER_UP        = 0,
-        IPOIB_FLAG_ADMIN_UP       = 1,
-        IPOIB_PKEY_ASSIGNED       = 2,
-        IPOIB_PKEY_STOP           = 3,
-        IPOIB_FLAG_SUBINTERFACE   = 4,
-        IPOIB_MCAST_RUN           = 5,
-        IPOIB_STOP_REAPER         = 6,
-        IPOIB_MCAST_STARTED       = 7,
+        IPOIB_FLAG_INITIALIZED    = 1,
+        IPOIB_FLAG_ADMIN_UP       = 2,
+        IPOIB_PKEY_ASSIGNED       = 3,
+        IPOIB_PKEY_STOP           = 4,
+        IPOIB_FLAG_SUBINTERFACE   = 5,
+        IPOIB_MCAST_RUN           = 6,
+        IPOIB_STOP_REAPER         = 7,
+        IPOIB_MCAST_STARTED       = 8,
 
         IPOIB_MAX_BACKOFF_SECONDS = 16,
 
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
index f2be500f62c8..ed65202878d8 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
@@ -423,13 +423,33 @@ int ipoib_ib_dev_open(struct net_device *dev)
         clear_bit(IPOIB_STOP_REAPER, &priv->flags);
         queue_delayed_work(ipoib_workqueue, &priv->ah_reap_task, HZ);
 
+        set_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
+
         return 0;
 }
 
+static void ipoib_pkey_dev_check_presence(struct net_device *dev)
+{
+        struct ipoib_dev_priv *priv = netdev_priv(dev);
+        u16 pkey_index = 0;
+
+        if (ib_find_cached_pkey(priv->ca, priv->port, priv->pkey, &pkey_index))
+                clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
+        else
+                set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
+}
+
 int ipoib_ib_dev_up(struct net_device *dev)
 {
         struct ipoib_dev_priv *priv = netdev_priv(dev);
 
+        ipoib_pkey_dev_check_presence(dev);
+
+        if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
+                ipoib_dbg(priv, "PKEY is not assigned.\n");
+                return 0;
+        }
+
         set_bit(IPOIB_FLAG_OPER_UP, &priv->flags);
 
         return ipoib_mcast_start_thread(dev);
@@ -483,6 +503,8 @@ int ipoib_ib_dev_stop(struct net_device *dev)
         struct ipoib_tx_buf *tx_req;
         int i;
 
+        clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags);
+
         /*
          * Move our QP to the error state and then reinitialize in
          * when all work requests have completed or have been flushed.
@@ -587,8 +609,15 @@ void ipoib_ib_dev_flush(void *_dev)
         struct net_device *dev = (struct net_device *)_dev;
         struct ipoib_dev_priv *priv = netdev_priv(dev), *cpriv;
 
-        if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
+        if (!test_bit(IPOIB_FLAG_INITIALIZED, &priv->flags) ) {
+                ipoib_dbg(priv, "Not flushing - IPOIB_FLAG_INITIALIZED not set.\n");
+                return;
+        }
+
+        if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) {
+                ipoib_dbg(priv, "Not flushing - IPOIB_FLAG_ADMIN_UP not set.\n");
                 return;
+        }
 
         ipoib_dbg(priv, "flushing\n");
 
@@ -632,17 +661,6 @@ void ipoib_ib_dev_cleanup(struct net_device *dev)
  * change async notification is available.
  */
 
-static void ipoib_pkey_dev_check_presence(struct net_device *dev)
-{
-        struct ipoib_dev_priv *priv = netdev_priv(dev);
-        u16 pkey_index = 0;
-
-        if (ib_find_cached_pkey(priv->ca, priv->port, priv->pkey, &pkey_index))
-                clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
-        else
-                set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
-}
-
 void ipoib_pkey_poll(void *dev_ptr)
 {
         struct net_device *dev = dev_ptr;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 37da8d3dc388..53a32f65788d 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -736,6 +736,11 @@ static void ipoib_set_mcast_list(struct net_device *dev)
 {
         struct ipoib_dev_priv *priv = netdev_priv(dev);
 
+        if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) {
+                ipoib_dbg(priv, "IPOIB_FLAG_OPER_UP not set");
+                return;
+        }
+
         queue_work(ipoib_workqueue, &priv->restart_task);
 }
 
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
index a35b798b8128..5f0388027b25 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
@@ -252,6 +252,7 @@ void ipoib_event(struct ib_event_handler *handler,
                 container_of(handler, struct ipoib_dev_priv, event_handler);
 
         if (record->event == IB_EVENT_PORT_ERR    ||
+            record->event == IB_EVENT_PKEY_CHANGE ||
             record->event == IB_EVENT_PORT_ACTIVE ||
             record->event == IB_EVENT_LID_CHANGE  ||
             record->event == IB_EVENT_SM_CHANGE) {