[SCSI] fcoe, bnx2fc, libfcoe: SW FCoE and bnx2fc use FCoE Syfs

This patch has the SW FCoE driver and the bnx2fc
driver make use of the new fcoe_sysfs API added
earlier in this patch series.

After this patch a fcoe_ctlr_device is allocated with
private data in this order.

+------------------+   +------------------+
| fcoe_ctlr_device |   | fcoe_ctlr_device |
+------------------+   +------------------+
| fcoe_ctlr        |   | fcoe_ctlr        |
+------------------+   +------------------+
| fcoe_interface   |   | bnx2fc_interface |
+------------------+   +------------------+

libfcoe also takes part in this new model since it
discovers and manages fcoe_fcf instances. The memory
allocation is different for FCFs. I didn't want to
impact libfcoe's fcoe_fcf processing, so this patch
creates fcoe_fcf_device instances for each discovered
fcoe_fcf. The two are paired using a (void * priv)
member of the fcoe_ctlr_device. This allows libfcoe
to continue maintaining its list of fcoe_fcf instances
and simply attaches and detaches them from existing
or new fcoe_fcf_device instances.

Signed-off-by: Robert Love <robert.w.love@intel.com>
Tested-by: Ross Brattain <ross.b.brattain@intel.com>
Signed-off-by: James Bottomley <JBottomley@Parallels.com>

2 files changed, 209 insertions, 21 deletions

diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c
index 4bb42e19d537..fe30b1b65e1d 100644
--- a/drivers/scsi/fcoe/fcoe.c
+++ b/drivers/scsi/fcoe/fcoe.c
@@ -41,6 +41,7 @@
 
 #include <scsi/fc/fc_encaps.h>
 #include <scsi/fc/fc_fip.h>
+#include <scsi/fc/fc_fcoe.h>
 
 #include <scsi/libfc.h>
 #include <scsi/fc_frame.h>
@@ -150,6 +151,21 @@ static int fcoe_vport_create(struct fc_vport *, bool disabled);
 static int fcoe_vport_disable(struct fc_vport *, bool disable);
 static void fcoe_set_vport_symbolic_name(struct fc_vport *);
 static void fcoe_set_port_id(struct fc_lport *, u32, struct fc_frame *);
+static void fcoe_ctlr_get_lesb(struct fcoe_ctlr_device *);
+static void fcoe_fcf_get_vlan_id(struct fcoe_fcf_device *);
+
+static struct fcoe_sysfs_function_template fcoe_sysfs_templ = {
+        .get_fcoe_ctlr_mode = fcoe_ctlr_get_fip_mode,
+        .get_fcoe_ctlr_link_fail = fcoe_ctlr_get_lesb,
+        .get_fcoe_ctlr_vlink_fail = fcoe_ctlr_get_lesb,
+        .get_fcoe_ctlr_miss_fka = fcoe_ctlr_get_lesb,
+        .get_fcoe_ctlr_symb_err = fcoe_ctlr_get_lesb,
+        .get_fcoe_ctlr_err_block = fcoe_ctlr_get_lesb,
+        .get_fcoe_ctlr_fcs_error = fcoe_ctlr_get_lesb,
+
+        .get_fcoe_fcf_selected = fcoe_fcf_get_selected,
+        .get_fcoe_fcf_vlan_id = fcoe_fcf_get_vlan_id,
+};
 
 static struct libfc_function_template fcoe_libfc_fcn_templ = {
         .frame_send = fcoe_xmit,
@@ -366,6 +382,7 @@ static int fcoe_interface_setup(struct fcoe_interface *fcoe,
 static struct fcoe_interface *fcoe_interface_create(struct net_device *netdev,
                                                     enum fip_state fip_mode)
 {
+        struct fcoe_ctlr_device *ctlr_dev;
         struct fcoe_ctlr *ctlr;
         struct fcoe_interface *fcoe;
         int size;
@@ -379,14 +396,17 @@ static struct fcoe_interface *fcoe_interface_create(struct net_device *netdev,
         }
 
         size = sizeof(struct fcoe_ctlr) + sizeof(struct fcoe_interface);
-        ctlr = kzalloc(size, GFP_KERNEL);
-        fcoe = fcoe_ctlr_priv(ctlr);
-        if (!fcoe) {
-                FCOE_NETDEV_DBG(netdev, "Could not allocate fcoe structure\n");
+        ctlr_dev = fcoe_ctlr_device_add(&netdev->dev, &fcoe_sysfs_templ,
+                                        size);
+        if (!ctlr_dev) {
+                FCOE_DBG("Failed to add fcoe_ctlr_device\n");
                 fcoe = ERR_PTR(-ENOMEM);
                 goto out_putmod;
         }
 
+        ctlr = fcoe_ctlr_device_priv(ctlr_dev);
+        fcoe = fcoe_ctlr_priv(ctlr);
+
         dev_hold(netdev);
 
         /*
@@ -400,6 +420,7 @@ static struct fcoe_interface *fcoe_interface_create(struct net_device *netdev,
         err = fcoe_interface_setup(fcoe, netdev);
         if (err) {
                 fcoe_ctlr_destroy(ctlr);
+                fcoe_ctlr_device_delete(ctlr_dev);
                 dev_put(netdev);
                 fcoe = ERR_PTR(err);
                 goto out_putmod;
@@ -466,6 +487,7 @@ static void fcoe_interface_cleanup(struct fcoe_interface *fcoe)
 {
         struct net_device *netdev = fcoe->netdev;
         struct fcoe_ctlr *fip = fcoe_to_ctlr(fcoe);
+        struct fcoe_ctlr_device *ctlr_dev = fcoe_ctlr_to_ctlr_dev(fip);
 
         rtnl_lock();
         if (!fcoe->removed)
@@ -476,7 +498,7 @@ static void fcoe_interface_cleanup(struct fcoe_interface *fcoe)
         /* tear-down the FCoE controller */
         fcoe_ctlr_destroy(fip);
         scsi_host_put(fip->lp->host);
-        kfree(fip);
+        fcoe_ctlr_device_delete(ctlr_dev);
         dev_put(netdev);
         module_put(THIS_MODULE);
 }
@@ -2196,6 +2218,7 @@ static void fcoe_dcb_create(struct fcoe_interface *fcoe)
 static int fcoe_create(struct net_device *netdev, enum fip_state fip_mode)
 {
         int rc = 0;
+        struct fcoe_ctlr_device *ctlr_dev;
         struct fcoe_ctlr *ctlr;
         struct fcoe_interface *fcoe;
         struct fc_lport *lport;
@@ -2216,8 +2239,8 @@ static int fcoe_create(struct net_device *netdev, enum fip_state fip_mode)
         }
 
         ctlr = fcoe_to_ctlr(fcoe);
-
-        lport = fcoe_if_create(fcoe, &netdev->dev, 0);
+        ctlr_dev = fcoe_ctlr_to_ctlr_dev(ctlr);
+        lport = fcoe_if_create(fcoe, &ctlr_dev->dev, 0);
         if (IS_ERR(lport)) {
                 printk(KERN_ERR "fcoe: Failed to create interface (%s)\n",
                        netdev->name);
@@ -2768,6 +2791,40 @@ static void fcoe_get_lesb(struct fc_lport *lport,
         __fcoe_get_lesb(lport, fc_lesb, netdev);
 }
 
+static void fcoe_ctlr_get_lesb(struct fcoe_ctlr_device *ctlr_dev)
+{
+        struct fcoe_ctlr *fip = fcoe_ctlr_device_priv(ctlr_dev);
+        struct net_device *netdev = fcoe_netdev(fip->lp);
+        struct fcoe_fc_els_lesb *fcoe_lesb;
+        struct fc_els_lesb fc_lesb;
+
+        __fcoe_get_lesb(fip->lp, &fc_lesb, netdev);
+        fcoe_lesb = (struct fcoe_fc_els_lesb *)(&fc_lesb);
+
+        ctlr_dev->lesb.lesb_link_fail =
+                ntohl(fcoe_lesb->lesb_link_fail);
+        ctlr_dev->lesb.lesb_vlink_fail =
+                ntohl(fcoe_lesb->lesb_vlink_fail);
+        ctlr_dev->lesb.lesb_miss_fka =
+                ntohl(fcoe_lesb->lesb_miss_fka);
+        ctlr_dev->lesb.lesb_symb_err =
+                ntohl(fcoe_lesb->lesb_symb_err);
+        ctlr_dev->lesb.lesb_err_block =
+                ntohl(fcoe_lesb->lesb_err_block);
+        ctlr_dev->lesb.lesb_fcs_error =
+                ntohl(fcoe_lesb->lesb_fcs_error);
+}
+
+static void fcoe_fcf_get_vlan_id(struct fcoe_fcf_device *fcf_dev)
+{
+        struct fcoe_ctlr_device *ctlr_dev =
+                fcoe_fcf_dev_to_ctlr_dev(fcf_dev);
+        struct fcoe_ctlr *ctlr = fcoe_ctlr_device_priv(ctlr_dev);
+        struct fcoe_interface *fcoe = fcoe_ctlr_priv(ctlr);
+
+        fcf_dev->vlan_id = vlan_dev_vlan_id(fcoe->netdev);
+}
+
 /**
  * fcoe_set_port_id() - Callback from libfc when Port_ID is set.
  * @lport: the local port
diff --git a/drivers/scsi/fcoe/fcoe_ctlr.c b/drivers/scsi/fcoe/fcoe_ctlr.c
index 5a4c7250aa77..d68d57241ee6 100644
--- a/drivers/scsi/fcoe/fcoe_ctlr.c
+++ b/drivers/scsi/fcoe/fcoe_ctlr.c
@@ -160,6 +160,76 @@ void fcoe_ctlr_init(struct fcoe_ctlr *fip, enum fip_state mode)
 }
 EXPORT_SYMBOL(fcoe_ctlr_init);
 
+static int fcoe_sysfs_fcf_add(struct fcoe_fcf *new)
+{
+        struct fcoe_ctlr *fip = new->fip;
+        struct fcoe_ctlr_device *ctlr_dev = fcoe_ctlr_to_ctlr_dev(fip);
+        struct fcoe_fcf_device temp, *fcf_dev;
+        int rc = 0;
+
+        LIBFCOE_FIP_DBG(fip, "New FCF fab %16.16llx mac %pM\n",
+                        new->fabric_name, new->fcf_mac);
+
+        mutex_lock(&ctlr_dev->lock);
+
+        temp.fabric_name = new->fabric_name;
+        temp.switch_name = new->switch_name;
+        temp.fc_map = new->fc_map;
+        temp.vfid = new->vfid;
+        memcpy(temp.mac, new->fcf_mac, ETH_ALEN);
+        temp.priority = new->pri;
+        temp.fka_period = new->fka_period;
+        temp.selected = 0; /* default to unselected */
+
+        fcf_dev = fcoe_fcf_device_add(ctlr_dev, &temp);
+        if (unlikely(!fcf_dev)) {
+                rc = -ENOMEM;
+                goto out;
+        }
+
+        /*
+         * The fcoe_sysfs layer can return a CONNECTED fcf that
+         * has a priv (fcf was never deleted) or a CONNECTED fcf
+         * that doesn't have a priv (fcf was deleted). However,
+         * libfcoe will always delete FCFs before trying to add
+         * them. This is ensured because both recv_adv and
+         * age_fcfs are protected by the the fcoe_ctlr's mutex.
+         * This means that we should never get a FCF with a
+         * non-NULL priv pointer.
+         */
+        BUG_ON(fcf_dev->priv);
+
+        fcf_dev->priv = new;
+        new->fcf_dev = fcf_dev;
+
+        list_add(&new->list, &fip->fcfs);
+        fip->fcf_count++;
+
+out:
+        mutex_unlock(&ctlr_dev->lock);
+        return rc;
+}
+
+static void fcoe_sysfs_fcf_del(struct fcoe_fcf *new)
+{
+        struct fcoe_ctlr *fip = new->fip;
+        struct fcoe_ctlr_device *ctlr_dev = fcoe_ctlr_to_ctlr_dev(fip);
+        struct fcoe_fcf_device *fcf_dev;
+
+        list_del(&new->list);
+        fip->fcf_count--;
+
+        mutex_lock(&ctlr_dev->lock);
+
+        fcf_dev = fcoe_fcf_to_fcf_dev(new);
+        WARN_ON(!fcf_dev);
+        new->fcf_dev = NULL;
+        fcoe_fcf_device_delete(fcf_dev);
+        kfree(new);
+
+        mutex_unlock(&ctlr_dev->lock);
+}
+
 /**
  * fcoe_ctlr_reset_fcfs() - Reset and free all FCFs for a controller
  * @fip: The FCoE controller whose FCFs are to be reset
@@ -173,10 +243,10 @@ static void fcoe_ctlr_reset_fcfs(struct fcoe_ctlr *fip)
 
         fip->sel_fcf = NULL;
         list_for_each_entry_safe(fcf, next, &fip->fcfs, list) {
-                list_del(&fcf->list);
-                kfree(fcf);
+                fcoe_sysfs_fcf_del(fcf);
         }
-        fip->fcf_count = 0;
+        WARN_ON(fip->fcf_count);
+
         fip->sel_time = 0;
 }
 
@@ -717,8 +787,11 @@ static unsigned long fcoe_ctlr_age_fcfs(struct fcoe_ctlr *fip)
         unsigned long next_timer = jiffies + msecs_to_jiffies(FIP_VN_KA_PERIOD);
         unsigned long deadline;
         unsigned long sel_time = 0;
+        struct list_head del_list;
         struct fcoe_dev_stats *stats;
 
+        INIT_LIST_HEAD(&del_list);
+
         stats = per_cpu_ptr(fip->lp->dev_stats, get_cpu());
 
         list_for_each_entry_safe(fcf, next, &fip->fcfs, list) {
@@ -739,10 +812,13 @@ static unsigned long fcoe_ctlr_age_fcfs(struct fcoe_ctlr *fip)
                 if (time_after_eq(jiffies, deadline)) {
                         if (fip->sel_fcf == fcf)
                                 fip->sel_fcf = NULL;
+                        /*
+                         * Move to delete list so we can call
+                         * fcoe_sysfs_fcf_del (which can sleep)
+                         * after the put_cpu().
+                         */
                         list_del(&fcf->list);
-                        WARN_ON(!fip->fcf_count);
-                        fip->fcf_count--;
-                        kfree(fcf);
+                        list_add(&fcf->list, &del_list);
                         stats->VLinkFailureCount++;
                 } else {
                         if (time_after(next_timer, deadline))
@@ -753,6 +829,12 @@ static unsigned long fcoe_ctlr_age_fcfs(struct fcoe_ctlr *fip)
                 }
         }
         put_cpu();
+
+        list_for_each_entry_safe(fcf, next, &del_list, list) {
+                /* Removes fcf from current list */
+                fcoe_sysfs_fcf_del(fcf);
+        }
+
         if (sel_time && !fip->sel_fcf && !fip->sel_time) {
                 sel_time += msecs_to_jiffies(FCOE_CTLR_START_DELAY);
                 fip->sel_time = sel_time;
@@ -903,23 +985,23 @@ static void fcoe_ctlr_recv_adv(struct fcoe_ctlr *fip, struct sk_buff *skb)
 {
         struct fcoe_fcf *fcf;
         struct fcoe_fcf new;
-        struct fcoe_fcf *found;
         unsigned long sol_tov = msecs_to_jiffies(FCOE_CTRL_SOL_TOV);
         int first = 0;
         int mtu_valid;
+        int found = 0;
+        int rc = 0;
 
         if (fcoe_ctlr_parse_adv(fip, skb, &new))
                 return;
 
         mutex_lock(&fip->ctlr_mutex);
         first = list_empty(&fip->fcfs);
-        found = NULL;
         list_for_each_entry(fcf, &fip->fcfs, list) {
                 if (fcf->switch_name == new.switch_name &&
                     fcf->fabric_name == new.fabric_name &&
                     fcf->fc_map == new.fc_map &&
                     compare_ether_addr(fcf->fcf_mac, new.fcf_mac) == 0) {
-                        found = fcf;
+                        found = 1;
                         break;
                 }
         }
@@ -931,9 +1013,16 @@ static void fcoe_ctlr_recv_adv(struct fcoe_ctlr *fip, struct sk_buff *skb)
                 if (!fcf)
                         goto out;
 
-                fip->fcf_count++;
                 memcpy(fcf, &new, sizeof(new));
-                list_add(&fcf->list, &fip->fcfs);
+                fcf->fip = fip;
+                rc = fcoe_sysfs_fcf_add(fcf);
+                if (rc) {
+                        printk(KERN_ERR "Failed to allocate sysfs instance "
+                               "for FCF, fab %16.16llx mac %pM\n",
+                               new.fabric_name, new.fcf_mac);
+                        kfree(fcf);
+                        goto out;
+                }
         } else {
                 /*
                  * Update the FCF's keep-alive descriptor flags.
@@ -954,6 +1043,7 @@ static void fcoe_ctlr_recv_adv(struct fcoe_ctlr *fip, struct sk_buff *skb)
                 fcf->fka_period = new.fka_period;
                 memcpy(fcf->fcf_mac, new.fcf_mac, ETH_ALEN);
         }
+
         mtu_valid = fcoe_ctlr_mtu_valid(fcf);
         fcf->time = jiffies;
         if (!found)
@@ -996,6 +1086,7 @@ static void fcoe_ctlr_recv_adv(struct fcoe_ctlr *fip, struct sk_buff *skb)
                     time_before(fip->sel_time, fip->timer.expires))
                         mod_timer(&fip->timer, fip->sel_time);
         }
+
 out:
         mutex_unlock(&fip->ctlr_mutex);
 }
@@ -2718,9 +2809,9 @@ unlock:
 
 /**
  * fcoe_libfc_config() - Sets up libfc related properties for local port
- * @lp: The local port to configure libfc for
- * @fip: The FCoE controller in use by the local port
- * @tt: The libfc function template
+ * @lport:    The local port to configure libfc for
+ * @fip:      The FCoE controller in use by the local port
+ * @tt:       The libfc function template
  * @init_fcp: If non-zero, the FCP portion of libfc should be initialized
  *
  * Returns : 0 for success
@@ -2753,3 +2844,43 @@ int fcoe_libfc_config(struct fc_lport *lport, struct fcoe_ctlr *fip,
         return 0;
 }
 EXPORT_SYMBOL_GPL(fcoe_libfc_config);
+
+void fcoe_fcf_get_selected(struct fcoe_fcf_device *fcf_dev)
+{
+        struct fcoe_ctlr_device *ctlr_dev = fcoe_fcf_dev_to_ctlr_dev(fcf_dev);
+        struct fcoe_ctlr *fip = fcoe_ctlr_device_priv(ctlr_dev);
+        struct fcoe_fcf *fcf;
+
+        mutex_lock(&fip->ctlr_mutex);
+        mutex_lock(&ctlr_dev->lock);
+
+        fcf = fcoe_fcf_device_priv(fcf_dev);
+        if (fcf)
+                fcf_dev->selected = (fcf == fip->sel_fcf) ? 1 : 0;
+        else
+                fcf_dev->selected = 0;
+
+        mutex_unlock(&ctlr_dev->lock);
+        mutex_unlock(&fip->ctlr_mutex);
+}
+EXPORT_SYMBOL(fcoe_fcf_get_selected);
+
+void fcoe_ctlr_get_fip_mode(struct fcoe_ctlr_device *ctlr_dev)
+{
+        struct fcoe_ctlr *ctlr = fcoe_ctlr_device_priv(ctlr_dev);
+
+        mutex_lock(&ctlr->ctlr_mutex);
+        switch (ctlr->mode) {
+        case FIP_MODE_FABRIC:
+                ctlr_dev->mode = FIP_CONN_TYPE_FABRIC;
+                break;
+        case FIP_MODE_VN2VN:
+                ctlr_dev->mode = FIP_CONN_TYPE_VN2VN;
+                break;
+        default:
+                ctlr_dev->mode = FIP_CONN_TYPE_UNKNOWN;
+                break;
+        }
+        mutex_unlock(&ctlr->ctlr_mutex);
+}
+EXPORT_SYMBOL(fcoe_ctlr_get_fip_mode);